From 7113c80289d62de95bca818109324601a51fefd8 Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Fri, 15 Dec 2023 10:26:01 -0800 Subject: [PATCH 001/884] [lldb][NFCI] Remove unused parameter from BreakpointResolver*::CreateFromStructuredData (#75374) These appear to be unused. --- .../lldb/Breakpoint/BreakpointResolverAddress.h | 3 +-- .../lldb/Breakpoint/BreakpointResolverFileLine.h | 3 +-- .../lldb/Breakpoint/BreakpointResolverFileRegex.h | 3 +-- lldb/include/lldb/Breakpoint/BreakpointResolverName.h | 3 +-- .../lldb/Breakpoint/BreakpointResolverScripted.h | 3 +-- lldb/source/Breakpoint/BreakpointResolver.cpp | 10 +++++----- lldb/source/Breakpoint/BreakpointResolverAddress.cpp | 5 ++--- lldb/source/Breakpoint/BreakpointResolverFileLine.cpp | 5 ++--- lldb/source/Breakpoint/BreakpointResolverFileRegex.cpp | 5 ++--- lldb/source/Breakpoint/BreakpointResolverName.cpp | 8 ++++---- lldb/source/Breakpoint/BreakpointResolverScripted.cpp | 7 +++---- 11 files changed, 23 insertions(+), 32 deletions(-) diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverAddress.h b/lldb/include/lldb/Breakpoint/BreakpointResolverAddress.h index 03ae69acae4c4..3a09892f3f194 100644 --- a/lldb/include/lldb/Breakpoint/BreakpointResolverAddress.h +++ b/lldb/include/lldb/Breakpoint/BreakpointResolverAddress.h @@ -31,8 +31,7 @@ class BreakpointResolverAddress : public BreakpointResolver { ~BreakpointResolverAddress() override = default; static lldb::BreakpointResolverSP - CreateFromStructuredData(const lldb::BreakpointSP &bkpt, - const StructuredData::Dictionary &options_dict, + CreateFromStructuredData(const StructuredData::Dictionary &options_dict, Status &error); StructuredData::ObjectSP SerializeToStructuredData() override; diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverFileLine.h b/lldb/include/lldb/Breakpoint/BreakpointResolverFileLine.h index 7635729c50a6e..610d81727c6c6 100644 --- a/lldb/include/lldb/Breakpoint/BreakpointResolverFileLine.h +++ b/lldb/include/lldb/Breakpoint/BreakpointResolverFileLine.h @@ -28,8 +28,7 @@ class BreakpointResolverFileLine : public BreakpointResolver { std::optional removed_prefix_opt = std::nullopt); static lldb::BreakpointResolverSP - CreateFromStructuredData(const lldb::BreakpointSP &bkpt, - const StructuredData::Dictionary &data_dict, + CreateFromStructuredData(const StructuredData::Dictionary &data_dict, Status &error); StructuredData::ObjectSP SerializeToStructuredData() override; diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverFileRegex.h b/lldb/include/lldb/Breakpoint/BreakpointResolverFileRegex.h index 43e1217c13d5e..1dcdba91a5a8d 100644 --- a/lldb/include/lldb/Breakpoint/BreakpointResolverFileRegex.h +++ b/lldb/include/lldb/Breakpoint/BreakpointResolverFileRegex.h @@ -28,8 +28,7 @@ class BreakpointResolverFileRegex : public BreakpointResolver { const std::unordered_set &func_name_set, bool exact_match); static lldb::BreakpointResolverSP - CreateFromStructuredData(const lldb::BreakpointSP &bkpt, - const StructuredData::Dictionary &options_dict, + CreateFromStructuredData(const StructuredData::Dictionary &options_dict, Status &error); StructuredData::ObjectSP SerializeToStructuredData() override; diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverName.h b/lldb/include/lldb/Breakpoint/BreakpointResolverName.h index 94b19db3085d7..c83814c174e88 100644 --- a/lldb/include/lldb/Breakpoint/BreakpointResolverName.h +++ b/lldb/include/lldb/Breakpoint/BreakpointResolverName.h @@ -51,8 +51,7 @@ class BreakpointResolverName : public BreakpointResolver { bool skip_prologue); static lldb::BreakpointResolverSP - CreateFromStructuredData(const lldb::BreakpointSP &bkpt, - const StructuredData::Dictionary &data_dict, + CreateFromStructuredData(const StructuredData::Dictionary &data_dict, Status &error); StructuredData::ObjectSP SerializeToStructuredData() override; diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h index c0bbc1c2bafb7..133fa8058637b 100644 --- a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h +++ b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h @@ -31,8 +31,7 @@ class BreakpointResolverScripted : public BreakpointResolver { ~BreakpointResolverScripted() override = default; static lldb::BreakpointResolverSP - CreateFromStructuredData(const lldb::BreakpointSP &bkpt, - const StructuredData::Dictionary &options_dict, + CreateFromStructuredData(const StructuredData::Dictionary &options_dict, Status &error); StructuredData::ObjectSP SerializeToStructuredData() override; diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp index 60406bdc44625..89ea308b1eb07 100644 --- a/lldb/source/Breakpoint/BreakpointResolver.cpp +++ b/lldb/source/Breakpoint/BreakpointResolver.cpp @@ -113,23 +113,23 @@ BreakpointResolverSP BreakpointResolver::CreateFromStructuredData( switch (resolver_type) { case FileLineResolver: result_sp = BreakpointResolverFileLine::CreateFromStructuredData( - nullptr, *subclass_options, error); + *subclass_options, error); break; case AddressResolver: result_sp = BreakpointResolverAddress::CreateFromStructuredData( - nullptr, *subclass_options, error); + *subclass_options, error); break; case NameResolver: result_sp = BreakpointResolverName::CreateFromStructuredData( - nullptr, *subclass_options, error); + *subclass_options, error); break; case FileRegexResolver: result_sp = BreakpointResolverFileRegex::CreateFromStructuredData( - nullptr, *subclass_options, error); + *subclass_options, error); break; case PythonResolver: result_sp = BreakpointResolverScripted::CreateFromStructuredData( - nullptr, *subclass_options, error); + *subclass_options, error); break; case ExceptionResolver: error.SetErrorString("Exception resolvers are hard."); diff --git a/lldb/source/Breakpoint/BreakpointResolverAddress.cpp b/lldb/source/Breakpoint/BreakpointResolverAddress.cpp index 19fc81a28a7f4..a0c628a8e299c 100644 --- a/lldb/source/Breakpoint/BreakpointResolverAddress.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverAddress.cpp @@ -31,8 +31,7 @@ BreakpointResolverAddress::BreakpointResolverAddress(const BreakpointSP &bkpt, m_addr(addr), m_resolved_addr(LLDB_INVALID_ADDRESS) {} BreakpointResolverSP BreakpointResolverAddress::CreateFromStructuredData( - const BreakpointSP &bkpt, const StructuredData::Dictionary &options_dict, - Status &error) { + const StructuredData::Dictionary &options_dict, Status &error) { llvm::StringRef module_name; lldb::offset_t addr_offset; FileSpec module_filespec; @@ -56,7 +55,7 @@ BreakpointResolverSP BreakpointResolverAddress::CreateFromStructuredData( } module_filespec.SetFile(module_name, FileSpec::Style::native); } - return std::make_shared(bkpt, address, + return std::make_shared(nullptr, address, module_filespec); } diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp index c28b78a0056f9..61bef498438bd 100644 --- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp @@ -31,8 +31,7 @@ BreakpointResolverFileLine::BreakpointResolverFileLine( m_removed_prefix_opt(removed_prefix_opt) {} BreakpointResolverSP BreakpointResolverFileLine::CreateFromStructuredData( - const BreakpointSP &bkpt, const StructuredData::Dictionary &options_dict, - Status &error) { + const StructuredData::Dictionary &options_dict, Status &error) { llvm::StringRef filename; uint32_t line; uint16_t column; @@ -91,7 +90,7 @@ BreakpointResolverSP BreakpointResolverFileLine::CreateFromStructuredData( return nullptr; return std::make_shared( - bkpt, offset, skip_prologue, location_spec); + nullptr, offset, skip_prologue, location_spec); } StructuredData::ObjectSP diff --git a/lldb/source/Breakpoint/BreakpointResolverFileRegex.cpp b/lldb/source/Breakpoint/BreakpointResolverFileRegex.cpp index 13c7f17fd807e..06d346afb9f62 100644 --- a/lldb/source/Breakpoint/BreakpointResolverFileRegex.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverFileRegex.cpp @@ -27,7 +27,6 @@ BreakpointResolverFileRegex::BreakpointResolverFileRegex( m_function_names(func_names) {} BreakpointResolverSP BreakpointResolverFileRegex::CreateFromStructuredData( - const lldb::BreakpointSP &bkpt, const StructuredData::Dictionary &options_dict, Status &error) { bool success; @@ -67,8 +66,8 @@ BreakpointResolverSP BreakpointResolverFileRegex::CreateFromStructuredData( } } - return std::make_shared(bkpt, std::move(regex), - names_set, exact_match); + return std::make_shared( + nullptr, std::move(regex), names_set, exact_match); } StructuredData::ObjectSP diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp index 0097046cf511b..82eef43ad6cfd 100644 --- a/lldb/source/Breakpoint/BreakpointResolverName.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp @@ -87,8 +87,7 @@ BreakpointResolverName::BreakpointResolverName( m_language(rhs.m_language), m_skip_prologue(rhs.m_skip_prologue) {} BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData( - const BreakpointSP &bkpt, const StructuredData::Dictionary &options_dict, - Status &error) { + const StructuredData::Dictionary &options_dict, Status &error) { LanguageType language = eLanguageTypeUnknown; llvm::StringRef language_name; bool success = options_dict.GetValueForKeyAsString( @@ -123,7 +122,8 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData( GetKey(OptionNames::RegexString), regex_text); if (success) { return std::make_shared( - bkpt, RegularExpression(regex_text), language, offset, skip_prologue); + nullptr, RegularExpression(regex_text), language, offset, + skip_prologue); } else { StructuredData::Array *names_array; success = options_dict.GetValueForKeyAsArray( @@ -173,7 +173,7 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData( std::shared_ptr resolver_sp = std::make_shared( - bkpt, names[0].c_str(), name_masks[0], language, + nullptr, names[0].c_str(), name_masks[0], language, Breakpoint::MatchType::Exact, offset, skip_prologue); for (size_t i = 1; i < num_elem; i++) { resolver_sp->AddNameLookup(ConstString(names[i]), name_masks[i]); diff --git a/lldb/source/Breakpoint/BreakpointResolverScripted.cpp b/lldb/source/Breakpoint/BreakpointResolverScripted.cpp index 664ce4d573f94..dbfa8b8572f2a 100644 --- a/lldb/source/Breakpoint/BreakpointResolverScripted.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverScripted.cpp @@ -59,8 +59,7 @@ void BreakpointResolverScripted::NotifyBreakpointSet() { } BreakpointResolverSP BreakpointResolverScripted::CreateFromStructuredData( - const BreakpointSP &bkpt, const StructuredData::Dictionary &options_dict, - Status &error) { + const StructuredData::Dictionary &options_dict, Status &error) { llvm::StringRef class_name; bool success; @@ -79,8 +78,8 @@ BreakpointResolverSP BreakpointResolverScripted::CreateFromStructuredData( if (options_dict.GetValueForKeyAsDictionary(GetKey(OptionNames::ScriptArgs), args_dict)) args_data_impl.SetObjectSP(args_dict->shared_from_this()); - return std::make_shared(bkpt, class_name, depth, - args_data_impl); + return std::make_shared(nullptr, class_name, + depth, args_data_impl); } StructuredData::ObjectSP From 59f7f35a9047cccded7b8d3a01926e03f1e10efa Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Fri, 15 Dec 2023 19:12:33 +0100 Subject: [PATCH 002/884] [SystemZ] ABI support for single-element vector types Support passing and returning values of single-element vector types (i.e. <1 x i128> and <1 x fp128>). Now that i128 is a legal type, supporting these types can be done simply by providing a getRegisterTypeForCallingConv implementation that handles them. Fixes https://github.com/llvm/llvm-project/issues/61291 --- .../Target/SystemZ/SystemZISelLowering.cpp | 36 ------ llvm/lib/Target/SystemZ/SystemZISelLowering.h | 9 ++ llvm/test/CodeGen/SystemZ/vec-add-01.ll | 9 ++ llvm/test/CodeGen/SystemZ/vec-add-02.ll | 10 ++ llvm/test/CodeGen/SystemZ/vec-args-08.ll | 119 ++++++++++++++++++ .../test/CodeGen/SystemZ/vec-args-error-01.ll | 9 -- .../test/CodeGen/SystemZ/vec-args-error-02.ll | 9 -- .../test/CodeGen/SystemZ/vec-args-error-03.ll | 12 -- .../test/CodeGen/SystemZ/vec-args-error-04.ll | 12 -- .../test/CodeGen/SystemZ/vec-args-error-05.ll | 9 -- .../test/CodeGen/SystemZ/vec-args-error-06.ll | 9 -- .../test/CodeGen/SystemZ/vec-args-error-07.ll | 12 -- .../test/CodeGen/SystemZ/vec-args-error-08.ll | 12 -- .../test/CodeGen/SystemZ/vec-strict-add-02.ll | 15 +++ .../test/CodeGen/SystemZ/vec-strict-sub-02.ll | 14 +++ llvm/test/CodeGen/SystemZ/vec-sub-01.ll | 9 ++ llvm/test/CodeGen/SystemZ/vec-sub-02.ll | 10 ++ 17 files changed, 195 insertions(+), 120 deletions(-) create mode 100644 llvm/test/CodeGen/SystemZ/vec-args-08.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-01.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-02.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-03.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-04.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-05.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-06.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-07.ll delete mode 100644 llvm/test/CodeGen/SystemZ/vec-args-error-08.ll diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index d82910a0b2177..a1803cf9a042f 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -1429,24 +1429,6 @@ bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { return CI->isTailCall(); } -// We do not yet support 128-bit single-element vector types. If the user -// attempts to use such types as function argument or return type, prefer -// to error out instead of emitting code violating the ABI. -static void VerifyVectorType(MVT VT, EVT ArgVT) { - if (ArgVT.isVector() && !VT.isVector()) - report_fatal_error("Unsupported vector argument or return type"); -} - -static void VerifyVectorTypes(const SmallVectorImpl &Ins) { - for (unsigned i = 0; i < Ins.size(); ++i) - VerifyVectorType(Ins[i].VT, Ins[i].ArgVT); -} - -static void VerifyVectorTypes(const SmallVectorImpl &Outs) { - for (unsigned i = 0; i < Outs.size(); ++i) - VerifyVectorType(Outs[i].VT, Outs[i].ArgVT); -} - // Value is a value that has been passed to us in the location described by VA // (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining // any loads onto Chain. @@ -1586,10 +1568,6 @@ SDValue SystemZTargetLowering::LowerFormalArguments( auto *TFL = Subtarget.getFrameLowering(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); - // Detect unsupported vector argument types. - if (Subtarget.hasVector()) - VerifyVectorTypes(Ins); - // Assign locations to all of the incoming arguments. SmallVector ArgLocs; SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); @@ -1890,12 +1868,6 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, if (Subtarget.isTargetXPLINK64()) IsTailCall = false; - // Detect unsupported vector argument and return types. - if (Subtarget.hasVector()) { - VerifyVectorTypes(Outs); - VerifyVectorTypes(Ins); - } - // Analyze the operands of the call, assigning locations to each operand. SmallVector ArgLocs; SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx); @@ -2139,10 +2111,6 @@ CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl &Outs, LLVMContext &Context) const { - // Detect unsupported vector return types. - if (Subtarget.hasVector()) - VerifyVectorTypes(Outs); - // Special case that we cannot easily detect in RetCC_SystemZ since // i128 may not be a legal type. for (auto &Out : Outs) @@ -2162,10 +2130,6 @@ SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, const SDLoc &DL, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); - // Detect unsupported vector return types. - if (Subtarget.hasVector()) - VerifyVectorTypes(Outs); - // Assign locations to each returned value. SmallVector RetLocs; CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 3e614a1186b26..6b3ce3f8c1d2b 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -443,6 +443,15 @@ class SystemZTargetLowering : public TargetLowering { return 1; return TargetLowering::getNumRegisters(Context, VT); } + MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, + EVT VT) const override { + // 128-bit single-element vector types are passed like other vectors, + // not like their element type. + if (VT.isVector() && VT.getSizeInBits() == 128 && + VT.getVectorNumElements() == 1) + return MVT::v16i8; + return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); + } bool isCheapToSpeculateCtlz(Type *) const override { return true; } bool isCheapToSpeculateCttz(Type *) const override { return true; } bool preferZeroCompareBranch() const override { return true; } diff --git a/llvm/test/CodeGen/SystemZ/vec-add-01.ll b/llvm/test/CodeGen/SystemZ/vec-add-01.ll index 317034377671f..a6fde5f32d1ba 100644 --- a/llvm/test/CodeGen/SystemZ/vec-add-01.ll +++ b/llvm/test/CodeGen/SystemZ/vec-add-01.ll @@ -58,3 +58,12 @@ define double @f6(<2 x double> %val1, <2 x double> %val2) { %ret = fadd double %scalar1, %scalar2 ret double %ret } + +; Test a v1i128 addition. +define <1 x i128> @f7(<1 x i128> %dummy, <1 x i128> %val1, <1 x i128> %val2) { +; CHECK-LABEL: f7: +; CHECK: vaq %v24, %v26, %v28 +; CHECK: br %r14 + %ret = add <1 x i128> %val1, %val2 + ret <1 x i128> %ret +} diff --git a/llvm/test/CodeGen/SystemZ/vec-add-02.ll b/llvm/test/CodeGen/SystemZ/vec-add-02.ll index 97a9b84a063c4..9b37fd964aec6 100644 --- a/llvm/test/CodeGen/SystemZ/vec-add-02.ll +++ b/llvm/test/CodeGen/SystemZ/vec-add-02.ll @@ -22,3 +22,13 @@ define float @f2(<4 x float> %val1, <4 x float> %val2) { %ret = fadd float %scalar1, %scalar2 ret float %ret } + +; Test a v1f128 addition. +define <1 x fp128> @f3(<1 x fp128> %dummy, <1 x fp128> %val1, + <1 x fp128> %val2) { +; CHECK-LABEL: f3: +; CHECK: wfaxb %v24, %v26, %v28 +; CHECK: br %r14 + %ret = fadd <1 x fp128> %val1, %val2 + ret <1 x fp128> %ret +} diff --git a/llvm/test/CodeGen/SystemZ/vec-args-08.ll b/llvm/test/CodeGen/SystemZ/vec-args-08.ll new file mode 100644 index 0000000000000..96ef7db06849a --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/vec-args-08.ll @@ -0,0 +1,119 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; Verify that we handle single-element vector types correctly. + +; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s + +define void @f1(<1 x i128> %a, ptr %ptr) { +; CHECK-LABEL: f1: +; CHECK: # %bb.0: +; CHECK-NEXT: vst %v24, 0(%r2), 3 +; CHECK-NEXT: br %r14 + store <1 x i128> %a, ptr %ptr + ret void +} + +define <1 x i128> @f2() { +; CHECK-LABEL: f2: +; CHECK: # %bb.0: +; CHECK-NEXT: vgbm %v24, 0 +; CHECK-NEXT: br %r14 + ret <1 x i128> +} + +declare void @bar3(<1 x i128>) + +define void @f3() { +; CHECK-LABEL: f3: +; CHECK: # %bb.0: +; CHECK-NEXT: stmg %r14, %r15, 112(%r15) +; CHECK-NEXT: .cfi_offset %r14, -48 +; CHECK-NEXT: .cfi_offset %r15, -40 +; CHECK-NEXT: aghi %r15, -160 +; CHECK-NEXT: .cfi_def_cfa_offset 320 +; CHECK-NEXT: vgbm %v24, 0 +; CHECK-NEXT: brasl %r14, bar3@PLT +; CHECK-NEXT: lmg %r14, %r15, 272(%r15) +; CHECK-NEXT: br %r14 + call void @bar3 (<1 x i128> ) + ret void +} + +declare <1 x i128> @bar4() + +define void @f4(ptr %ptr) { +; CHECK-LABEL: f4: +; CHECK: # %bb.0: +; CHECK-NEXT: stmg %r13, %r15, 104(%r15) +; CHECK-NEXT: .cfi_offset %r13, -56 +; CHECK-NEXT: .cfi_offset %r14, -48 +; CHECK-NEXT: .cfi_offset %r15, -40 +; CHECK-NEXT: aghi %r15, -160 +; CHECK-NEXT: .cfi_def_cfa_offset 320 +; CHECK-NEXT: lgr %r13, %r2 +; CHECK-NEXT: brasl %r14, bar4@PLT +; CHECK-NEXT: vst %v24, 0(%r13), 3 +; CHECK-NEXT: lmg %r13, %r15, 264(%r15) +; CHECK-NEXT: br %r14 + %res = call <1 x i128> @bar4 () + store <1 x i128> %res, ptr %ptr + ret void +} + +define void @f5(<1 x fp128> %a, ptr %ptr) { +; CHECK-LABEL: f5: +; CHECK: # %bb.0: +; CHECK-NEXT: vst %v24, 0(%r2), 3 +; CHECK-NEXT: br %r14 + store <1 x fp128> %a, ptr %ptr + ret void +} + +define <1 x fp128> @f6() { +; CHECK-LABEL: f6: +; CHECK: # %bb.0: +; CHECK-NEXT: lzxr %f0 +; CHECK-NEXT: vmrhg %v24, %v0, %v2 +; CHECK-NEXT: br %r14 + ret <1 x fp128> +} + +declare void @bar7(<1 x fp128>) + +define void @f7() { +; CHECK-LABEL: f7: +; CHECK: # %bb.0: +; CHECK-NEXT: stmg %r14, %r15, 112(%r15) +; CHECK-NEXT: .cfi_offset %r14, -48 +; CHECK-NEXT: .cfi_offset %r15, -40 +; CHECK-NEXT: aghi %r15, -160 +; CHECK-NEXT: .cfi_def_cfa_offset 320 +; CHECK-NEXT: lzxr %f0 +; CHECK-NEXT: vmrhg %v24, %v0, %v2 +; CHECK-NEXT: brasl %r14, bar7@PLT +; CHECK-NEXT: lmg %r14, %r15, 272(%r15) +; CHECK-NEXT: br %r14 + call void @bar7 (<1 x fp128> ) + ret void +} + +declare <1 x fp128> @bar8() + +define void @f8(ptr %ptr) { +; CHECK-LABEL: f8: +; CHECK: # %bb.0: +; CHECK-NEXT: stmg %r13, %r15, 104(%r15) +; CHECK-NEXT: .cfi_offset %r13, -56 +; CHECK-NEXT: .cfi_offset %r14, -48 +; CHECK-NEXT: .cfi_offset %r15, -40 +; CHECK-NEXT: aghi %r15, -160 +; CHECK-NEXT: .cfi_def_cfa_offset 320 +; CHECK-NEXT: lgr %r13, %r2 +; CHECK-NEXT: brasl %r14, bar8@PLT +; CHECK-NEXT: vst %v24, 0(%r13), 3 +; CHECK-NEXT: lmg %r13, %r15, 264(%r15) +; CHECK-NEXT: br %r14 + %res = call <1 x fp128> @bar8 () + store <1 x fp128> %res, ptr %ptr + ret void +} + diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-01.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-01.ll deleted file mode 100644 index 5680873fb8ee6..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-01.ll +++ /dev/null @@ -1,9 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -define void @foo(<1 x i128>) { - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-02.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-02.ll deleted file mode 100644 index 7c0efe5b8afe2..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-02.ll +++ /dev/null @@ -1,9 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -define <1 x i128> @foo() { - ret <1 x i128> -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-03.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-03.ll deleted file mode 100644 index 7c8be0136345c..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-03.ll +++ /dev/null @@ -1,12 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -declare void @bar(<1 x i128>) - -define void @foo() { - call void @bar (<1 x i128> ) - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-04.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-04.ll deleted file mode 100644 index f0b248c934867..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-04.ll +++ /dev/null @@ -1,12 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -declare <1 x i128> @bar() - -define void @foo() { - %res = call <1 x i128> @bar () - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-05.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-05.ll deleted file mode 100644 index c04095e7a737f..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-05.ll +++ /dev/null @@ -1,9 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -define void @foo(<1 x fp128>) { - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-06.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-06.ll deleted file mode 100644 index 73891be036a80..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-06.ll +++ /dev/null @@ -1,9 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -define <1 x fp128> @foo() { - ret <1 x fp128> -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-07.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-07.ll deleted file mode 100644 index 4914217f00264..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-07.ll +++ /dev/null @@ -1,12 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -declare void @bar(<1 x fp128>) - -define void @foo() { - call void @bar (<1 x fp128> ) - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-args-error-08.ll b/llvm/test/CodeGen/SystemZ/vec-args-error-08.ll deleted file mode 100644 index 8670b8fa4c785..0000000000000 --- a/llvm/test/CodeGen/SystemZ/vec-args-error-08.ll +++ /dev/null @@ -1,12 +0,0 @@ -; Verify that we detect unsupported single-element vector types. - -; RUN: not --crash llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 2>&1 | FileCheck %s - -declare <1 x fp128> @bar() - -define void @foo() { - %res = call <1 x fp128> @bar () - ret void -} - -; CHECK: LLVM ERROR: Unsupported vector argument or return type diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll index 7cdd6383178d7..fc3a8cbcad474 100644 --- a/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll +++ b/llvm/test/CodeGen/SystemZ/vec-strict-add-02.ll @@ -4,6 +4,7 @@ declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata) declare <4 x float> @llvm.experimental.constrained.fadd.v4f32(<4 x float>, <4 x float>, metadata, metadata) +declare <1 x fp128> @llvm.experimental.constrained.fadd.v1f128(<1 x fp128>, <1 x fp128>, metadata, metadata) ; Test a v4f32 addition. define <4 x float> @f1(<4 x float> %dummy, <4 x float> %val1, @@ -31,3 +32,17 @@ define float @f2(<4 x float> %val1, <4 x float> %val2) strictfp { metadata !"fpexcept.strict") strictfp ret float %ret } + +; Test a v1f128 addition. +define <1 x fp128> @f3(<1 x fp128> %dummy, <1 x fp128> %val1, + <1 x fp128> %val2) strictfp { +; CHECK-LABEL: f3: +; CHECK: wfaxb %v24, %v26, %v28 +; CHECK: br %r14 + %ret = call <1 x fp128> @llvm.experimental.constrained.fadd.v1f128( + <1 x fp128> %val1, <1 x fp128> %val2, + metadata !"round.dynamic", + metadata !"fpexcept.strict") strictfp + ret <1 x fp128> %ret +} + diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll b/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll index fc93e6a091855..89d810fbf5c5b 100644 --- a/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll +++ b/llvm/test/CodeGen/SystemZ/vec-strict-sub-02.ll @@ -4,6 +4,7 @@ declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata) declare <4 x float> @llvm.experimental.constrained.fsub.v4f32(<4 x float>, <4 x float>, metadata, metadata) +declare <1 x fp128> @llvm.experimental.constrained.fsub.v1f128(<1 x fp128>, <1 x fp128>, metadata, metadata) ; Test a v4f32 subtraction. define <4 x float> @f6(<4 x float> %dummy, <4 x float> %val1, @@ -32,4 +33,17 @@ define float @f7(<4 x float> %val1, <4 x float> %val2) #0 { ret float %ret } +; Test a v1f128 subtraction. +define <1 x fp128> @f8(<1 x fp128> %dummy, <1 x fp128> %val1, + <1 x fp128> %val2) #0 { +; CHECK-LABEL: f8: +; CHECK: wfsxb %v24, %v26, %v28 +; CHECK: br %r14 + %ret = call <1 x fp128> @llvm.experimental.constrained.fsub.v1f128( + <1 x fp128> %val1, <1 x fp128> %val2, + metadata !"round.dynamic", + metadata !"fpexcept.strict") #0 + ret <1 x fp128> %ret +} + attributes #0 = { strictfp } diff --git a/llvm/test/CodeGen/SystemZ/vec-sub-01.ll b/llvm/test/CodeGen/SystemZ/vec-sub-01.ll index cc276c3b76978..e1e08ebaaef47 100644 --- a/llvm/test/CodeGen/SystemZ/vec-sub-01.ll +++ b/llvm/test/CodeGen/SystemZ/vec-sub-01.ll @@ -145,3 +145,12 @@ define <2 x float> @f14(<2 x float> %val1, <2 x float> %val2) { %ret = fsub <2 x float> %val1, %val2 ret <2 x float> %ret } + +; Test a v1i128 subtraction. +define <1 x i128> @f15(<1 x i128> %dummy, <1 x i128> %val1, <1 x i128> %val2) { +; CHECK-LABEL: f15: +; CHECK: vsq %v24, %v26, %v28 +; CHECK: br %r14 + %ret = sub <1 x i128> %val1, %val2 + ret <1 x i128> %ret +} diff --git a/llvm/test/CodeGen/SystemZ/vec-sub-02.ll b/llvm/test/CodeGen/SystemZ/vec-sub-02.ll index 83c76b5d4aa61..b7d502d5f75aa 100644 --- a/llvm/test/CodeGen/SystemZ/vec-sub-02.ll +++ b/llvm/test/CodeGen/SystemZ/vec-sub-02.ll @@ -29,3 +29,13 @@ define <2 x float> @f14(<2 x float> %val1, <2 x float> %val2) { %ret = fsub <2 x float> %val1, %val2 ret <2 x float> %ret } + +; Test a v1f128 subtraction. +define <1 x fp128> @f15(<1 x fp128> %dummy, <1 x fp128> %val1, + <1 x fp128> %val2) { +; CHECK-LABEL: f15: +; CHECK: wfsxb %v24, %v26, %v28 +; CHECK: br %r14 + %ret = fsub <1 x fp128> %val1, %val2 + ret <1 x fp128> %ret +} From 0b7dda3d4cbe6a4180fd80f91e9f29e474c1d896 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Fri, 15 Dec 2023 10:47:26 -0800 Subject: [PATCH 003/884] Revert "[flang][nfc] Refactor linker invocation logic (#75534)" This reverts commit 71bbfabd08d90a3007f6034e420daa66c41027db. Breaks check-flang on x86_64 host. --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 136 +++++++++------------ flang/test/Driver/linker-flags.f90 | 8 +- 2 files changed, 65 insertions(+), 79 deletions(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 6de41642a734a..3d1df58190ce0 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1116,87 +1116,73 @@ bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC, return true; } -/// Determines if --whole-archive is active in the list of arguments. -static bool isWholeArchivePresent(const ArgList &Args) { - bool WholeArchiveActive = false; - for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) { - if (Arg) { - for (StringRef ArgValue : Arg->getValues()) { - if (ArgValue == "--whole-archive") - WholeArchiveActive = true; - if (ArgValue == "--no-whole-archive") - WholeArchiveActive = false; - } - } - } - - return WholeArchiveActive; -} - -/// Add Fortran runtime libs for MSVC -static void addFortranRuntimeLibsMSVC(const ArgList &Args, - llvm::opt::ArgStringList &CmdArgs) { - unsigned RTOptionID = options::OPT__SLASH_MT; - if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { - RTOptionID = llvm::StringSwitch(rtl->getValue()) - .Case("static", options::OPT__SLASH_MT) - .Case("static_dbg", options::OPT__SLASH_MTd) - .Case("dll", options::OPT__SLASH_MD) - .Case("dll_dbg", options::OPT__SLASH_MDd) - .Default(options::OPT__SLASH_MT); - } - switch (RTOptionID) { - case options::OPT__SLASH_MT: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); - break; - case options::OPT__SLASH_MTd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); - break; - case options::OPT__SLASH_MD: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); - break; - case options::OPT__SLASH_MDd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); - break; - } -} - -/// Add Fortran runtime libs void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args, llvm::opt::ArgStringList &CmdArgs) { - // 1. Link FortranRuntime and FortranDecimal - // These are handled earlier on Windows by telling the frontend driver to - // add the correct libraries to link against as dependents in the object - // file. - if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { - CmdArgs.push_back("-lFortranRuntime"); - CmdArgs.push_back("-lFortranDecimal"); - } + // These are handled earlier on Windows by telling the frontend driver to add + // the correct libraries to link against as dependents in the object file. - // 2. Link FortranMain - // If -fno-fortran-main has been passed, skip linking Fortran_main.a - if (Args.hasArg(options::OPT_no_fortran_main)) - return; + // if -fno-fortran-main has been passed, skip linking Fortran_main.a + bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); + if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { + if (LinkFortranMain) { + // The --whole-archive option needs to be part of the link line to + // make sure that the main() function from Fortran_main.a is pulled + // in by the linker. Determine if --whole-archive is active when + // flang will try to link Fortran_main.a. If it is, don't add the + // --whole-archive flag to the link line. If it's not, add a proper + // --whole-archive/--no-whole-archive bracket to the link line. + bool WholeArchiveActive = false; + for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) { + if (Arg) { + for (StringRef ArgValue : Arg->getValues()) { + if (ArgValue == "--whole-archive") + WholeArchiveActive = true; + if (ArgValue == "--no-whole-archive") + WholeArchiveActive = false; + } + } + } - // 2.1. MSVC - if (TC.getTriple().isKnownWindowsMSVCEnvironment()) { - addFortranRuntimeLibsMSVC(Args, CmdArgs); - return; - } + // TODO: Find an equivalent of `--whole-archive` for Darwin. + if (!WholeArchiveActive && !TC.getTriple().isMacOSX()) { + CmdArgs.push_back("--whole-archive"); + CmdArgs.push_back("-lFortran_main"); + CmdArgs.push_back("--no-whole-archive"); + } else { + CmdArgs.push_back("-lFortran_main"); + } - // 2.2. GNU and similar - // The --whole-archive option needs to be part of the link line to make - // sure that the main() function from Fortran_main.a is pulled in by the - // linker. However, it shouldn't be used if it's already active. - // TODO: Find an equivalent of `--whole-archive` for Darwin. - if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX()) { - CmdArgs.push_back("--whole-archive"); - CmdArgs.push_back("-lFortran_main"); - CmdArgs.push_back("--no-whole-archive"); - return; + // Perform regular linkage of the remaining runtime libraries. + CmdArgs.push_back("-lFortranRuntime"); + CmdArgs.push_back("-lFortranDecimal"); + } + } else { + if (LinkFortranMain) { + unsigned RTOptionID = options::OPT__SLASH_MT; + if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { + RTOptionID = llvm::StringSwitch(rtl->getValue()) + .Case("static", options::OPT__SLASH_MT) + .Case("static_dbg", options::OPT__SLASH_MTd) + .Case("dll", options::OPT__SLASH_MD) + .Case("dll_dbg", options::OPT__SLASH_MDd) + .Default(options::OPT__SLASH_MT); + } + switch (RTOptionID) { + case options::OPT__SLASH_MT: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); + break; + case options::OPT__SLASH_MTd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); + break; + case options::OPT__SLASH_MD: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); + break; + case options::OPT__SLASH_MDd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); + break; + } + } } - - CmdArgs.push_back("-lFortran_main"); } void tools::addFortranRuntimeLibraryPath(const ToolChain &TC, diff --git a/flang/test/Driver/linker-flags.f90 b/flang/test/Driver/linker-flags.f90 index 31e7ab71aacdb..ea91946316cfa 100644 --- a/flang/test/Driver/linker-flags.f90 +++ b/flang/test/Driver/linker-flags.f90 @@ -28,23 +28,23 @@ ! executable and may find the GNU linker from MinGW or Cygwin. ! UNIX-LABEL: "{{.*}}ld{{(\.exe)?}}" ! UNIX-SAME: "[[object_file]]" -! UNIX-SAME: "-lFortranRuntime" "-lFortranDecimal" "--whole-archive" "-lFortran_main" "--no-whole-archive" "-lm" +! UNIX-SAME: "--whole-archive" "-lFortran_main" "--no-whole-archive" "-lFortranRuntime" "-lFortranDecimal" "-lm" ! DARWIN-LABEL: "{{.*}}ld{{(\.exe)?}}" ! DARWIN-SAME: "[[object_file]]" +! DARWIN-SAME: -lFortran_main ! DARWIN-SAME: -lFortranRuntime ! DARWIN-SAME: -lFortranDecimal -! DARWIN-SAME: -lFortran_main ! HAIKU-LABEL: "{{.*}}ld{{(\.exe)?}}" ! HAIKU-SAME: "[[object_file]]" -! HAIKU-SAME: "-lFortranRuntime" "-lFortranDecimal" "--whole-archive" "-lFortran_main" "--no-whole-archive" +! HAIKU-SAME: "--whole-archive" "-lFortran_main" "--no-whole-archive" "-lFortranRuntime" "-lFortranDecimal" ! MINGW-LABEL: "{{.*}}ld{{(\.exe)?}}" ! MINGW-SAME: "[[object_file]]" +! MINGW-SAME: -lFortran_main ! MINGW-SAME: -lFortranRuntime ! MINGW-SAME: -lFortranDecimal -! MINGW-SAME: -lFortran_main ! NOTE: This also matches lld-link (when CLANG_DEFAULT_LINKER=lld) and ! any .exe suffix that is added when resolving to the full path of From 3017adb37ec3ef0c81f4991d08804e3b6a127384 Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Fri, 15 Dec 2023 12:17:01 -0700 Subject: [PATCH 004/884] fixup! [GlobalISel] Always direct-call IFuncs and Aliases (#74902) The codegen change broke one of the BOLT tests. --- bolt/test/AArch64/ifunc.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bolt/test/AArch64/ifunc.c b/bolt/test/AArch64/ifunc.c index 8edb913ee70d5..79c035ed45373 100644 --- a/bolt/test/AArch64/ifunc.c +++ b/bolt/test/AArch64/ifunc.c @@ -6,7 +6,7 @@ // RUN: -o %t.O0.exe -Wl,-q // RUN: llvm-bolt %t.O0.exe -o %t.O0.bolt.exe \ // RUN: --print-disasm --print-only=_start | \ -// RUN: FileCheck --check-prefix=O0_CHECK %s +// RUN: FileCheck --check-prefix=CHECK %s // RUN: llvm-readelf -aW %t.O0.bolt.exe | \ // RUN: FileCheck --check-prefix=REL_CHECK %s @@ -18,7 +18,7 @@ // RUN: FileCheck --check-prefix=NON_DYN_CHECK %s // RUN: llvm-bolt %t.O3_nopie.exe -o %t.O3_nopie.bolt.exe \ // RUN: --print-disasm --print-only=_start | \ -// RUN: FileCheck --check-prefix=O3_CHECK %s +// RUN: FileCheck --check-prefix=CHECK %s // RUN: llvm-readelf -aW %t.O3_nopie.bolt.exe | \ // RUN: FileCheck --check-prefix=REL_CHECK %s @@ -29,7 +29,7 @@ // RUN: -o %t.O3_pie.exe -Wl,-q // RUN: llvm-bolt %t.O3_pie.exe -o %t.O3_pie.bolt.exe \ // RUN: --print-disasm --print-only=_start | \ -// RUN: FileCheck --check-prefix=O3_CHECK %s +// RUN: FileCheck --check-prefix=CHECK %s // RUN: llvm-readelf -aW %t.O3_pie.bolt.exe | \ // RUN: FileCheck --check-prefix=REL_CHECK %s @@ -39,14 +39,13 @@ // RUN: -T %p/Inputs/iplt.ld -o %t.iplt_O3_pie.exe -Wl,-q // RUN: llvm-bolt %t.iplt_O3_pie.exe -o %t.iplt_O3_pie.bolt.exe \ // RUN: --print-disasm --print-only=_start | \ -// RUN: FileCheck --check-prefix=O3_CHECK %s +// RUN: FileCheck --check-prefix=CHECK %s // RUN: llvm-readelf -aW %t.iplt_O3_pie.bolt.exe | \ // RUN: FileCheck --check-prefix=REL_CHECK %s // NON_DYN_CHECK-NOT: DYNAMIC -// O0_CHECK: adr x{{[0-9]+}}, ifoo -// O3_CHECK: b "{{resolver_foo|ifoo}}{{.*}}@PLT" +// CHECK: b{{l?}} "{{resolver_foo|ifoo}}{{.*}}@PLT" // REL_CHECK: R_AARCH64_IRELATIVE [[#%x,REL_SYMB_ADDR:]] // REL_CHECK: [[#REL_SYMB_ADDR]] {{.*}} FUNC {{.*}} resolver_foo From 4fa9697b478cccc6930a667acfc0d77995b8c263 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Fri, 15 Dec 2023 11:19:24 -0800 Subject: [PATCH 005/884] [RISCV][InsertVSETVLI] Factor out isNonZeroLoadImmediate helper [nfc] Just reducing a bit of code duplication. --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index b2d36b362b3a0..3400b24e0abb0 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -149,6 +149,13 @@ static std::optional getEEWForLoadStore(const MachineInstr &MI) { } } +static bool isNonZeroLoadImmediate(MachineInstr &MI) { + return MI.getOpcode() == RISCV::ADDI && + MI.getOperand(1).isReg() && MI.getOperand(2).isImm() && + MI.getOperand(1).getReg() == RISCV::X0 && + MI.getOperand(2).getImm() != 0; +} + /// Return true if this is an operation on mask registers. Note that /// this includes both arithmetic/logical ops and load/store (vlm/vsm). static bool isMaskRegOp(const MachineInstr &MI) { @@ -501,10 +508,7 @@ class VSETVLIInfo { if (getAVLReg() == RISCV::X0) return true; if (MachineInstr *MI = MRI.getVRegDef(getAVLReg()); - MI && MI->getOpcode() == RISCV::ADDI && - MI->getOperand(1).isReg() && MI->getOperand(2).isImm() && - MI->getOperand(1).getReg() == RISCV::X0 && - MI->getOperand(2).getImm() != 0) + MI && isNonZeroLoadImmediate(*MI)) return true; return false; } @@ -1461,10 +1465,7 @@ static bool isNonZeroAVL(const MachineOperand &MO, if (MO.getReg() == RISCV::X0) return true; if (MachineInstr *MI = MRI.getVRegDef(MO.getReg()); - MI && MI->getOpcode() == RISCV::ADDI && - MI->getOperand(1).isReg() && MI->getOperand(2).isImm() && - MI->getOperand(1).getReg() == RISCV::X0 && - MI->getOperand(2).getImm() != 0) + MI && isNonZeroLoadImmediate(*MI)) return true; return false; } From d6a3607ff5ebae6575a713cdf12f1a2dda7cc72f Mon Sep 17 00:00:00 2001 From: vdonaldson <37090318+vdonaldson@users.noreply.github.com> Date: Fri, 15 Dec 2023 11:21:53 -0800 Subject: [PATCH 006/884] [flang] legacy branch target (#75628) Branching to an endif statement from outside of the if is nonconformant: subroutine jump(n) goto 6 if (n == 3) then goto 7 6 end if print *, 'pass' return 7 print *, 'fail' end However, this branch was permitted up to f90. Account for this usage when rewriting if constructs and if statements by suppressing rewriting if the end statement is labeled. --- flang/lib/Lower/PFTBuilder.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp index 8e224c17edad1..0e32e2c7d96a7 100644 --- a/flang/lib/Lower/PFTBuilder.cpp +++ b/flang/lib/Lower/PFTBuilder.cpp @@ -541,15 +541,15 @@ class PFTBuilder { /// The transformation is only valid for forward branch targets at the same /// construct nesting level as the IfConstruct. The result must not violate /// construct nesting requirements or contain an EntryStmt. The result - /// is subject to normal un/structured code classification analysis. The - /// result is allowed to violate the F18 Clause 11.1.2.1 prohibition on - /// transfer of control into the interior of a construct block, as that does - /// not compromise correct code generation. When two transformation - /// candidates overlap, at least one must be disallowed. In such cases, - /// the current heuristic favors simple code generation, which happens to - /// favor later candidates over earlier candidates. That choice is probably - /// not significant, but could be changed. - /// + /// is subject to normal un/structured code classification analysis. Except + /// for a branch to the EndIfStmt, the result is allowed to violate the F18 + /// Clause 11.1.2.1 prohibition on transfer of control into the interior of + /// a construct block, as that does not compromise correct code generation. + /// When two transformation candidates overlap, at least one must be + /// disallowed. In such cases, the current heuristic favors simple code + /// generation, which happens to favor later candidates over earlier + /// candidates. That choice is probably not significant, but could be + /// changed. void rewriteIfGotos() { auto &evaluationList = *evaluationListStack.back(); if (!evaluationList.size()) @@ -616,7 +616,8 @@ class PFTBuilder { if (eval.isA() && eval.evaluationList->size() == 3) { const auto bodyEval = std::next(eval.evaluationList->begin()); if (const auto *gotoStmt = bodyEval->getIf()) { - ifCandidateStack.push_back({it, gotoStmt->v}); + if (!bodyEval->lexicalSuccessor->label) + ifCandidateStack.push_back({it, gotoStmt->v}); } else if (doStmt) { if (const auto *cycleStmt = bodyEval->getIf()) { std::string cycleName = getConstructName(*cycleStmt); From aa165edca8545b212de084d5b18c3d30347f774a Mon Sep 17 00:00:00 2001 From: Rob Suderman Date: Fri, 15 Dec 2023 11:35:40 -0800 Subject: [PATCH 007/884] [mlir][math] Added `math.sinh` with expansions to `math.exp` (#75517) Includes end-to-end tests for the cpu running, folders using `libm` and lowerings to the corresponding `libm` operations. --- mlir/include/mlir/Dialect/Math/IR/MathOps.td | 21 +++++ .../mlir/Dialect/Math/Transforms/Passes.h | 2 + mlir/lib/Conversion/MathToLibm/MathToLibm.cpp | 1 + mlir/lib/Dialect/Math/IR/MathOps.cpp | 18 ++++ .../Math/Transforms/ExpandPatterns.cpp | 40 ++++++++ .../MathToLibm/convert-to-libm.mlir | 12 +++ mlir/test/lib/Dialect/Math/TestExpandMath.cpp | 2 + .../test-expand-math-approx.mlir | 94 +++++++++++++++++++ 8 files changed, 190 insertions(+) diff --git a/mlir/include/mlir/Dialect/Math/IR/MathOps.td b/mlir/include/mlir/Dialect/Math/IR/MathOps.td index b9daa91b28a9b..211cb31d50bdc 100644 --- a/mlir/include/mlir/Dialect/Math/IR/MathOps.td +++ b/mlir/include/mlir/Dialect/Math/IR/MathOps.td @@ -375,6 +375,27 @@ def Math_SinOp : Math_FloatUnaryOp<"sin"> { let hasFolder = 1; } +//===----------------------------------------------------------------------===// +// SinhOp +//===----------------------------------------------------------------------===// + +def Math_SinhOp : Math_FloatUnaryOp<"sinh"> { + let summary = "hyperbolic sine of the specified value"; + let description = [{ + The `sinh` operation computes the hyperbolic sine. It takes one operand + of floating point type (i.e., scalar, tensor or vector) and returns one + result of the same type. It has no standard attributes. + + Example: + + ```mlir + // Scalar hyperbolic sine value. + %a = math.sinh %b : f64 + ``` + }]; + let hasFolder = 1; +} + //===----------------------------------------------------------------------===// // CountLeadingZerosOp //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h index 817d6e1dae051..9e6759ef229d6 100644 --- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h +++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h @@ -25,6 +25,8 @@ class RewritePatternSet; void populateExpandCtlzPattern(RewritePatternSet &patterns); void populateExpandTanPattern(RewritePatternSet &patterns); +void populateExpandSinhPattern(RewritePatternSet &patterns); +void populateExpandCoshPattern(RewritePatternSet &patterns); void populateExpandTanhPattern(RewritePatternSet &patterns); void populateExpandFmaFPattern(RewritePatternSet &patterns); void populateExpandFloorFPattern(RewritePatternSet &patterns); diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp index 6e30c07de4d57..80eec9b2df745 100644 --- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp +++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp @@ -177,6 +177,7 @@ void mlir::populateMathToLibmConversionPatterns(RewritePatternSet &patterns) { "roundeven"); populatePatternsForOp(patterns, ctx, "roundf", "round"); populatePatternsForOp(patterns, ctx, "sinf", "sin"); + populatePatternsForOp(patterns, ctx, "sinhf", "sinh"); populatePatternsForOp(patterns, ctx, "tanf", "tan"); populatePatternsForOp(patterns, ctx, "tanhf", "tanh"); populatePatternsForOp(patterns, ctx, "truncf", "trunc"); diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp index 6b8c3a53a422f..bac46996fce73 100644 --- a/mlir/lib/Dialect/Math/IR/MathOps.cpp +++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp @@ -180,6 +180,24 @@ OpFoldResult math::SinOp::fold(FoldAdaptor adaptor) { }); } +//===----------------------------------------------------------------------===// +// SinhOp folder +//===----------------------------------------------------------------------===// + +OpFoldResult math::SinhOp::fold(FoldAdaptor adaptor) { + return constFoldUnaryOpConditional( + adaptor.getOperands(), [](const APFloat &a) -> std::optional { + switch (a.getSizeInBits(a.getSemantics())) { + case 64: + return APFloat(sinh(a.convertToDouble())); + case 32: + return APFloat(sinhf(a.convertToFloat())); + default: + return {}; + } + }); +} + //===----------------------------------------------------------------------===// // CountLeadingZerosOp folder //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp index 9c46a4ca10a8e..989a3e5536ec6 100644 --- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp +++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp @@ -58,6 +58,38 @@ static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) { return b.create(fpFixedConvert, operand); } +// sinhf(float x) -> (exp(x) - exp(-x)) / 2 +static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) { + ImplicitLocOpBuilder b(op->getLoc(), rewriter); + Value operand = op.getOperand(); + Type opType = operand.getType(); + Value exp = b.create(operand); + + Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter); + Value nexp = b.create(one, exp); + Value sub = b.create(exp, nexp); + Value two = createFloatConst(op->getLoc(), opType, 2.0, rewriter); + Value div = b.create(sub, two); + rewriter.replaceOp(op, div); + return success(); +} + +// coshf(float x) -> (exp(x) + exp(-x)) / 2 +static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) { + ImplicitLocOpBuilder b(op->getLoc(), rewriter); + Value operand = op.getOperand(); + Type opType = operand.getType(); + Value exp = b.create(operand); + + Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter); + Value nexp = b.create(one, exp); + Value add = b.create(exp, nexp); + Value two = createFloatConst(op->getLoc(), opType, 2.0, rewriter); + Value div = b.create(add, two); + rewriter.replaceOp(op, div); + return success(); +} + /// Expands tanh op into /// 1) 1-exp^{-2x} / 1+exp^{-2x}, if x => 0 /// 2) exp^{2x}-1 / exp^{2x}+1 , if x < 0 @@ -445,6 +477,14 @@ void mlir::populateExpandCtlzPattern(RewritePatternSet &patterns) { patterns.add(convertCtlzOp); } +void mlir::populateExpandSinhPattern(RewritePatternSet &patterns) { + patterns.add(convertSinhOp); +} + +void mlir::populateExpandCoshPattern(RewritePatternSet &patterns) { + patterns.add(convertCoshOp); +} + void mlir::populateExpandTanPattern(RewritePatternSet &patterns) { patterns.add(convertTanOp); } diff --git a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir index eb9226dee2619..bfe084b6ca0ab 100644 --- a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir +++ b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir @@ -141,6 +141,18 @@ func.func @cosh_caller(%float: f32, %double: f64) -> (f32, f64) { return %float_result, %double_result : f32, f64 } +// CHECK-LABEL: func @sinh_caller +// CHECK-SAME: %[[FLOAT:.*]]: f32 +// CHECK-SAME: %[[DOUBLE:.*]]: f64 +func.func @sinh_caller(%float: f32, %double: f64) -> (f32, f64) { + // CHECK-DAG: %[[FLOAT_RESULT:.*]] = call @sinhf(%[[FLOAT]]) : (f32) -> f32 + %float_result = math.sinh %float : f32 + // CHECK-DAG: %[[DOUBLE_RESULT:.*]] = call @sinh(%[[DOUBLE]]) : (f64) -> f64 + %double_result = math.sinh %double : f64 + // CHECK: return %[[FLOAT_RESULT]], %[[DOUBLE_RESULT]] + return %float_result, %double_result : f32, f64 +} + // CHECK-LABEL: func @atan2_caller // CHECK-SAME: %[[FLOAT:.*]]: f32 // CHECK-SAME: %[[DOUBLE:.*]]: f64 diff --git a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp index 6dae8213dd41e..7ce8b5a7cfe9b 100644 --- a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp +++ b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp @@ -39,6 +39,8 @@ void TestExpandMathPass::runOnOperation() { populateExpandCtlzPattern(patterns); populateExpandExp2FPattern(patterns); populateExpandTanPattern(patterns); + populateExpandSinhPattern(patterns); + populateExpandCoshPattern(patterns); populateExpandTanhPattern(patterns); populateExpandFmaFPattern(patterns); populateExpandFloorFPattern(patterns); diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir index 3bf474ea47f37..541a201c94c58 100644 --- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir +++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir @@ -591,10 +591,104 @@ func.func @roundeven() { return } +// -------------------------------------------------------------------------- // +// Sinh. +// -------------------------------------------------------------------------- // + +func.func @sinh_f32(%a : f32) { + %r = math.sinh %a : f32 + vector.print %r : f32 + return +} + +func.func @sinh_4xf32(%a : vector<4xf32>) { + %r = math.sinh %a : vector<4xf32> + vector.print %r : vector<4xf32> + return +} + +func.func @sinh_8xf32(%a : vector<8xf32>) { + %r = math.sinh %a : vector<8xf32> + vector.print %r : vector<8xf32> + return +} + +func.func @sinh() { + // CHECK: 1.60192 + %f0 = arith.constant 1.25 : f32 + call @sinh_f32(%f0) : (f32) -> () + + // CHECK: 0.252612, 0.822317, 1.1752, 1.60192 + %v1 = arith.constant dense<[0.25, 0.75, 1.0, 1.25]> : vector<4xf32> + call @sinh_4xf32(%v1) : (vector<4xf32>) -> () + + // CHECK: 0.100167, 0.201336, 0.30452, 0.410752, 0.521095, 0.636654, 0.758584, 0.888106 + %v2 = arith.constant dense<[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]> : vector<8xf32> + call @sinh_8xf32(%v2) : (vector<8xf32>) -> () + + // CHECK: -0.100167, -0.201336, -0.30452, -0.410752, -0.521095, -0.636654, -0.758584, -0.888106 + %v3 = arith.constant dense<[-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8]> : vector<8xf32> + call @sinh_8xf32(%v3) : (vector<8xf32>) -> () + + // CHECK: nan + %nan = arith.constant 0x7fc00000 : f32 + call @sinh_f32(%nan) : (f32) -> () + + return +} + +// -------------------------------------------------------------------------- // +// Cosh. +// -------------------------------------------------------------------------- // + +func.func @cosh_f32(%a : f32) { + %r = math.cosh %a : f32 + vector.print %r : f32 + return +} + +func.func @cosh_4xf32(%a : vector<4xf32>) { + %r = math.cosh %a : vector<4xf32> + vector.print %r : vector<4xf32> + return +} + +func.func @cosh_8xf32(%a : vector<8xf32>) { + %r = math.cosh %a : vector<8xf32> + vector.print %r : vector<8xf32> + return +} + +func.func @cosh() { + // CHECK: 1.88842 + %f0 = arith.constant 1.25 : f32 + call @cosh_f32(%f0) : (f32) -> () + + // CHECK: 1.03141, 1.29468, 1.54308, 1.88842 + %v1 = arith.constant dense<[0.25, 0.75, 1.0, 1.25]> : vector<4xf32> + call @cosh_4xf32(%v1) : (vector<4xf32>) -> () + + // CHECK: 1.005, 1.02007, 1.04534, 1.08107, 1.12763, 1.18547, 1.25517, 1.33743 + %v2 = arith.constant dense<[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]> : vector<8xf32> + call @cosh_8xf32(%v2) : (vector<8xf32>) -> () + + // CHECK: 1.005, 1.02007, 1.04534, 1.08107, 1.12763, 1.18547, 1.25517, 1.33743 + %v3 = arith.constant dense<[-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8]> : vector<8xf32> + call @cosh_8xf32(%v3) : (vector<8xf32>) -> () + + // CHECK: nan + %nan = arith.constant 0x7fc00000 : f32 + call @cosh_f32(%nan) : (f32) -> () + + return +} + func.func @main() { call @exp2f() : () -> () call @roundf() : () -> () call @powf() : () -> () call @roundeven() : () -> () + call @sinh() : () -> () + call @cosh() : () -> () return } From 1a5299491a455356b4aae8ee47ceceebd00c2103 Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Fri, 15 Dec 2023 11:38:21 -0800 Subject: [PATCH 008/884] [MemProf][NFC] Free large data structures after last use (#75120) The MemProf InstrProfWriter uses a couple of MapVector for building the lists of records it needs to write. Once its entries are all added to the associated OnDiskChainedHashTableGenerator, it is no longer used. Clearing these MapVectors, which grow quite large for large profiles, saved 4G for a large memory profile. --- llvm/lib/ProfileData/InstrProfWriter.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 595c9aa1adc10..68b77a1482976 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -536,6 +536,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { // Insert the key (func hash) and value (memprof record). RecordTableGenerator.insert(I.first, I.second); } + // Release the memory of this MapVector as it is no longer needed. + MemProfRecordData.clear(); uint64_t RecordTableOffset = RecordTableGenerator.Emit(OS.OS, *RecordWriter); @@ -549,6 +551,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { // Insert the key (frame id) and value (frame contents). FrameTableGenerator.insert(I.first, I.second); } + // Release the memory of this MapVector as it is no longer needed. + MemProfFrameData.clear(); uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter); From 35a003c2b21082f3c47c8b01d9d1955af5ab098e Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Fri, 15 Dec 2023 11:38:33 -0800 Subject: [PATCH 009/884] [MemProf][NFC] Clear each IndexedMemProfRecord after it is written (#75205) The on-disk hash table for the memprof writer holds copies of all the memprof records to be written. These hold a lot of memory in aggregate, due to the lists of alloc sites (which each have a list of context frames) and call sites. Clear each one after emitting it. This drops the peak memory when writing a very large indexed memprof profile by about 2.5G. --- llvm/include/llvm/ProfileData/MemProf.h | 5 +++++ llvm/lib/ProfileData/InstrProfWriter.cpp | 3 +++ 2 files changed, 8 insertions(+) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 6557481870d8c..37c19094bc2a6 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -538,6 +538,11 @@ class RecordWriterTrait { offset_type /*Unused*/) { assert(Schema != nullptr && "MemProf schema is not initialized!"); V.serialize(*Schema, Out); + // Clear the IndexedMemProfRecord which results in clearing/freeing its + // vectors of allocs and callsites. This is owned by the associated on-disk + // hash table, but unused after this point. See also the comment added to + // the client which constructs the on-disk hash table for this trait. + V.clear(); } }; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 68b77a1482976..d65f8fe50313d 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -539,6 +539,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { // Release the memory of this MapVector as it is no longer needed. MemProfRecordData.clear(); + // The call to Emit invokes RecordWriterTrait::EmitData which destructs + // the memprof record copies owned by the RecordTableGenerator. This works + // because the RecordTableGenerator is not used after this point. uint64_t RecordTableOffset = RecordTableGenerator.Emit(OS.OS, *RecordWriter); From 8c262ed2e3a2efa455e7641ad15b8440929c3b53 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 15 Dec 2023 13:47:28 -0600 Subject: [PATCH 010/884] [NVPTX] Add ELF flags for Nvidia cubin files (#75624) Summary: Nvidia uses ELF as its file format for cubin files. This patch adds support to allow detecting the architecture using the ELF flags only. This will be used in the offloading runtime in the future. These values are completely undocumented. They were determined by manually modifying the ELF header of the cubin and checking the output of the `nvisasm` tool. --- llvm/include/llvm/BinaryFormat/ELF.h | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index da38f6ef064f9..0f968eac36e72 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -846,6 +846,49 @@ enum { #include "ELFRelocs/AMDGPU.def" }; +// NVPTX specific e_flags. +enum : unsigned { + // Processor selection mask for EF_CUDA_SM* values. + EF_CUDA_SM = 0xff, + + // SM based processor values. + EF_CUDA_SM20 = 0x14, + EF_CUDA_SM21 = 0x15, + EF_CUDA_SM30 = 0x1e, + EF_CUDA_SM32 = 0x20, + EF_CUDA_SM35 = 0x23, + EF_CUDA_SM37 = 0x25, + EF_CUDA_SM50 = 0x32, + EF_CUDA_SM52 = 0x34, + EF_CUDA_SM53 = 0x35, + EF_CUDA_SM60 = 0x3c, + EF_CUDA_SM61 = 0x3d, + EF_CUDA_SM62 = 0x3e, + EF_CUDA_SM70 = 0x46, + EF_CUDA_SM72 = 0x48, + EF_CUDA_SM75 = 0x4b, + EF_CUDA_SM80 = 0x50, + EF_CUDA_SM86 = 0x56, + EF_CUDA_SM87 = 0x57, + EF_CUDA_SM89 = 0x59, + // The sm_90a variant uses the same machine flag. + EF_CUDA_SM90 = 0x5a, + + // Unified texture binding is enabled. + EF_CUDA_TEXMODE_UNIFIED = 0x100, + // Independent texture binding is enabled. + EF_CUDA_TEXMODE_INDEPENDANT = 0x200, + // The target is using 64-bit addressing. + EF_CUDA_64BIT_ADDRESS = 0x400, + // Set when using the sm_90a processor. + EF_CUDA_ACCELERATORS = 0x800, + // Undocumented software feature. + EF_CUDA_SW_FLAG_V2 = 0x1000, + + // Virtual processor selection mask for EF_CUDA_VIRTUAL_SM* values. + EF_CUDA_VIRTUAL_SM = 0xff0000, +}; + // ELF Relocation types for BPF enum { #include "ELFRelocs/BPF.def" From d6f772074c48cf9bb57191cb065b5ce60012ed74 Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Fri, 15 Dec 2023 12:46:05 -0700 Subject: [PATCH 011/884] fixup! fixup! [GlobalISel] Always direct-call IFuncs and Aliases (#74902) Apparently some BOLT bots build with a pre-installed system clang, and others use the just-built one. These two clangs now behave slightly differently when it comes to ifunc codegen after https://github.com/llvm/llvm-project/pull/74902 Change the test to accept both patterns. --- bolt/test/AArch64/ifunc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/test/AArch64/ifunc.c b/bolt/test/AArch64/ifunc.c index 79c035ed45373..c8c5c7c29d4a5 100644 --- a/bolt/test/AArch64/ifunc.c +++ b/bolt/test/AArch64/ifunc.c @@ -45,7 +45,7 @@ // NON_DYN_CHECK-NOT: DYNAMIC -// CHECK: b{{l?}} "{{resolver_foo|ifoo}}{{.*}}@PLT" +// CHECK: {{(bl? "(resolver_foo|ifoo).*@PLT"|adr x[0-9]+, ifoo)}} // REL_CHECK: R_AARCH64_IRELATIVE [[#%x,REL_SYMB_ADDR:]] // REL_CHECK: [[#REL_SYMB_ADDR]] {{.*}} FUNC {{.*}} resolver_foo From e4de6a602f4dbf189b0e2621b57932676300e16c Mon Sep 17 00:00:00 2001 From: dhruvachak Date: Fri, 15 Dec 2023 12:07:42 -0800 Subject: [PATCH 012/884] [OpenMP] [OMPT] A pointer to HostOpId should be passed in EMI callbacks. (#75574) With this change, TargetRegionOpId is no more used and hence deleted. --- .../include/OpenMP/OMPT/Interface.h | 3 --- .../libomptarget/src/OpenMP/OMPT/Callback.cpp | 21 +++++++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/openmp/libomptarget/include/OpenMP/OMPT/Interface.h b/openmp/libomptarget/include/OpenMP/OMPT/Interface.h index 8c3ef105c4993..ed34cbed785bc 100644 --- a/openmp/libomptarget/include/OpenMP/OMPT/Interface.h +++ b/openmp/libomptarget/include/OpenMP/OMPT/Interface.h @@ -195,9 +195,6 @@ class Interface { /// Target task data representing the target task region ompt_data_t *TargetTaskData = nullptr; - /// Correlation id that is incremented with target operations - uint64_t TargetRegionOpId = 1; - /// Used for marking begin of a data operation void beginTargetDataOperation(); diff --git a/openmp/libomptarget/src/OpenMP/OMPT/Callback.cpp b/openmp/libomptarget/src/OpenMP/OMPT/Callback.cpp index f1a8ad0e3cafb..da955e101956f 100644 --- a/openmp/libomptarget/src/OpenMP/OMPT/Callback.cpp +++ b/openmp/libomptarget/src/OpenMP/OMPT/Callback.cpp @@ -86,7 +86,7 @@ void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_alloc, HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), *TgtPtrBegin, /* TgtDeviceNum */ DeviceId, Size, Code); @@ -109,7 +109,7 @@ void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_alloc, HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), *TgtPtrBegin, /* TgtDeviceNum */ DeviceId, Size, Code); @@ -125,7 +125,7 @@ void Interface::beginTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_transfer_to_device, HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code); @@ -148,7 +148,7 @@ void Interface::endTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_transfer_to_device, HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code); @@ -163,7 +163,7 @@ void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_delete, TgtPtrBegin, DeviceId, /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code); } else if (ompt_callback_target_data_op_fn) { @@ -184,7 +184,7 @@ void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_delete, TgtPtrBegin, DeviceId, /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code); } @@ -199,7 +199,7 @@ void Interface::beginTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId, HstPtrBegin, /* TgtDeviceNum */ omp_get_initial_device(), Size, Code); @@ -222,7 +222,7 @@ void Interface::endTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin, // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback ompt_callback_target_data_op_emi_fn( - ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_scope_end, TargetTaskData, &TargetData, &HostOpId, ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId, HstPtrBegin, /* TgtDeviceNum */ omp_get_initial_device(), Size, Code); @@ -364,12 +364,11 @@ void Interface::endTarget(int64_t DeviceId, void *Code) { } void Interface::beginTargetDataOperation() { - DP("in ompt_target_region_begin (TargetRegionOpId = %lu)\n", - TargetData.value); + DP("in ompt_target_region_begin (TargetRegionId = %lu)\n", TargetData.value); } void Interface::endTargetDataOperation() { - DP("in ompt_target_region_end (TargetRegionOpId = %lu)\n", TargetData.value); + DP("in ompt_target_region_end (TargetRegionId = %lu)\n", TargetData.value); } void Interface::beginTargetRegion() { From a4d1d5f5b54b2f93d7290588734f59ff24fc515c Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Fri, 15 Dec 2023 15:02:21 -0500 Subject: [PATCH 013/884] [OpenMP] Use simple VLA implementation to replace uses of actual VLA Use of VLA can cause compile warning that was introduced in D156565. This patch implements a simple stack/heap-based VLA that can miminc the behavior of an actual VLA and prevent the warning. By default the stack accomodates the elements. If the number of emelements is greater than N, which by default is 8, a heap buffer will be allocated and used to acccomodate the elements. --- openmp/runtime/src/kmp_gsupport.cpp | 7 ++-- openmp/runtime/src/kmp_runtime.cpp | 3 +- openmp/runtime/src/kmp_utils.h | 55 +++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 openmp/runtime/src/kmp_utils.h diff --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp index dc843f78dc697..78af395335498 100644 --- a/openmp/runtime/src/kmp_gsupport.cpp +++ b/openmp/runtime/src/kmp_gsupport.cpp @@ -12,6 +12,7 @@ #include "kmp.h" #include "kmp_atomic.h" +#include "kmp_utils.h" #if OMPT_SUPPORT #include "ompt-specific.h" @@ -1280,7 +1281,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, KMP_ASSERT(depend); kmp_gomp_depends_info_t gomp_depends(depend); kmp_int32 ndeps = gomp_depends.get_num_deps(); - kmp_depend_info_t dep_list[ndeps]; + SimpleVLA dep_list(ndeps); for (kmp_int32 i = 0; i < ndeps; i++) dep_list[i] = gomp_depends.get_kmp_depend(i); kmp_int32 ndeps_cnv; @@ -1309,7 +1310,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, KMP_ASSERT(depend); kmp_gomp_depends_info_t gomp_depends(depend); kmp_int32 ndeps = gomp_depends.get_num_deps(); - kmp_depend_info_t dep_list[ndeps]; + SimpleVLA dep_list(ndeps); for (kmp_int32 i = 0; i < ndeps; i++) dep_list[i] = gomp_depends.get_kmp_depend(i); __kmpc_omp_wait_deps(&loc, gtid, ndeps, dep_list, 0, NULL); @@ -1993,7 +1994,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT_DEPEND)(void **depend) { KA_TRACE(20, ("GOMP_taskwait_depend: T#%d\n", gtid)); kmp_gomp_depends_info_t gomp_depends(depend); kmp_int32 ndeps = gomp_depends.get_num_deps(); - kmp_depend_info_t dep_list[ndeps]; + SimpleVLA dep_list(ndeps); for (kmp_int32 i = 0; i < ndeps; i++) dep_list[i] = gomp_depends.get_kmp_depend(i); #if OMPT_SUPPORT diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp index 7d2e7a99ade43..4e1074a893a28 100644 --- a/openmp/runtime/src/kmp_runtime.cpp +++ b/openmp/runtime/src/kmp_runtime.cpp @@ -24,6 +24,7 @@ #include "kmp_wait_release.h" #include "kmp_wrapper_getpid.h" #include "kmp_dispatch.h" +#include "kmp_utils.h" #if KMP_USE_HIER_SCHED #include "kmp_dispatch_hier.h" #endif @@ -1653,7 +1654,7 @@ __kmp_serial_fork_call(ident_t *loc, int gtid, enum fork_context_e call_context, /* josh todo: hypothetical question: what do we do for OS X*? */ #if KMP_OS_LINUX && \ (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) - void *args[argc]; + SimpleVLA args(argc); #else void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ diff --git a/openmp/runtime/src/kmp_utils.h b/openmp/runtime/src/kmp_utils.h new file mode 100644 index 0000000000000..a557f929e6e72 --- /dev/null +++ b/openmp/runtime/src/kmp_utils.h @@ -0,0 +1,55 @@ +/* + * kmp_utils.h -- Utilities that used internally + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef __KMP_UTILS_H__ +#define __KMP_UTILS_H__ + +#include + +#include "kmp.h" + +/// A simple pure header implementation of VLA that aims to replace uses of +/// actual VLA, which can cause compile warning. This class by default creates a +/// stack buffer that can accomodate \p N elements. If the number of elements is +/// greater than \p N, then a heap buffer will be allocated and used to +/// accomodate the elements. Similar to the actual VLA, we don't check boundary +/// (for now), so we will not store the number of elements. We can always revise +/// it later. +template class SimpleVLA final { + T StackBuffer[N]; + T *HeapBuffer = nullptr; + T *Ptr = StackBuffer; + +public: + SimpleVLA() = delete; + SimpleVLA(const SimpleVLA &) = delete; + SimpleVLA(SimpleVLA &&) = delete; + SimpleVLA &operator=(const SimpleVLA &) = delete; + SimpleVLA &operator=(SimpleVLA &&) = delete; + + explicit SimpleVLA(unsigned NumOfElements) noexcept { + if (NumOfElements > N) { + HeapBuffer = + reinterpret_cast(__kmp_allocate(NumOfElements * sizeof(T))); + Ptr = HeapBuffer; + } + } + + ~SimpleVLA() { + if (HeapBuffer) + __kmp_free(HeapBuffer); + } + + operator T *() noexcept { return Ptr; } + operator const T *() const noexcept { return Ptr; } +}; + +#endif From cb56ba635017500d6e0d7bb2a2a708b0161b8602 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 15 Dec 2023 20:26:33 +0000 Subject: [PATCH 014/884] [VPlan] Unswitch cond in replaceUsesWithIf in optimizeInductions (NFC) As suggested post-commit for a00227197, unswitch the condition in replaceUsesWithIf to simplify the check. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 570da97c3cfaf..33132880d5a44 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -529,10 +529,12 @@ void VPlanTransforms::optimizeInductions(VPlan &Plan, ScalarEvolution &SE) { WideIV->getStartValue(), WideIV->getStepValue()); // Update scalar users of IV to use Step instead. - WideIV->replaceUsesWithIf( - Steps, [HasOnlyVectorVFs, WideIV](VPUser &U, unsigned) { - return !HasOnlyVectorVFs || U.usesScalars(WideIV); - }); + if (!HasOnlyVectorVFs) + WideIV->replaceAllUsesWith(Steps); + else + WideIV->replaceUsesWithIf(Steps, [WideIV](VPUser &U, unsigned) { + return U.usesScalars(WideIV); + }); } } From 809ee6cfcf48c3a3681e3dbde425e65b35fa7010 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Fri, 15 Dec 2023 12:51:33 -0800 Subject: [PATCH 015/884] [X86][test] Update tagged-globals*.ll tests Use update_llc_test_checks.py. Split out jump table tests into separate file since we don't want to check the exact instruction sequence for it. --- .../CodeGen/X86/tagged-globals-jump-table.ll | 29 +++++++ llvm/test/CodeGen/X86/tagged-globals-pic.ll | 77 +++++-------------- .../test/CodeGen/X86/tagged-globals-static.ll | 60 +++++---------- 3 files changed, 67 insertions(+), 99 deletions(-) create mode 100644 llvm/test/CodeGen/X86/tagged-globals-jump-table.ll diff --git a/llvm/test/CodeGen/X86/tagged-globals-jump-table.ll b/llvm/test/CodeGen/X86/tagged-globals-jump-table.ll new file mode 100644 index 0000000000000..a436df0c3f555 --- /dev/null +++ b/llvm/test/CodeGen/X86/tagged-globals-jump-table.ll @@ -0,0 +1,29 @@ +; RUN: llc --relocation-model=pic < %s | FileCheck %s +; RUN: llc --relocation-model=static < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Jump tables shouldn't go through the GOT. +define i32 @jump_table(i32 %x) #0 { +; CHECK-LABEL: jump_table: +; CHECK-NOT: @GOT + switch i32 %x, label %default [ + i32 0, label %1 + i32 1, label %2 + i32 2, label %3 + i32 3, label %4 + ] +1: + ret i32 7 +2: + ret i32 42 +3: + ret i32 3 +4: + ret i32 8 +default: + ret i32 %x +} + +attributes #0 = { "target-features"="+tagged-globals" } diff --git a/llvm/test/CodeGen/X86/tagged-globals-pic.ll b/llvm/test/CodeGen/X86/tagged-globals-pic.ll index 89b424340ef98..4f85b5ed99695 100644 --- a/llvm/test/CodeGen/X86/tagged-globals-pic.ll +++ b/llvm/test/CodeGen/X86/tagged-globals-pic.ll @@ -1,5 +1,5 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc --relocation-model=pic < %s | FileCheck %s -; RUN: llc --relocation-model=pic --relax-elf-relocations --filetype=obj -o - < %s | llvm-objdump -d -r - | FileCheck %s --check-prefix=OBJ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -8,78 +8,39 @@ target triple = "x86_64-unknown-linux-gnu" declare void @func() define ptr @global_addr() #0 { - ; CHECK-LABEL: global_addr: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), %rax - ; CHECK: retq - - ; OBJ-LABEL: : - ; OBJ: movq (%rip), - ; OBJ-NEXT: R_X86_64_GOTPCREL global - +; CHECK-LABEL: global_addr: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: retq ret ptr @global } define i32 @global_load() #0 { - ; CHECK-LABEL: global_load: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), [[REG:%r[0-9a-z]+]] - ; CHECK: movl ([[REG]]), %eax - ; CHECK: retq - - ; OBJ-LABEL: : - ; OBJ: movq (%rip), - ; OBJ-NEXT: R_X86_64_GOTPCREL global - +; CHECK-LABEL: global_load: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: movl (%rax), %eax +; CHECK-NEXT: retq %load = load i32, ptr @global ret i32 %load } define void @global_store() #0 { - ; CHECK-LABEL: global_store: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), [[REG:%r[0-9a-z]+]] - ; CHECK: movl $0, ([[REG]]) - ; CHECK: retq - - ; OBJ-LABEL: : - ; OBJ: movq (%rip), - ; OBJ-NEXT: R_X86_64_GOTPCREL global - +; CHECK-LABEL: global_store: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: movl $0, (%rax) +; CHECK-NEXT: retq store i32 0, ptr @global ret void } define ptr @func_addr() #0 { - ; CHECK-LABEL: func_addr: - ; CHECK: movq func@GOTPCREL(%rip), %rax - ; CHECK: retq - - ; OBJ-LABEL: : - ; OBJ: movq (%rip), - ; OBJ-NEXT: R_X86_64_REX_GOTPCRELX func - +; CHECK-LABEL: func_addr: +; CHECK: # %bb.0: +; CHECK-NEXT: movq func@GOTPCREL(%rip), %rax +; CHECK-NEXT: retq ret ptr @func } -; Jump tables shouldn't go through the GOT. -define i32 @jump_table(i32 %x) #0 { - ; CHECK-LABEL: jump_table: - ; CHECK-NOT: @GOT - - switch i32 %x, label %default [ - i32 0, label %1 - i32 1, label %2 - i32 2, label %3 - i32 3, label %4 - ] -1: - ret i32 7 -2: - ret i32 42 -3: - ret i32 3 -4: - ret i32 8 -default: - ret i32 %x -} - attributes #0 = { "target-features"="+tagged-globals" } diff --git a/llvm/test/CodeGen/X86/tagged-globals-static.ll b/llvm/test/CodeGen/X86/tagged-globals-static.ll index 35fb6a23ee900..bddbaa5592da5 100644 --- a/llvm/test/CodeGen/X86/tagged-globals-static.ll +++ b/llvm/test/CodeGen/X86/tagged-globals-static.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc --relocation-model=static < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -7,62 +8,39 @@ target triple = "x86_64-unknown-linux-gnu" declare dso_local void @func() define ptr @global_addr() #0 { - ; CHECK-LABEL: global_addr: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), %rax - ; CHECK: retq - +; CHECK-LABEL: global_addr: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: retq ret ptr @global } define i32 @global_load() #0 { - ; CHECK-LABEL: global_load: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), [[REG:%r[0-9a-z]+]] - ; CHECK: movl ([[REG]]), %eax - ; CHECK: retq - +; CHECK-LABEL: global_load: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: movl (%rax), %eax +; CHECK-NEXT: retq %load = load i32, ptr @global ret i32 %load } define void @global_store() #0 { - ; CHECK-LABEL: global_store: - ; CHECK: movq global@GOTPCREL_NORELAX(%rip), [[REG:%r[0-9a-z]+]] - ; CHECK: movl $0, ([[REG]]) - ; CHECK: retq - +; CHECK-LABEL: global_store: +; CHECK: # %bb.0: +; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax +; CHECK-NEXT: movl $0, (%rax) +; CHECK-NEXT: retq store i32 0, ptr @global ret void } define ptr @func_addr() #0 { - ; CHECK-LABEL: func_addr: - ; CHECK: movl $func, %eax - ; CHECK: retq - +; CHECK-LABEL: func_addr: +; CHECK: # %bb.0: +; CHECK-NEXT: movl $func, %eax +; CHECK-NEXT: retq ret ptr @func } -; Jump tables shouldn't go through the GOT. -define i32 @jump_table(i32 %x) #0 { - ; CHECK-LABEL: jump_table: - ; CHECK-NOT: @GOT - - switch i32 %x, label %default [ - i32 0, label %1 - i32 1, label %2 - i32 2, label %3 - i32 3, label %4 - ] -1: - ret i32 7 -2: - ret i32 42 -3: - ret i32 3 -4: - ret i32 8 -default: - ret i32 %x -} - attributes #0 = { "target-features"="+tagged-globals" } From 22426d9ecda11b5be428a35e9bcdeec2744ce30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Fri, 15 Dec 2023 13:02:40 -0800 Subject: [PATCH 016/884] [flang][openacc/mp] Do not read bounds on absent box (#75252) Make sure we only load box and read its bounds when it is present. - Add `AddrAndBoundInfo` struct to be able to carry around the `addr` and `isPresent` values. This is likely to grow so we can make all the access in a single `fir.if` operation. --- flang/lib/Lower/DirectivesCommon.h | 183 ++++++++++++++++++------ flang/lib/Lower/OpenACC.cpp | 108 ++++++++------ flang/lib/Lower/OpenMP.cpp | 27 ++-- flang/test/Lower/OpenACC/acc-bounds.f90 | 35 +++++ flang/test/Lower/OpenACC/acc-data.f90 | 1 - 5 files changed, 256 insertions(+), 98 deletions(-) diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h index 52d0b24500168..ffbd8ae1558ed 100644 --- a/flang/lib/Lower/DirectivesCommon.h +++ b/flang/lib/Lower/DirectivesCommon.h @@ -47,6 +47,17 @@ namespace Fortran { namespace lower { +/// Information gathered to generate bounds operation and data entry/exit +/// operations. +struct AddrAndBoundsInfo { + explicit AddrAndBoundsInfo() {} + explicit AddrAndBoundsInfo(mlir::Value addr) : addr(addr) {} + explicit AddrAndBoundsInfo(mlir::Value addr, mlir::Value isPresent) + : addr(addr), isPresent(isPresent) {} + mlir::Value addr = nullptr; + mlir::Value isPresent = nullptr; +}; + /// Checks if the assignment statement has a single variable on the RHS. static inline bool checkForSingleVariableOnRHS( const Fortran::parser::AssignmentStmt &assignmentStmt) { @@ -598,7 +609,7 @@ void createEmptyRegionBlocks( } } -inline mlir::Value +inline AddrAndBoundsInfo getDataOperandBaseAddr(Fortran::lower::AbstractConverter &converter, fir::FirOpBuilder &builder, Fortran::lower::SymbolRef sym, mlir::Location loc) { @@ -620,25 +631,42 @@ getDataOperandBaseAddr(Fortran::lower::AbstractConverter &converter, // Load the box when baseAddr is a `fir.ref>` or a // `fir.ref>` type. - if (symAddr.getType().isa()) - return builder.create(loc, symAddr); + if (symAddr.getType().isa()) { + if (Fortran::semantics::IsOptional(sym)) { + mlir::Value isPresent = + builder.create(loc, builder.getI1Type(), symAddr); + mlir::Value addr = + builder.genIfOp(loc, {boxTy}, isPresent, /*withElseRegion=*/true) + .genThen([&]() { + mlir::Value load = builder.create(loc, symAddr); + builder.create(loc, mlir::ValueRange{load}); + }) + .genElse([&] { + mlir::Value absent = + builder.create(loc, boxTy); + builder.create(loc, mlir::ValueRange{absent}); + }) + .getResults()[0]; + return AddrAndBoundsInfo(addr, isPresent); + } + mlir::Value addr = builder.create(loc, symAddr); + return AddrAndBoundsInfo(addr); + ; + } } - return symAddr; + return AddrAndBoundsInfo(symAddr); } -/// Generate the bounds operation from the descriptor information. template llvm::SmallVector -genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc, - Fortran::lower::AbstractConverter &converter, - fir::ExtendedValue dataExv, mlir::Value box) { - llvm::SmallVector bounds; +gatherBoundsOrBoundValues(fir::FirOpBuilder &builder, mlir::Location loc, + fir::ExtendedValue dataExv, mlir::Value box, + bool collectValuesOnly = false) { + llvm::SmallVector values; + mlir::Value byteStride; mlir::Type idxTy = builder.getIndexType(); mlir::Type boundTy = builder.getType(); mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1); - assert(box.getType().isa() && - "expect fir.box or fir.class"); - mlir::Value byteStride; for (unsigned dim = 0; dim < dataExv.rank(); ++dim) { mlir::Value d = builder.createIntegerConstant(loc, idxTy, dim); mlir::Value baseLb = @@ -650,12 +678,79 @@ genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc, builder.create(loc, dimInfo.getExtent(), one); if (dim == 0) // First stride is the element size. byteStride = dimInfo.getByteStride(); - mlir::Value bound = builder.create( - loc, boundTy, lb, ub, dimInfo.getExtent(), byteStride, true, baseLb); + if (collectValuesOnly) { + values.push_back(lb); + values.push_back(ub); + values.push_back(dimInfo.getExtent()); + values.push_back(byteStride); + values.push_back(baseLb); + } else { + mlir::Value bound = builder.create( + loc, boundTy, lb, ub, dimInfo.getExtent(), byteStride, true, baseLb); + values.push_back(bound); + } // Compute the stride for the next dimension. byteStride = builder.create(loc, byteStride, dimInfo.getExtent()); - bounds.push_back(bound); + } + return values; +} + +/// Generate the bounds operation from the descriptor information. +template +llvm::SmallVector +genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc, + Fortran::lower::AbstractConverter &converter, + fir::ExtendedValue dataExv, + Fortran::lower::AddrAndBoundsInfo &info) { + llvm::SmallVector bounds; + mlir::Type idxTy = builder.getIndexType(); + mlir::Type boundTy = builder.getType(); + + assert(info.addr.getType().isa() && + "expect fir.box or fir.class"); + + if (info.isPresent) { + llvm::SmallVector resTypes; + constexpr unsigned nbValuesPerBound = 5; + for (unsigned dim = 0; dim < dataExv.rank() * nbValuesPerBound; ++dim) + resTypes.push_back(idxTy); + + mlir::Operation::result_range ifRes = + builder.genIfOp(loc, resTypes, info.isPresent, /*withElseRegion=*/true) + .genThen([&]() { + llvm::SmallVector boundValues = + gatherBoundsOrBoundValues( + builder, loc, dataExv, info.addr, + /*collectValuesOnly=*/true); + builder.create(loc, boundValues); + }) + .genElse([&] { + // Box is not present. Populate bound values with default values. + llvm::SmallVector boundValues; + mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0); + mlir::Value mOne = builder.createIntegerConstant(loc, idxTy, -1); + for (unsigned dim = 0; dim < dataExv.rank(); ++dim) { + boundValues.push_back(zero); // lb + boundValues.push_back(mOne); // ub + boundValues.push_back(zero); // extent + boundValues.push_back(zero); // byteStride + boundValues.push_back(zero); // baseLb + } + builder.create(loc, boundValues); + }) + .getResults(); + // Create the bound operations outside the if-then-else with the if op + // results. + for (unsigned i = 0; i < ifRes.size(); i += nbValuesPerBound) { + mlir::Value bound = builder.create( + loc, boundTy, ifRes[i], ifRes[i + 1], ifRes[i + 2], ifRes[i + 3], + true, ifRes[i + 4]); + bounds.push_back(bound); + } + } else { + bounds = gatherBoundsOrBoundValues( + builder, loc, dataExv, info.addr); } return bounds; } @@ -843,14 +938,13 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, } template -mlir::Value gatherDataOperandAddrAndBounds( +AddrAndBoundsInfo gatherDataOperandAddrAndBounds( Fortran::lower::AbstractConverter &converter, fir::FirOpBuilder &builder, Fortran::semantics::SemanticsContext &semanticsContext, Fortran::lower::StatementContext &stmtCtx, const ObjectType &object, mlir::Location operandLocation, std::stringstream &asFortran, llvm::SmallVector &bounds, bool treatIndexAsSection = false) { - mlir::Value baseAddr; - + AddrAndBoundsInfo info; std::visit( Fortran::common::visitors{ [&](const Fortran::parser::Designator &designator) { @@ -872,13 +966,13 @@ mlir::Value gatherDataOperandAddrAndBounds( semanticsContext, arrayElement->base); dataExv = converter.genExprAddr(operandLocation, *exprBase, stmtCtx); - baseAddr = fir::getBase(dataExv); + info.addr = fir::getBase(dataExv); asFortran << (*exprBase).AsFortran(); } else { const Fortran::parser::Name &name = Fortran::parser::GetLastName(*dataRef); - baseAddr = getDataOperandBaseAddr( - converter, builder, *name.symbol, operandLocation); + info = getDataOperandBaseAddr(converter, builder, + *name.symbol, operandLocation); dataExv = converter.getSymbolExtendedValue(*name.symbol); asFortran << name.ToString(); } @@ -887,27 +981,33 @@ mlir::Value gatherDataOperandAddrAndBounds( asFortran << '('; bounds = genBoundsOps( builder, operandLocation, converter, stmtCtx, - arrayElement->subscripts, asFortran, dataExv, baseAddr, + arrayElement->subscripts, asFortran, dataExv, info.addr, treatIndexAsSection); } asFortran << ')'; - } else if (Fortran::parser::Unwrap< + } else if (auto structComp = Fortran::parser::Unwrap< Fortran::parser::StructureComponent>(designator)) { fir::ExtendedValue compExv = converter.genExprAddr(operandLocation, *expr, stmtCtx); - baseAddr = fir::getBase(compExv); - if (fir::unwrapRefType(baseAddr.getType()) + info.addr = fir::getBase(compExv); + if (fir::unwrapRefType(info.addr.getType()) .isa()) bounds = genBaseBoundsOps( builder, operandLocation, converter, compExv); asFortran << (*expr).AsFortran(); + bool isOptional = Fortran::semantics::IsOptional( + *Fortran::parser::GetLastName(*structComp).symbol); + if (isOptional) + info.isPresent = builder.create( + operandLocation, builder.getI1Type(), info.addr); + if (auto loadOp = mlir::dyn_cast_or_null( - baseAddr.getDefiningOp())) { + info.addr.getDefiningOp())) { if (fir::isAllocatableType(loadOp.getType()) || fir::isPointerType(loadOp.getType())) - baseAddr = builder.create(operandLocation, - baseAddr); + info.addr = builder.create(operandLocation, + info.addr); } // If the component is an allocatable or pointer the result of @@ -915,10 +1015,10 @@ mlir::Value gatherDataOperandAddrAndBounds( // a fir.box_addr has been inserted just before. // Retrieve the box so we handle it like other descriptor. if (auto boxAddrOp = mlir::dyn_cast_or_null( - baseAddr.getDefiningOp())) { - baseAddr = boxAddrOp.getVal(); + info.addr.getDefiningOp())) { + info.addr = boxAddrOp.getVal(); bounds = genBoundsOpsFromBox( - builder, operandLocation, converter, compExv, baseAddr); + builder, operandLocation, converter, compExv, info); } } else { if (Fortran::parser::Unwrap( @@ -930,7 +1030,7 @@ mlir::Value gatherDataOperandAddrAndBounds( (void)arrayElement; fir::ExtendedValue compExv = converter.genExprAddr(operandLocation, *expr, stmtCtx); - baseAddr = fir::getBase(compExv); + info.addr = fir::getBase(compExv); asFortran << (*expr).AsFortran(); } else if (const auto *dataRef{ std::get_if( @@ -940,13 +1040,14 @@ mlir::Value gatherDataOperandAddrAndBounds( Fortran::parser::GetLastName(*dataRef); fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(*name.symbol); - baseAddr = getDataOperandBaseAddr( - converter, builder, *name.symbol, operandLocation); - if (fir::unwrapRefType(baseAddr.getType()) - .isa()) + info = getDataOperandBaseAddr(converter, builder, + *name.symbol, operandLocation); + if (fir::unwrapRefType(info.addr.getType()) + .isa()) { bounds = genBoundsOpsFromBox( - builder, operandLocation, converter, dataExv, baseAddr); - if (fir::unwrapRefType(baseAddr.getType()) + builder, operandLocation, converter, dataExv, info); + } + if (fir::unwrapRefType(info.addr.getType()) .isa()) bounds = genBaseBoundsOps( builder, operandLocation, converter, dataExv); @@ -959,12 +1060,12 @@ mlir::Value gatherDataOperandAddrAndBounds( } }, [&](const Fortran::parser::Name &name) { - baseAddr = getDataOperandBaseAddr(converter, builder, *name.symbol, - operandLocation); + info = getDataOperandBaseAddr(converter, builder, *name.symbol, + operandLocation); asFortran << name.ToString(); }}, object.u); - return baseAddr; + return info; } } // namespace lower diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index 531685948bc84..75432db33a790 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -62,11 +62,29 @@ static Op createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value baseAddr, std::stringstream &name, mlir::SmallVector bounds, bool structured, bool implicit, - mlir::acc::DataClause dataClause, - mlir::Type retTy) { + mlir::acc::DataClause dataClause, mlir::Type retTy, + mlir::Value isPresent = {}) { mlir::Value varPtrPtr; if (auto boxTy = baseAddr.getType().dyn_cast()) { - baseAddr = builder.create(loc, baseAddr); + if (isPresent) { + baseAddr = + builder + .genIfOp(loc, {boxTy.getEleTy()}, isPresent, + /*withElseRegion=*/true) + .genThen([&]() { + mlir::Value boxAddr = + builder.create(loc, baseAddr); + builder.create(loc, mlir::ValueRange{boxAddr}); + }) + .genElse([&] { + mlir::Value absent = + builder.create(loc, boxTy.getEleTy()); + builder.create(loc, mlir::ValueRange{absent}); + }) + .getResults()[0]; + } else { + baseAddr = builder.create(loc, baseAddr); + } retTy = baseAddr.getType(); } @@ -265,15 +283,17 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList, llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); - mlir::Value baseAddr = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds, - /*treatIndexAsSection=*/true); - Op op = createDataEntryOp(builder, operandLocation, baseAddr, asFortran, - bounds, structured, implicit, dataClause, - baseAddr.getType()); + Fortran::lower::AddrAndBoundsInfo info = + Fortran::lower::gatherDataOperandAddrAndBounds< + Fortran::parser::AccObject, mlir::acc::DataBoundsOp, + mlir::acc::DataBoundsType>(converter, builder, semanticsContext, + stmtCtx, accObject, operandLocation, + asFortran, bounds, + /*treatIndexAsSection=*/true); + + Op op = createDataEntryOp( + builder, operandLocation, info.addr, asFortran, bounds, structured, + implicit, dataClause, info.addr.getType(), info.isPresent); dataOperands.push_back(op.getAccPtr()); } } @@ -291,27 +311,28 @@ static void genDeclareDataOperandOperations( llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); - mlir::Value baseAddr = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); + Fortran::lower::AddrAndBoundsInfo info = + Fortran::lower::gatherDataOperandAddrAndBounds< + Fortran::parser::AccObject, mlir::acc::DataBoundsOp, + mlir::acc::DataBoundsType>(converter, builder, semanticsContext, + stmtCtx, accObject, operandLocation, + asFortran, bounds); EntryOp op = createDataEntryOp( - builder, operandLocation, baseAddr, asFortran, bounds, structured, - implicit, dataClause, baseAddr.getType()); + builder, operandLocation, info.addr, asFortran, bounds, structured, + implicit, dataClause, info.addr.getType()); dataOperands.push_back(op.getAccPtr()); addDeclareAttr(builder, op.getVarPtr().getDefiningOp(), dataClause); - if (mlir::isa(fir::unwrapRefType(baseAddr.getType()))) { + if (mlir::isa(fir::unwrapRefType(info.addr.getType()))) { mlir::OpBuilder modBuilder(builder.getModule().getBodyRegion()); modBuilder.setInsertionPointAfter(builder.getFunction()); std::string prefix = converter.mangleName(getSymbolFromAccObject(accObject)); createDeclareAllocFuncWithArg( - modBuilder, builder, operandLocation, baseAddr.getType(), prefix, + modBuilder, builder, operandLocation, info.addr.getType(), prefix, asFortran, dataClause); if constexpr (!std::is_same_v) createDeclareDeallocFuncWithArg( - modBuilder, builder, operandLocation, baseAddr.getType(), prefix, + modBuilder, builder, operandLocation, info.addr.getType(), prefix, asFortran, dataClause); } } @@ -749,21 +770,21 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList, llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); - mlir::Value baseAddr = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); - + Fortran::lower::AddrAndBoundsInfo info = + Fortran::lower::gatherDataOperandAddrAndBounds< + Fortran::parser::AccObject, mlir::acc::DataBoundsOp, + mlir::acc::DataBoundsType>(converter, builder, semanticsContext, + stmtCtx, accObject, operandLocation, + asFortran, bounds); RecipeOp recipe; - mlir::Type retTy = getTypeFromBounds(bounds, baseAddr.getType()); + mlir::Type retTy = getTypeFromBounds(bounds, info.addr.getType()); if constexpr (std::is_same_v) { std::string recipeName = fir::getTypeAsString(retTy, converter.getKindMap(), "privatization"); recipe = Fortran::lower::createOrGetPrivateRecipe(builder, recipeName, operandLocation, retTy); auto op = createDataEntryOp( - builder, operandLocation, baseAddr, asFortran, bounds, true, + builder, operandLocation, info.addr, asFortran, bounds, true, /*implicit=*/false, mlir::acc::DataClause::acc_private, retTy); dataOperands.push_back(op.getAccPtr()); } else { @@ -774,7 +795,7 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList, recipe = Fortran::lower::createOrGetFirstprivateRecipe( builder, recipeName, operandLocation, retTy, bounds); auto op = createDataEntryOp( - builder, operandLocation, baseAddr, asFortran, bounds, true, + builder, operandLocation, info.addr, asFortran, bounds, true, /*implicit=*/false, mlir::acc::DataClause::acc_firstprivate, retTy); dataOperands.push_back(op.getAccPtr()); } @@ -1326,13 +1347,14 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList, llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); - mlir::Value baseAddr = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); - - mlir::Type reductionTy = fir::unwrapRefType(baseAddr.getType()); + Fortran::lower::AddrAndBoundsInfo info = + Fortran::lower::gatherDataOperandAddrAndBounds< + Fortran::parser::AccObject, mlir::acc::DataBoundsOp, + mlir::acc::DataBoundsType>(converter, builder, semanticsContext, + stmtCtx, accObject, operandLocation, + asFortran, bounds); + + mlir::Type reductionTy = fir::unwrapRefType(info.addr.getType()); if (auto seqTy = mlir::dyn_cast(reductionTy)) reductionTy = seqTy.getEleTy(); @@ -1340,14 +1362,14 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList, TODO(operandLocation, "reduction with unsupported type"); auto op = createDataEntryOp( - builder, operandLocation, baseAddr, asFortran, bounds, + builder, operandLocation, info.addr, asFortran, bounds, /*structured=*/true, /*implicit=*/false, - mlir::acc::DataClause::acc_reduction, baseAddr.getType()); + mlir::acc::DataClause::acc_reduction, info.addr.getType()); mlir::Type ty = op.getAccPtr().getType(); if (!areAllBoundConstant(bounds) || - fir::isAssumedShape(baseAddr.getType()) || - fir::isAllocatableOrPointerArray(baseAddr.getType())) - ty = baseAddr.getType(); + fir::isAssumedShape(info.addr.getType()) || + fir::isAllocatableOrPointerArray(info.addr.getType())) + ty = info.addr.getType(); std::string suffix = areAllBoundConstant(bounds) ? getBoundsString(bounds) : ""; std::string recipeName = fir::getTypeAsString( diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp index 22d7cc2441886..9213cff95d3f1 100644 --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -1794,27 +1794,28 @@ bool ClauseProcessor::processMap( std::get(mapClause->v.t).v) { llvm::SmallVector bounds; std::stringstream asFortran; - mlir::Value baseAddr = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::OmpObject, mlir::omp::DataBoundsOp, - mlir::omp::DataBoundsType>( - converter, firOpBuilder, semanticsContext, stmtCtx, ompObject, - clauseLocation, asFortran, bounds, treatIndexAsSection); + Fortran::lower::AddrAndBoundsInfo info = + Fortran::lower::gatherDataOperandAddrAndBounds< + Fortran::parser::OmpObject, mlir::omp::DataBoundsOp, + mlir::omp::DataBoundsType>( + converter, firOpBuilder, semanticsContext, stmtCtx, ompObject, + clauseLocation, asFortran, bounds, treatIndexAsSection); // Explicit map captures are captured ByRef by default, // optimisation passes may alter this to ByCopy or other capture // types to optimise mlir::Value mapOp = createMapInfoOp( - firOpBuilder, clauseLocation, baseAddr, asFortran, bounds, + firOpBuilder, clauseLocation, info.addr, asFortran, bounds, static_cast< std::underlying_type_t>( mapTypeBits), - mlir::omp::VariableCaptureKind::ByRef, baseAddr.getType()); + mlir::omp::VariableCaptureKind::ByRef, info.addr.getType()); mapOperands.push_back(mapOp); if (mapSymTypes) - mapSymTypes->push_back(baseAddr.getType()); + mapSymTypes->push_back(info.addr.getType()); if (mapSymLocs) - mapSymLocs->push_back(baseAddr.getLoc()); + mapSymLocs->push_back(info.addr.getLoc()); if (mapSymbols) mapSymbols->push_back(getOmpObjectSymbol(ompObject)); } @@ -2655,16 +2656,16 @@ genTargetOp(Fortran::lower::AbstractConverter &converter, fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym); name << sym.name().ToString(); - mlir::Value baseAddr = + Fortran::lower::AddrAndBoundsInfo info = getDataOperandBaseAddr(converter, converter.getFirOpBuilder(), sym, converter.getCurrentLocation()); - if (fir::unwrapRefType(baseAddr.getType()).isa()) + if (fir::unwrapRefType(info.addr.getType()).isa()) bounds = Fortran::lower::genBoundsOpsFromBox( converter.getFirOpBuilder(), converter.getCurrentLocation(), - converter, dataExv, baseAddr); - if (fir::unwrapRefType(baseAddr.getType()).isa()) + converter, dataExv, info); + if (fir::unwrapRefType(info.addr.getType()).isa()) bounds = Fortran::lower::genBaseBoundsOps( converter.getFirOpBuilder(), converter.getCurrentLocation(), diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90 index 8db18ab5aa9c4..9e8e54bc2f7fa 100644 --- a/flang/test/Lower/OpenACC/acc-bounds.f90 +++ b/flang/test/Lower/OpenACC/acc-bounds.f90 @@ -116,4 +116,39 @@ subroutine acc_multi_strides(a) ! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[BOX_ADDR]] : !fir.ref>) bounds(%29, %33, %37) -> !fir.ref> {name = "a"} ! CHECK: acc.kernels dataOperands(%[[PRESENT]] : !fir.ref>) { + subroutine acc_optional_data(a) + real, pointer, optional :: a(:) + !$acc data attach(a) + !$acc end data + end subroutine + +! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_optional_data( +! CHECK-SAME: %[[ARG0:.*]]: !fir.ref>>> {fir.bindc_name = "a", fir.optional}) { +! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs, uniq_name = "_QMopenacc_boundsFacc_optional_dataEa"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[IS_PRESENT:.*]] = fir.is_present %[[ARG0_DECL]]#1 : (!fir.ref>>>) -> i1 +! CHECK: %[[BOX:.*]] = fir.if %[[IS_PRESENT]] -> (!fir.box>>) { +! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0_DECL]]#1 : !fir.ref>>> +! CHECK: fir.result %[[LOAD]] : !fir.box>> +! CHECK: } else { +! CHECK: %[[ABSENT:.*]] = fir.absent !fir.box>> +! CHECK: fir.result %[[ABSENT]] : !fir.box>> +! CHECK: } +! CHECK: %[[RES:.*]]:5 = fir.if %[[IS_PRESENT]] -> (index, index, index, index, index) { +! CHECK: fir.result %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : index, index, index, index, index +! CHECK: } else { +! CHECK: %[[C0:.*]] = arith.constant 0 : index +! CHECK: %[[CM1:.*]] = arith.constant -1 : index +! CHECK: fir.result %[[C0]], %[[CM1]], %[[C0]], %[[C0]], %[[C0]] : index, index, index, index, index +! CHECK: } +! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[RES]]#0 : index) upperbound(%[[RES]]#1 : index) extent(%[[RES]]#2 : index) stride(%[[RES]]#3 : index) startIdx(%[[RES]]#4 : index) {strideInBytes = true} +! CHECK: %[[BOX_ADDR:.*]] = fir.if %[[IS_PRESENT]] -> (!fir.ptr>) { +! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box>>) -> !fir.ptr> +! CHECK: fir.result %[[ADDR]] : !fir.ptr> +! CHECK: } else { +! CHECK: %[[ABSENT:.*]] = fir.absent !fir.ptr> +! CHECK: fir.result %[[ABSENT]] : !fir.ptr> +! CHECK: } +! CHECK: %[[ATTACH:.*]] = acc.attach varPtr(%[[BOX_ADDR]] : !fir.ptr>) bounds(%[[BOUND]]) -> !fir.ptr> {name = "a"} +! CHECK: acc.data dataOperands(%[[ATTACH]] : !fir.ptr>) + end module diff --git a/flang/test/Lower/OpenACC/acc-data.f90 b/flang/test/Lower/OpenACC/acc-data.f90 index d302be85c5df4..a6572e1470760 100644 --- a/flang/test/Lower/OpenACC/acc-data.f90 +++ b/flang/test/Lower/OpenACC/acc-data.f90 @@ -198,4 +198,3 @@ subroutine acc_data ! CHECK-NOT: acc.data end subroutine acc_data - From 70bcd81e7a7365d2fa98a2d08ba81305eb7846f1 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Fri, 15 Dec 2023 16:06:12 -0500 Subject: [PATCH 017/884] [libc++] Fix constexpr initialization of std::array (#74667) This patch fixes constexpr default initialization of empty arrays and improves the tests accordingly. Fixes #74375 --- libcxx/include/array | 5 +- .../default_initializable.compile.pass.cpp | 2 +- .../array/array.cons/initialization.pass.cpp | 58 ++++--- .../array/size_and_alignment.compile.pass.cpp | 143 ++++++++++++++++++ .../array/size_and_alignment.pass.cpp | 78 ---------- .../mdspan/extents/CtorTestCombinations.h | 2 +- 6 files changed, 187 insertions(+), 101 deletions(-) create mode 100644 libcxx/test/std/containers/sequences/array/size_and_alignment.compile.pass.cpp delete mode 100644 libcxx/test/std/containers/sequences/array/size_and_alignment.pass.cpp diff --git a/libcxx/include/array b/libcxx/include/array index 127092f6bca9b..d00bf278e6354 100644 --- a/libcxx/include/array +++ b/libcxx/include/array @@ -131,6 +131,7 @@ template const T&& get(const array&&) noexce #include <__type_traits/is_same.h> #include <__type_traits/is_swappable.h> #include <__type_traits/remove_cv.h> +#include <__utility/empty.h> #include <__utility/integer_sequence.h> #include <__utility/move.h> #include <__utility/unreachable.h> @@ -280,10 +281,10 @@ struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0> typedef std::reverse_iterator reverse_iterator; typedef std::reverse_iterator const_reverse_iterator; - typedef __conditional_t::value, const char, char> _CharType; + typedef __conditional_t::value, const __empty, __empty> _EmptyType; struct _ArrayInStructT { _Tp __data_[1]; }; - _ALIGNAS_TYPE(_ArrayInStructT) _CharType __elems_[sizeof(_ArrayInStructT)]; + _ALIGNAS_TYPE(_ArrayInStructT) _EmptyType __elems_[sizeof(_ArrayInStructT)]; _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT {return nullptr;} diff --git a/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp index ee77b1ac3c48b..ee1405f1f889d 100644 --- a/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp +++ b/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp @@ -199,7 +199,7 @@ void test() test_not_const(); // Sequence containers - test_not_const>(); + test_true >(); test_not_const>(); test_false >(); test_not_const>(); diff --git a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp index 9153106b384fc..7991d4738d969 100644 --- a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp +++ b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp @@ -18,26 +18,33 @@ struct NoDefault { TEST_CONSTEXPR NoDefault(int) { } }; -// Test default initialization -// This one isn't constexpr because omitting to initialize fundamental types -// isn't valid in a constexpr context. -struct test_default_initialization { +struct test_initialization { template - void operator()() const + TEST_CONSTEXPR_CXX14 void operator()() const { - std::array a0; (void)a0; - std::array a1; (void)a1; - std::array a2; (void)a2; - std::array a3; (void)a3; + // Check default initalization + { + std::array a0; (void)a0; + // Before C++20, default initialization doesn't work inside constexpr for + // trivially default constructible types. This only apply to non-empty arrays, + // since empty arrays don't hold an element of type T. + if (TEST_STD_AT_LEAST_20_OR_RUNTIME_EVALUATED || !std::is_trivially_default_constructible::value) { + std::array a1; (void)a1; + std::array a2; (void)a2; + std::array a3; (void)a3; + } - std::array nodefault; (void)nodefault; - } -}; + std::array nodefault; (void)nodefault; + } + + // A const empty array can also be default-initialized regardless of the type + // it contains. For non-empty arrays, this doesn't work whenever T doesn't + // have a user-provided default constructor. + { + const std::array a0; (void)a0; + const std::array nodefault; (void)nodefault; + } -struct test_nondefault_initialization { - template - TEST_CONSTEXPR_CXX14 void operator()() const - { // Check direct-list-initialization syntax (introduced in C++11) #if TEST_STD_VER >= 11 { @@ -174,13 +181,26 @@ TEST_CONSTEXPR_CXX14 bool with_all_types() return true; } +// This is a regression test -- previously, libc++ would implement empty arrays by +// storing an array of characters, which means that the array would be initializable +// from nonsense like an integer (or anything else that can be narrowed to char). +#if TEST_STD_VER >= 20 +template +concept is_list_initializable_int = requires { + { T{123} }; +}; + +struct Foo { }; +static_assert(!is_list_initializable_int>); +static_assert(!is_list_initializable_int>); +#endif + int main(int, char**) { - with_all_types(); - with_all_types(); // not constexpr + with_all_types(); test_initializer_list(); #if TEST_STD_VER >= 14 - static_assert(with_all_types(), ""); + static_assert(with_all_types(), ""); static_assert(test_initializer_list(), ""); #endif diff --git a/libcxx/test/std/containers/sequences/array/size_and_alignment.compile.pass.cpp b/libcxx/test/std/containers/sequences/array/size_and_alignment.compile.pass.cpp new file mode 100644 index 0000000000000..b7fb40c988678 --- /dev/null +++ b/libcxx/test/std/containers/sequences/array/size_and_alignment.compile.pass.cpp @@ -0,0 +1,143 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// template +// struct array + +// Make sure std::array has the correct object size and alignment. +// This test is mostly meant to catch subtle ABI-breaking regressions. + +// Ignore error about requesting a large alignment not being ABI compatible with older AIX systems. +#if defined(_AIX) +# pragma clang diagnostic ignored "-Waix-compat" +#endif + +#include +#include +#include +#include <__type_traits/datasizeof.h> + +#include "test_macros.h" + +template +struct MyArray { + T elems[Size]; +}; + +template +void test_type() { + { + using Array = std::array; + LIBCPP_STATIC_ASSERT(sizeof(Array) == sizeof(T), ""); + LIBCPP_STATIC_ASSERT(TEST_ALIGNOF(Array) == TEST_ALIGNOF(T), ""); + LIBCPP_STATIC_ASSERT(sizeof(Array) == sizeof(T[1]), ""); + LIBCPP_STATIC_ASSERT(sizeof(Array) == sizeof(MyArray), ""); + LIBCPP_STATIC_ASSERT(TEST_ALIGNOF(Array) == TEST_ALIGNOF(MyArray), ""); + static_assert(!std::is_empty::value, ""); + + // Make sure empty arrays don't have padding bytes + LIBCPP_STATIC_ASSERT(std::__libcpp_datasizeof::value == sizeof(Array), ""); + } + + { + using Array = std::array; + static_assert(sizeof(Array) == sizeof(T), ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(T), ""); + static_assert(sizeof(Array) == sizeof(T[1]), ""); + static_assert(sizeof(Array) == sizeof(MyArray), ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(MyArray), ""); + static_assert(!std::is_empty::value, ""); + } + + { + using Array = std::array; + static_assert(sizeof(Array) == sizeof(T) * 2, ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(T), ""); + static_assert(sizeof(Array) == sizeof(T[2]), ""); + static_assert(sizeof(Array) == sizeof(MyArray), ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(MyArray), ""); + static_assert(!std::is_empty::value, ""); + } + + { + using Array = std::array; + static_assert(sizeof(Array) == sizeof(T) * 3, ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(T), ""); + static_assert(sizeof(Array) == sizeof(T[3]), ""); + static_assert(sizeof(Array) == sizeof(MyArray), ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(MyArray), ""); + static_assert(!std::is_empty::value, ""); + } + + { + using Array = std::array; + static_assert(sizeof(Array) == sizeof(T) * 444, ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(T), ""); + static_assert(sizeof(Array) == sizeof(T[444]), ""); + static_assert(sizeof(Array) == sizeof(MyArray), ""); + static_assert(TEST_ALIGNOF(Array) == TEST_ALIGNOF(MyArray), ""); + static_assert(!std::is_empty::value, ""); + } +} + +struct Empty {}; + +struct Aggregate { + int i; +}; + +struct WithPadding { + long double ld; + char c; +}; + +#if TEST_STD_VER >= 11 +struct alignas(TEST_ALIGNOF(std::max_align_t) * 2) Overaligned1 {}; + +struct alignas(TEST_ALIGNOF(std::max_align_t) * 2) Overaligned2 { + char data[1000]; +}; + +struct alignas(TEST_ALIGNOF(std::max_align_t)) Overaligned3 { + char data[1000]; +}; + +struct alignas(8) Overaligned4 { + char c; +}; + +struct alignas(8) Overaligned5 {}; +#endif + +void test() { + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + +#if TEST_STD_VER >= 11 + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); + test_type(); +#endif +} diff --git a/libcxx/test/std/containers/sequences/array/size_and_alignment.pass.cpp b/libcxx/test/std/containers/sequences/array/size_and_alignment.pass.cpp deleted file mode 100644 index 6fbc844a11eac..0000000000000 --- a/libcxx/test/std/containers/sequences/array/size_and_alignment.pass.cpp +++ /dev/null @@ -1,78 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template -// struct array - -// Test the size and alignment matches that of an array of a given type. - -// Ignore error about requesting a large alignment not being ABI compatible with older AIX systems. -#if defined(_AIX) -# pragma clang diagnostic ignored "-Waix-compat" -#endif - -#include -#include -#include -#include - -#include "test_macros.h" - -template -struct MyArray { - T elems[Size]; -}; - -template -void test() { - typedef T CArrayT[Size == 0 ? 1 : Size]; - typedef std::array ArrayT; - typedef MyArray MyArrayT; - static_assert(sizeof(ArrayT) == sizeof(CArrayT), ""); - static_assert(sizeof(ArrayT) == sizeof(MyArrayT), ""); - static_assert(TEST_ALIGNOF(ArrayT) == TEST_ALIGNOF(MyArrayT), ""); -} - -template -void test_type() { - test(); - test(); - test(); -} - -#if TEST_STD_VER >= 11 -struct alignas(alignof(std::max_align_t) * 2) TestType1 { - -}; - -struct alignas(alignof(std::max_align_t) * 2) TestType2 { - char data[1000]; -}; - -struct alignas(alignof(std::max_align_t)) TestType3 { - char data[1000]; -}; -#endif - -int main(int, char**) { - test_type(); - test_type(); - test_type(); - test_type(); - -#if TEST_STD_VER >= 11 - test_type(); - test_type(); - test_type(); - test_type(); -#endif - - return 0; -} diff --git a/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h b/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h index bf08c580f3746..18d4f4b61fb23 100644 --- a/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h +++ b/libcxx/test/std/containers/views/mdspan/extents/CtorTestCombinations.h @@ -45,7 +45,7 @@ constexpr void test_construction(AllExtents all_ext) { // test construction from just dynamic extents // create an array of just the extents corresponding to dynamic values - std::array dyn_ext{0}; + std::array dyn_ext{}; size_t dynamic_idx = 0; for (size_t r = 0; r < E::rank(); r++) { if (E::static_extent(r) == std::dynamic_extent) { From 5f423b7d1cb3474168d79827d2305b137be7160b Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 15 Dec 2023 13:06:00 -0800 Subject: [PATCH 018/884] [llvm-exegesis] Adjust page size in unit tests to fix ppc failures The llvm-exegesis unit tests currently fail on PPC after ceb196d9903f4db7250bbc6c8da13eeae1b85886 landed as the default page size on most common linux distributions for PPC is 64kb rather than 4kb. This patch changes the memory mappings to have addresses as multiples of 64kb rather than multiples of 4kb to fix this issue. --- .../llvm-exegesis/X86/SnippetFileTest.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp index 1638ede7f3d42..505a030675f64 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp @@ -147,8 +147,8 @@ TEST_F(X86SnippetFileTest, NoAsmStreamer) { TEST_F(X86SnippetFileTest, MemoryDefinitionTestSingleDef) { auto Snippets = TestCommon(R"( # LLVM-EXEGESIS-MEM-DEF test1 4096 ff - # LLVM-EXEGESIS-MEM-MAP test1 8192 - # LLVM-EXEGESIS-MEM-MAP test1 16384 + # LLVM-EXEGESIS-MEM-MAP test1 65536 + # LLVM-EXEGESIS-MEM-MAP test1 131072 movq $8192, %r10 movq (%r10), %r11 )"); @@ -158,16 +158,16 @@ TEST_F(X86SnippetFileTest, MemoryDefinitionTestSingleDef) { ASSERT_THAT(Snippet.Key.MemoryValues, UnorderedElementsAre(MemoryDefinitionIs("test1", 255, 4096))); ASSERT_THAT(Snippet.Key.MemoryMappings, - ElementsAre(MemoryMappingIs(8192, "test1"), - MemoryMappingIs(16384, "test1"))); + ElementsAre(MemoryMappingIs(65536, "test1"), + MemoryMappingIs(131072, "test1"))); } TEST_F(X86SnippetFileTest, MemoryDefinitionsTestTwoDef) { auto Snippets = TestCommon(R"( # LLVM-EXEGESIS-MEM-DEF test1 4096 ff # LLVM-EXEGESIS-MEM-DEF test2 4096 100 - # LLVM-EXEGESIS-MEM-MAP test1 8192 - # LLVM-EXEGESIS-MEM-MAP test2 16384 + # LLVM-EXEGESIS-MEM-MAP test1 65536 + # LLVM-EXEGESIS-MEM-MAP test2 131072 movq $8192, %r10 movq (%r10), %r11 )"); @@ -178,8 +178,8 @@ TEST_F(X86SnippetFileTest, MemoryDefinitionsTestTwoDef) { UnorderedElementsAre(MemoryDefinitionIs("test1", 255, 4096), MemoryDefinitionIs("test2", 256, 4096))); ASSERT_THAT(Snippet.Key.MemoryMappings, - ElementsAre(MemoryMappingIs(8192, "test1"), - MemoryMappingIs(16384, "test2"))); + ElementsAre(MemoryMappingIs(65536, "test1"), + MemoryMappingIs(131072, "test2"))); } TEST_F(X86SnippetFileTest, MemoryDefinitionMissingParameter) { @@ -202,7 +202,7 @@ TEST_F(X86SnippetFileTest, MemoryMappingMissingParameters) { TEST_F(X86SnippetFileTest, MemoryMappingNoDefinition) { auto Error = TestCommon(R"( - # LLVM-EXEGESIS-MEM-MAP test1 4096 + # LLVM-EXEGESIS-MEM-MAP test1 65536 )") .takeError(); EXPECT_TRUE((bool)Error); From 9a578a9f602547bb7b0cdf54d4759eab20d150b3 Mon Sep 17 00:00:00 2001 From: Paul Kirth Date: Fri, 15 Dec 2023 13:32:39 -0800 Subject: [PATCH 019/884] Revert "[StackColoring] Delete dead stack slots (#75351)" (#75655) This reverts commit 08b306dc8e7c0b2498f4f194a3c51686d56dbd20. it causes the following assertion failure: llvm/include/llvm/CodeGen/MachineFrameInfo.h:530: int64_t llvm::MachineFrameInfo::getObjectOffset(int) const: Assertion `!isDeadObjectIndex(ObjectIdx) && "Getting frame offset for a dead object?"' failed. --- llvm/lib/CodeGen/StackColoring.cpp | 20 ++------------- .../CodeGen/PowerPC/aix32-cc-abi-vaarg.ll | 1 + .../CodeGen/PowerPC/aix64-cc-abi-vaarg.ll | 8 ++++-- llvm/test/CodeGen/RISCV/dead-stack-slot.ll | 25 ------------------- llvm/test/CodeGen/X86/StackColoring-tbaa.mir | 2 +- llvm/test/DebugInfo/COFF/lexicalblock.ll | 24 ++++++++++++++++-- 6 files changed, 32 insertions(+), 48 deletions(-) delete mode 100644 llvm/test/CodeGen/RISCV/dead-stack-slot.ll diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp index fa01aa17b3a86..37f7aa9290054 100644 --- a/llvm/lib/CodeGen/StackColoring.cpp +++ b/llvm/lib/CodeGen/StackColoring.cpp @@ -900,15 +900,6 @@ void StackColoring::remapInstructions(DenseMap &SlotRemap) { unsigned FixedMemOp = 0; unsigned FixedDbg = 0; - // Remove debug information for deleted slots. - erase_if(MF->getVariableDbgInfo(), [&](auto &VI) { - if (!VI.inStackSlot()) - return false; - int Slot = VI.getStackSlot(); - return Slot >= 0 && Intervals[Slot]->empty() && - InterestingSlots.test(Slot) && !ConservativeSlots.test(Slot); - }); - // Remap debug information that refers to stack slots. for (auto &VI : MF->getVariableDbgInfo()) { if (!VI.Var || !VI.inStackSlot()) @@ -1259,15 +1250,8 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) { // Do not bother looking at empty intervals. for (unsigned I = 0; I < NumSlots; ++I) { - int Slot = SortedSlots[I]; - if (Intervals[Slot]->empty()) { - if (InterestingSlots.test(Slot) && !ConservativeSlots.test(Slot)) { - RemovedSlots += 1; - ReducedSize += MFI->getObjectSize(Slot); - MFI->RemoveStackObject(Slot); - } + if (Intervals[SortedSlots[I]]->empty()) SortedSlots[I] = -1; - } } // This is a simple greedy algorithm for merging allocas. First, sort the @@ -1355,7 +1339,7 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) { // Scan the entire function and update all machine operands that use frame // indices to use the remapped frame index. - if (RemovedSlots > 0) { + if (!SlotRemap.empty()) { expungeSlotMap(SlotRemap, NumSlots); remapInstructions(SlotRemap); } diff --git a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll index 1b0a803734ae9..bf66a1ed042d2 100644 --- a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll +++ b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll @@ -347,6 +347,7 @@ entry: ; 32BIT-LABEL: stack: ; 32BIT-DAG: - { id: 0, name: arg1, type: default, offset: 0, size: 4, alignment: 4, +; 32BIT-DAG: - { id: 1, name: arg2, type: default, offset: 0, size: 4, alignment: 4, ; 32BIT-DAG: - { id: 2, name: '', type: default, offset: 0, size: 8, alignment: 8, ; 32BIT-DAG: - { id: 3, name: '', type: default, offset: 0, size: 8, alignment: 8, diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll index a8684fdfe1c56..ccf89aac2d540 100644 --- a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll +++ b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll @@ -138,7 +138,9 @@ ; 64BIT-LABEL: fixedStack: ; 64BIT-DAG: - { id: 0, type: default, offset: 112, size: 8, alignment: 16, stack-id: default, -; 64BIT-LABEL: stack: [] +; 64BIT-LABEL: stack: +; 64BIT-DAG: - { id: 0, name: arg1, type: default, offset: 0, size: 8, alignment: 8, +; 64BIT-DAG: - { id: 1, name: arg2, type: default, offset: 0, size: 8, alignment: 8, ; 64BIT-LABEL: body: | ; 64BIT-DAG: liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10 @@ -303,7 +305,9 @@ ; 64BIT-LABEL: fixedStack: ; 64BIT-DAG: - { id: 0, type: default, offset: 152, size: 8 -; 64BIT-LABEL: stack: [] +; 64BIT-LABEL: stack: +; 64BIT-DAG: - { id: 0, name: arg1, type: default, offset: 0, size: 8 +; 64BIT-DAG: - { id: 1, name: arg2, type: default, offset: 0, size: 8 ; 64BIT-LABEL: body: | ; 64BIT-DAG: liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13 diff --git a/llvm/test/CodeGen/RISCV/dead-stack-slot.ll b/llvm/test/CodeGen/RISCV/dead-stack-slot.ll deleted file mode 100644 index 49b0d2ab58c4f..0000000000000 --- a/llvm/test/CodeGen/RISCV/dead-stack-slot.ll +++ /dev/null @@ -1,25 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \ -; RUN: | FileCheck %s -; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \ -; RUN: | FileCheck %s - -; Remove the lifetime-marked alloca, but not the unmarked one. -define signext i32 @f1() nounwind { -; CHECK-LABEL: f1: -; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -32 -; CHECK-NEXT: li a0, 0 -; CHECK-NEXT: addi sp, sp, 32 -; CHECK-NEXT: ret - %1 = alloca [32 x i8], align 4 - %2 = alloca [32 x i8], align 4 - %3 = getelementptr inbounds [32 x i8], ptr %1, i64 0, i64 0 - call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %3) - call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %3) - ret i32 0 -} - -declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) -declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) - diff --git a/llvm/test/CodeGen/X86/StackColoring-tbaa.mir b/llvm/test/CodeGen/X86/StackColoring-tbaa.mir index b4fdf4d2ec917..6d7f294549d79 100644 --- a/llvm/test/CodeGen/X86/StackColoring-tbaa.mir +++ b/llvm/test/CodeGen/X86/StackColoring-tbaa.mir @@ -53,7 +53,7 @@ body: | ; CHECK: [[LEA64r:%[0-9]+]]:gr64 = nuw LEA64r %stack.1.agg, 1, $noreg, 24, $noreg ; CHECK-NEXT: CMP8mi %stack.1.agg, 1, $noreg, 47, $noreg, 0, implicit-def $eflags :: (dereferenceable load (s8) from %ir.a22, !tbaa !2) ; CHECK-NEXT: [[CMOV64rm:%[0-9]+]]:gr64 = CMOV64rm [[LEA64r]], %stack.1.agg, 1, $noreg, 24, $noreg, 8, implicit $eflags :: (dereferenceable load (s64) from %ir.a2) - ; CHECK-NEXT: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm killed [[CMOV64rm]], 1, $noreg, 16, $noreg :: (load (s8) from %ir.add.ptr.i) + ; CHECK-NEXT: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm killed [[CMOV64rm]], 1, $noreg, 16, $noreg :: (load (s8) from %ir.add.ptr.i, !tbaa !2) ; CHECK-NEXT: $al = COPY [[MOV8rm]] ; CHECK-NEXT: RET 0, $al LIFETIME_START %stack.0.padding diff --git a/llvm/test/DebugInfo/COFF/lexicalblock.ll b/llvm/test/DebugInfo/COFF/lexicalblock.ll index 3bfae85f6c9ba..40dd8f894252c 100644 --- a/llvm/test/DebugInfo/COFF/lexicalblock.ll +++ b/llvm/test/DebugInfo/COFF/lexicalblock.ll @@ -63,12 +63,32 @@ ; CHECK: LocalSym { ; CHECK: VarName: localA ; CHECK: } +; CHECK: LocalSym { +; CHECK: VarName: localB +; CHECK: } ; CHECK: BlockSym { ; CHECK: Kind: S_BLOCK32 {{.*}} ; CHECK: BlockName: ; CHECK: } -; CHECK: LocalSym { -; CHECK: VarName: localB +; CHECK: ScopeEndSym { +; CHECK: Kind: S_END {{.*}} +; CHECK: } +; CHECK: BlockSym { +; CHECK: Kind: S_BLOCK32 {{.*}} +; CHECK: BlockName: +; CHECK: } +; CHECK: ScopeEndSym { +; CHECK: Kind: S_END {{.*}} +; CHECK: } +; CHECK: BlockSym { +; CHECK: Kind: S_BLOCK32 {{.*}} +; CHECK: BlockName: +; CHECK: } +; CHECK: ScopeEndSym { +; CHECK: } +; CHECK: BlockSym { +; CHECK: Kind: S_BLOCK32 {{.*}} +; CHECK: BlockName: ; CHECK: } ; CHECK: ScopeEndSym { ; CHECK: Kind: S_END {{.*}} From 21edad7d076c7fffbbf8ffdae3a83d93cb39b639 Mon Sep 17 00:00:00 2001 From: Peiming Liu <36770114+PeimingLiu@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:34:34 -0800 Subject: [PATCH 020/884] [mlir][sparse] set up the skeleton for SparseTensorLevel abstraction. (#75645) Note that at the current moment, the newly-introduced `SparseTensorLevel` classes are far from complete, we plan to migrate code generation related to accessing sparse tensor levels to these classes in the near future to simplify `LoopEmitter`. --- .../SparseTensor/Transforms/CMakeLists.txt | 1 + .../Transforms/Utils/LoopEmitter.cpp | 92 ++++++--------- .../Transforms/Utils/LoopEmitter.h | 11 +- .../Transforms/Utils/SparseTensorLevel.cpp | 54 +++++++++ .../Transforms/Utils/SparseTensorLevel.h | 109 ++++++++++++++++++ 5 files changed, 203 insertions(+), 64 deletions(-) create mode 100644 mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp create mode 100644 mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt index ad8b0d02eca35..456e45a040193 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt @@ -19,6 +19,7 @@ add_mlir_dialect_library(MLIRSparseTensorTransforms Utils/IterationGraphSorter.cpp Utils/LoopEmitter.cpp Utils/SparseTensorDescriptor.cpp + Utils/SparseTensorLevel.cpp ADDITIONAL_HEADER_DIRS ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SparseTensor diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp index 784c793c9bd11..0ba7cf33b6cba 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp @@ -126,15 +126,15 @@ static std::pair fromSliceCrd(OpBuilder &builder, Location loc, // Generates a bool value for while loop condition that tries to iterate over a // fully reduced level with affine index expression. static Value genSparseReducedAffineCond(OpBuilder &builder, Location loc, - Value crdBuf, Value crdHi, Value posit, - Value posHi) { + const SparseTensorLevel &level, + Value crdHi, Value posit, Value posHi) { Value inBound = CMPI(ult, posit, posHi); auto ifOp = builder.create(loc, builder.getI1Type(), inBound, true); // if (inbound) // yield coord < crdHi builder.setInsertionPointToStart(&ifOp.getThenRegion().front()); - Value crd = genIndexLoad(builder, loc, crdBuf, posit); + Value crd = level.peekCrdAt(builder, loc, posit); YIELD(CMPI(ult, crd, crdHi)); // else // yield false @@ -244,13 +244,12 @@ Value LoopEmitter::genAddress(OpBuilder &builder, Location loc, TensorId tid, Value LoopEmitter::genSegmentHigh(OpBuilder &builder, Location loc, TensorId tid, Level lvl, Value pLo, Value pHi) { - const auto coordinates = coordinatesBuffers[tid][lvl]; - const auto sameCrd = genIndexLoad(builder, loc, coordinates, pLo); + SparseTensorLevel &level = *lvls[tid][lvl]; + const Value sameCrd = level.peekCrdAt(builder, loc, pLo); auto whileOp = builder.create( loc, builder.getIndexType(), pLo, /*beforeBuilder=*/ - [pHi, coordinates, sameCrd](OpBuilder &builder, Location loc, - ValueRange ivs) { + [pHi, &level, sameCrd](OpBuilder &builder, Location loc, ValueRange ivs) { const auto pos = ivs[0]; Value inBound = builder.create( loc, arith::CmpIPredicate::ult, pos, pHi); @@ -261,7 +260,7 @@ Value LoopEmitter::genSegmentHigh(OpBuilder &builder, Location loc, // Load the next coordinates only when inbound (to avoid OOB // accesses). builder.setInsertionPointToStart(ifInBound.thenBlock()); - Value crd = genIndexLoad(builder, loc, coordinates, pos); + Value crd = level.peekCrdAt(builder, loc, pos); Value isSameCrd = builder.create( loc, arith::CmpIPredicate::eq, crd, sameCrd); YIELD(isSameCrd); @@ -284,11 +283,8 @@ Value LoopEmitter::genSegmentHigh(OpBuilder &builder, Location loc, Value LoopEmitter::genSparseCrd(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { - // A load on the coordinates array yields the coordinate. - const Value mem = coordinatesBuffers[tid][lvl]; - /// FIXME: See the [CLARIFY_POSITS_LVL] note in the header. const Value pos = posits[tid][lvl]; - const Value crd = genIndexLoad(builder, loc, mem, pos); + const Value crd = lvls[tid][lvl]->peekCrdAt(builder, loc, pos); return crd; } @@ -318,9 +314,8 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, this->segHi.assign(numTensors, std::vector()); this->posits.assign(numTensors, std::vector()); this->coords.assign(numTensors, std::vector()); - this->positionsBuffers.assign(numTensors, std::vector()); - this->coordinatesBuffers.assign(numTensors, std::vector()); this->valBuffer.assign(numTensors, nullptr); + this->lvls.resize(numTensors); this->isSparseSlices.assign(numTensors, false); this->sliceOffsets.assign(numTensors, std::vector()); this->sliceStrides.assign(numTensors, std::vector()); @@ -377,8 +372,8 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, segHi[tid].assign(lvlRank, Value()); posits[tid].assign(lvlRank, Value()); coords[tid].assign(lvlRank, Value()); - positionsBuffers[tid].assign(lvlRank, Value()); - coordinatesBuffers[tid].assign(lvlRank, Value()); + lvls[tid].resize(lvlRank); + sliceOffsets[tid].assign(lvlRank, Value()); sliceStrides[tid].assign(lvlRank, Value()); @@ -448,22 +443,7 @@ void LoopEmitter::initializeLoopEmit( // Scan all levels of current tensor. for (Level l = 0; l < lvlRank; l++) { - // This should be called only once at beginning. - assert(!positionsBuffers[t][l] && !coordinatesBuffers[t][l] && - !highs[t][l]); - const auto lvlTp = lvlTypes[t][l]; - // Handle sparse storage schemes. - if (isCompressedLT(lvlTp) || isLooseCompressedLT(lvlTp)) { - // Generate sparse primitives to obtain positions and coordinates. - positionsBuffers[t][l] = genToPositions(builder, loc, tensor, l); - coordinatesBuffers[t][l] = genToCoordinates(builder, loc, tensor, l); - } else if (isSingletonLT(lvlTp) || is2OutOf4LT(lvlTp)) { - // Singleton level, fetch coordinates. - coordinatesBuffers[t][l] = genToCoordinates(builder, loc, tensor, l); - } else { - // Dense level, nothing to fetch. - assert(isDenseLT(lvlTp)); - } + lvls[t][l] = makeSparseTensorLevel(builder, loc, tensor, l); // Find upper bound in current dimension. highs[t][l] = lvlSizes[t][l] = lvlSzs[l]; @@ -756,8 +736,7 @@ Value LoopEmitter::genWhileLoopConditions(OpBuilder &builder, Location loc, crdHi = ADDI(getMostRecentSliceOnLvl(tid, lvl).offset, remSz); } assert(crdHi); - return genSparseReducedAffineCond(builder, loc, - coordinatesBuffers[tid][lvl], crdHi, + return genSparseReducedAffineCond(builder, loc, *lvls[tid][lvl], crdHi, ivs[0], highs[tid][lvl]); } case LoopCondKind::SparseAffineUnRedCond: { @@ -802,10 +781,9 @@ std::optional LoopEmitter::genWhileLoopBody(OpBuilder &builder, sliceTupleFwdCnt[tid][lvl] = SUBI(ivs[0], posits[tid][lvl]); // Update c = absOffset[lvl][depth] - absOffset[lvl][depth - 1] Value posit = ivs[0]; - Value crdBuf = coordinatesBuffers[tid][lvl]; // We need to substract the offset to get relative coordinates. // TODO: Maybe assert relC >=0 during runtime in debug build? - Value absC = genIndexLoad(builder, loc, crdBuf, posit); + Value absC = lvls[tid][lvl]->peekCrdAt(builder, loc, posit); auto relC = SUBI(absC, getFinalSliceOnLvl(tid, lvl).offset); posits[tid][lvl] = posit; coords[tid][lvl] = relC; @@ -1189,9 +1167,7 @@ Operation *LoopEmitter::enterFilterLoopOverTensorAtLvl( // The induction variable gives the position. const Value pos = forOp.getInductionVar(); posits[tid][lvl] = pos; - // Generating a load on the coordinates array yields the crd. - const Value mem = coordinatesBuffers[tid][lvl]; - const Value crd = genIndexLoad(builder, loc, mem, pos); + const Value crd = lvls[tid][lvl]->peekCrdAt(builder, loc, pos); coords[tid][lvl] = crd; // Generate an if-condition to filter out coordinates that are not @@ -1255,7 +1231,11 @@ void LoopEmitter::prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc, /// FIXME: See the [CLARIFY_POSITS_LVL] note in the header. assert(lvl == 0 || posits[tid][lvl - 1]); if (isCompressedLT(lvlTp) || isLooseCompressedLT(lvlTp)) { - const Value mem = positionsBuffers[tid][lvl]; + // TODO: eliminate the cast upon feature complete. + const Value mem = + isCompressedLT(lvlTp) + ? static_cast(*lvls[tid][lvl]).posBuffer + : static_cast(*lvls[tid][lvl]).posBuffer; Value pLo = lvl == 0 ? c0 : posits[tid][lvl - 1]; if (isLooseCompressedLT(lvlTp)) @@ -1623,8 +1603,7 @@ std::pair LoopEmitter::genSliceLvlTraverseLoop( /*beforeBuilder=*/ [this, posHi, sliceHi, tid, lvl](OpBuilder &builder, Location loc, ValueRange args) { - Value cond = genSparseReducedAffineCond(builder, loc, - coordinatesBuffers[tid][lvl], + Value cond = genSparseReducedAffineCond(builder, loc, *lvls[tid][lvl], sliceHi, args[0], posHi); // continue if not yet break nor out of bound. builder.create(loc, cond, args); @@ -1848,12 +1827,14 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, Value pHi, pLo; if (lvl == 0) { pLo = c0; - pHi = genIndexLoad(builder, loc, positionsBuffers[tid][0], c1); + // TODO: eliminate the cast upon feature complete.pLo = c0; + Value pBuf = static_cast(*lvls[tid][0]).posBuffer; + pHi = genIndexLoad(builder, loc, pBuf, c1); } else { - pLo = genIndexLoad(builder, loc, positionsBuffers[tid][lvl], - posits[tid][lvl - 1]); - pHi = genIndexLoad(builder, loc, positionsBuffers[tid][lvl], - ADDI(posits[tid][lvl - 1], c1)); + // TODO: eliminate the cast upon feature complete.} else { + Value pBuf = static_cast(*lvls[tid][lvl]).posBuffer; + pLo = genIndexLoad(builder, loc, pBuf, posits[tid][lvl - 1]); + pHi = genIndexLoad(builder, loc, pBuf, ADDI(posits[tid][lvl - 1], c1)); } // Fills out pIdxBuffer[tid][lvl][0] with [pLo, pHi] updateSlicePos(builder, loc, sPtrBuf, pLo, c0, SlicePosKind::kLo); @@ -1868,7 +1849,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // nonempty. though we assume that even on empty sparse tensors, a non-empty // ptr/idx buffer is allocated for each level so it would not cause OOB to // avoid generating a ifOp here. - Value minCrd = genIndexLoad(builder, loc, coordinatesBuffers[tid][lvl], pLo); + Value minCrd = lvls[tid][lvl]->peekCrdAt(builder, loc, pLo); // FIXME: We need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, nxSz, isNonEmpty); @@ -1955,9 +1936,10 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, Value &curTupleCnt = reduc[2]; Value pHi = ADDI(iv, c1); - Value sPLo = genIndexLoad(builder, loc, positionsBuffers[tid][lvl], iv); - Value sPHi = - genIndexLoad(builder, loc, positionsBuffers[tid][lvl], pHi); + // TODO: eliminate the cast upon feature complete. + Value pBuf = static_cast(*lvls[tid][lvl]).posBuffer; + Value sPLo = genIndexLoad(builder, loc, pBuf, iv); + Value sPHi = genIndexLoad(builder, loc, pBuf, pHi); // isNonEmpty = isNonEmpty || lvlNonEmpty, i.e., as long as there is // one non-empty lvl, the slice is non-empty. @@ -1975,8 +1957,7 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, // } OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToStart(ifNonEmpty.thenBlock()); - Value curC = - genIndexLoad(builder, loc, coordinatesBuffers[tid][lvl], sPLo); + Value curC = lvls[tid][lvl]->peekCrdAt(builder, loc, sPLo); Value isSmaller = CMPI(ult, curC, minCrd); Value newMin = SELECT(isSmaller, curC, minCrd); YIELD(newMin); @@ -2176,8 +2157,7 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, /* if pLo < pHi */ { builder.setInsertionPointToStart(&advPLo.getThenRegion().front()); // coord = load[pLo] - Value coord = - genIndexLoad(builder, loc, coordinatesBuffers[tid][lvl], pLo); + Value coord = lvls[tid][lvl]->peekCrdAt(builder, loc, pLo); Value pred = CMPI(eq, coord, info.minCrd); auto ifEqual = builder.create(loc, idxTp, pred, true); /* if coord == minCrd */ { @@ -2209,7 +2189,7 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, auto newMin = builder.create(loc, idxTp, lvlNonEmpty, true); builder.setInsertionPointToStart(&newMin.getThenRegion().front()); - YIELD(genIndexLoad(builder, loc, coordinatesBuffers[tid][lvl], pLo)); + YIELD(lvls[tid][lvl]->peekCrdAt(builder, loc, pLo)); builder.setInsertionPointToStart(&newMin.getElseRegion().front()); YIELD(curMinCrd); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h index 78bb53e4483f6..eb577ee4acefe 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h @@ -11,6 +11,8 @@ #include +#include "SparseTensorLevel.h" + #include "mlir/Dialect/SparseTensor/IR/Enums.h" #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" #include "mlir/Dialect/SparseTensor/Utils/Merger.h" @@ -241,12 +243,6 @@ class LoopEmitter { const std::vector> &getPosits() const { return posits; }; const std::vector> &getCoords() const { return coords; }; const std::vector> &getHighs() const { return highs; }; - const std::vector> &getPositionBuffers() const { - return positionsBuffers; - }; - const std::vector> &getCoordinateBuffers() const { - return coordinatesBuffers; - }; const std::vector &getValBuffer() const { return valBuffer; }; constexpr static llvm::StringLiteral getLoopEmitterLoopAttrName() { @@ -648,8 +644,7 @@ class LoopEmitter { std::vector> segHi; std::vector> highs; std::vector> lvlSizes; - std::vector> positionsBuffers; // to_positions - std::vector> coordinatesBuffers; // to_coordinates + std::vector>> lvls; std::vector valBuffer; // to_value // diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp new file mode 100644 index 0000000000000..d9d26794d7bce --- /dev/null +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp @@ -0,0 +1,54 @@ +//===- SparseTensorLevel.cpp - Tensor management class -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SparseTensorLevel.h" +#include "CodegenUtils.h" + +#include "mlir/Dialect/Tensor/IR/Tensor.h" + +using namespace mlir; +using namespace mlir::sparse_tensor; + +std::unique_ptr +sparse_tensor::makeSparseTensorLevel(OpBuilder &builder, Location loc, Value t, + Level l) { + auto stt = getSparseTensorType(t); + + LevelType lt = stt.getLvlType(l); + Value lvlSz = stt.hasEncoding() + ? builder.create(loc, t, l).getResult() + : builder.create(loc, t, l).getResult(); + + switch (*getLevelFormat(lt)) { + case LevelFormat::Dense: + return std::make_unique(lvlSz); + case LevelFormat::Compressed: { + Value posBuf = genToPositions(builder, loc, t, l); + Value crdBuf = genToCoordinates(builder, loc, t, l); + return std::make_unique(lt, lvlSz, posBuf, crdBuf); + } + case LevelFormat::LooseCompressed: { + Value posBuf = genToPositions(builder, loc, t, l); + Value crdBuf = genToCoordinates(builder, loc, t, l); + return std::make_unique(lt, lvlSz, posBuf, crdBuf); + } + case LevelFormat::Singleton: { + Value crdBuf = genToCoordinates(builder, loc, t, l); + return std::make_unique(lt, lvlSz, crdBuf); + } + case LevelFormat::TwoOutOfFour: { + Value crdBuf = genToCoordinates(builder, loc, t, l); + return std::make_unique(lt, lvlSz, crdBuf); + } + } + llvm_unreachable("unrecognizable level format"); +} + +Value SparseLevel::peekCrdAt(OpBuilder &b, Location l, Value pos) const { + return genIndexLoad(b, l, crdBuffer, pos); +} diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h new file mode 100644 index 0000000000000..e10356a55cc7e --- /dev/null +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h @@ -0,0 +1,109 @@ +//===- SparseTensorLevel.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSORLEVEL_H_ +#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSORLEVEL_H_ + +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" + +namespace mlir { +namespace sparse_tensor { + +class SparseTensorLevel { + SparseTensorLevel(SparseTensorLevel &&) = delete; + SparseTensorLevel(const SparseTensorLevel &) = delete; + +public: + SparseTensorLevel() : SparseTensorLevel(LevelType::Undef, nullptr){}; + virtual ~SparseTensorLevel() = default; + + virtual Value peekCrdAt(OpBuilder &b, Location l, Value p) const = 0; + + LevelType getLT() const { return lt; } + Value getPos() const { return pos; } + Value getCrd() const { return crd; } + Value getLoopHi() const { return loopHi; } + Value getLoopLo() const { return loopLo; } + +protected: + SparseTensorLevel(LevelType lt, Value lvlSize) + : lt(lt), lvlSize(lvlSize), pos(nullptr), crd(nullptr), loopHi(nullptr), + loopLo(nullptr){}; + + const LevelType lt; + const Value lvlSize; + +public: // TODO: make these values private upon feature complete. + Value pos; + Value crd; + Value loopHi; + Value loopLo; +}; + +/// Helper function to create a TensorLevel object from given `tensor`. +std::unique_ptr +makeSparseTensorLevel(OpBuilder &builder, Location loc, Value t, Level l); + +class DenseLevel : public SparseTensorLevel { +public: + DenseLevel(Value lvlSize) : SparseTensorLevel(LevelType::Dense, lvlSize) { + // Dense level, loop upper bound equals to the level size. + loopHi = lvlSize; + } + + Value peekCrdAt(OpBuilder &, Location, Value pos) const override { + return pos; + } +}; + +class SparseLevel : public SparseTensorLevel { +public: + SparseLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseTensorLevel(lt, lvlSize), crdBuffer(crdBuffer) {} + + Value peekCrdAt(OpBuilder &b, Location l, Value pos) const override; + +public: // TODO: make these values private upon feature complete. + const Value crdBuffer; +}; + +class CompressedLevel : public SparseLevel { +public: + CompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} + +public: // TODO: make these values private upon feature complete. + const Value posBuffer; +}; + +class LooseCompressedLevel : public SparseLevel { +public: + LooseCompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, + Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} + +public: // TODO: make these values private upon feature complete. + const Value posBuffer; +}; + +class SingletonLevel : public SparseLevel { +public: + SingletonLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer) {} +}; + +class TwoOutFourLevel : public SparseLevel { +public: + TwoOutFourLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer) {} +}; + +} // namespace sparse_tensor +} // namespace mlir + +#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSORLEVEL_H_ From 2439bc4a801df88b17764de50664e6b3a8e3c507 Mon Sep 17 00:00:00 2001 From: quic-akaryaki <123192073+quic-akaryaki@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:18:25 -0600 Subject: [PATCH 021/884] [llvm-objcopy] Fix gap-fill/pad-to tests (#75631) The tests added in PR #65815 fail on Apple buildbot because the `od` printed addresses have a different number of leading zeroes. Mask leading zeroes with a regex. To support the `od` output format on z/OS, add `--ignore-case` to FileCheck. --- .../test/tools/llvm-objcopy/ELF/gap-fill.test | 100 +++++++++--------- llvm/test/tools/llvm-objcopy/ELF/pad-to.test | 32 +++--- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/llvm/test/tools/llvm-objcopy/ELF/gap-fill.test b/llvm/test/tools/llvm-objcopy/ELF/gap-fill.test index fa6230e64bc77..c11909746330b 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/gap-fill.test +++ b/llvm/test/tools/llvm-objcopy/ELF/gap-fill.test @@ -23,66 +23,66 @@ ## Test no gap fill with all allocatable output sections. # RUN: llvm-objcopy -O binary %t %t-default -# RUN: od -v -Ax -t x1 %t-default | FileCheck %s --check-prefix=DEFAULT --match-full-lines -# DEFAULT: 000000 ee ff 11 22 33 44 aa bb cc dd fe dc ba 00 a1 b2 -# DEFAULT-NEXT: 000010 c3 d4 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000060 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000080 00 00 89 ab cd ef -# DEFAULT-NEXT: 000086 +# RUN: od -v -Ax -t x1 %t-default | FileCheck %s --check-prefix=DEFAULT --ignore-case --match-full-lines +# DEFAULT: {{0*}}00 ee ff 11 22 33 44 aa bb cc dd fe dc ba 00 a1 b2 +# DEFAULT-NEXT: {{0*}}10 c3 d4 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}30 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}40 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}60 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}70 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}80 00 00 89 ab cd ef +# DEFAULT-NEXT: {{0*}}86 ## Test gap fill with all allocatable output sections. # RUN: llvm-objcopy -O binary --gap-fill=0xe9 %t %t-filled -# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=FULL --match-full-lines -# FULL: 000000 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 a1 b2 -# FULL-NEXT: 000010 c3 d4 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000020 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000030 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000040 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000050 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000060 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000070 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FULL-NEXT: 000080 e9 e9 89 ab cd ef -# FULL-NEXT: 000086 +# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=FULL --ignore-case --match-full-lines +# FULL: {{0*}}00 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 a1 b2 +# FULL-NEXT: {{0*}}10 c3 d4 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}20 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}30 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}40 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}50 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}60 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}70 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FULL-NEXT: {{0*}}80 e9 e9 89 ab cd ef +# FULL-NEXT: {{0*}}86 ## Test gap fill with a decimal value. # RUN: llvm-objcopy -O binary --gap-fill=99 %t %t-filled-decimal -# RUN: od -v -Ax -t x1 %t-filled-decimal | FileCheck %s --check-prefix=DEC --match-full-lines -# DEC: 000000 ee ff 11 22 33 44 aa bb cc dd fe dc ba 63 a1 b2 -# DEC-NEXT: 000010 c3 d4 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000020 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000030 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000040 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000050 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000060 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000070 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 -# DEC-NEXT: 000080 63 63 89 ab cd ef -# DEC-NEXT: 000086 +# RUN: od -v -Ax -t x1 %t-filled-decimal | FileCheck %s --check-prefix=DEC --ignore-case --match-full-lines +# DEC: {{0*}}00 ee ff 11 22 33 44 aa bb cc dd fe dc ba 63 a1 b2 +# DEC-NEXT: {{0*}}10 c3 d4 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}20 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}30 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}40 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}50 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}60 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}70 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 63 +# DEC-NEXT: {{0*}}80 63 63 89 ab cd ef +# DEC-NEXT: {{0*}}86 ## Test gap fill with the last section removed, should be truncated. # RUN: llvm-objcopy -O binary --gap-fill=0xe9 --remove-section=.foo %t %t-filled -# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-LAST-SECTION --match-full-lines -# REMOVE-LAST-SECTION: 000000 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 a1 b2 -# REMOVE-LAST-SECTION-NEXT: 000010 c3 d4 -# REMOVE-LAST-SECTION-NEXT: 000012 +# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-LAST-SECTION --ignore-case --match-full-lines +# REMOVE-LAST-SECTION: {{0*}}00 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 a1 b2 +# REMOVE-LAST-SECTION-NEXT: {{0*}}10 c3 d4 +# REMOVE-LAST-SECTION-NEXT: {{0*}}12 ## Test gap fill with the middle section removed, should be filled. # RUN: llvm-objcopy -O binary --gap-fill=0xe9 --remove-section=.gap2 %t %t-filled -# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-MIDDLE-SECTION --match-full-lines -# REMOVE-MIDDLE-SECTION: 000000 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000010 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000020 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000030 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000040 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000050 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000060 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000070 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-MIDDLE-SECTION-NEXT: 000080 e9 e9 89 ab cd ef -# REMOVE-MIDDLE-SECTION-NEXT: 000086 +# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-MIDDLE-SECTION --ignore-case --match-full-lines +# REMOVE-MIDDLE-SECTION: {{0*}}00 ee ff 11 22 33 44 aa bb cc dd fe dc ba e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}10 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}20 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}30 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}40 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}50 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}60 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}70 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}80 e9 e9 89 ab cd ef +# REMOVE-MIDDLE-SECTION-NEXT: {{0*}}86 --- !ELF FileHeader: @@ -149,8 +149,8 @@ Sections: # RUN: yaml2obj --docnum=2 %s -o %t.2 # RUN: llvm-objcopy -O binary --gap-fill=0xe9 %t.2 %t.2.filled -# RUN: od -v -Ax -t x1 %t.2.filled | FileCheck --match-full-lines %s -# CHECK: 000000 aa bb cc dd e9 e9 e9 e9 11 22 33 44 +# RUN: od -v -Ax -t x1 %t.2.filled | FileCheck --ignore-case --match-full-lines %s +# CHECK: {{0*}}00 aa bb cc dd e9 e9 e9 e9 11 22 33 44 --- !ELF FileHeader: diff --git a/llvm/test/tools/llvm-objcopy/ELF/pad-to.test b/llvm/test/tools/llvm-objcopy/ELF/pad-to.test index eb3608860a543..5ce57f63e6cd3 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/pad-to.test +++ b/llvm/test/tools/llvm-objcopy/ELF/pad-to.test @@ -29,31 +29,31 @@ ## Pad all allocatable sections to a valid address. # RUN: llvm-objcopy -O binary --pad-to=0x218 %t %t-pad-default -# RUN: od -v -Ax -t x1 %t-pad-default | FileCheck %s --check-prefix=DEFAULT --match-full-lines -# DEFAULT: 000000 11 22 33 44 55 66 00 00 00 00 00 00 00 00 00 00 -# DEFAULT-NEXT: 000010 77 88 99 aa 00 00 00 00 -# DEFAULT-NEXT: 000018 +# RUN: od -v -Ax -t x1 %t-pad-default | FileCheck %s --check-prefix=DEFAULT --ignore-case --match-full-lines +# DEFAULT: {{0*}}00 11 22 33 44 55 66 00 00 00 00 00 00 00 00 00 00 +# DEFAULT-NEXT: {{0*}}10 77 88 99 aa 00 00 00 00 +# DEFAULT-NEXT: {{0*}}18 ## Use a decimal number for the padding address and verify it is not misunderstood. # RUN: llvm-objcopy -O binary --pad-to=536 %t %t-pad-decimal -# RUN: od -v -Ax -t x1 %t-pad-decimal | FileCheck %s --check-prefix=DECIMAL --match-full-lines -# DECIMAL: 000000 11 22 33 44 55 66 00 00 00 00 00 00 00 00 00 00 -# DECIMAL-NEXT: 000010 77 88 99 aa 00 00 00 00 -# DECIMAL-NEXT: 000018 +# RUN: od -v -Ax -t x1 %t-pad-decimal | FileCheck %s --check-prefix=DECIMAL --ignore-case --match-full-lines +# DECIMAL: {{0*}}00 11 22 33 44 55 66 00 00 00 00 00 00 00 00 00 00 +# DECIMAL-NEXT: {{0*}}10 77 88 99 aa 00 00 00 00 +# DECIMAL-NEXT: {{0*}}18 ## Pad all allocatable sections to a valid address, using --gap-fill. # RUN: llvm-objcopy -O binary --pad-to=0x218 --gap-fill=0xe9 %t %t-pad-fill -# RUN: od -v -Ax -t x1 %t-pad-fill | FileCheck %s --check-prefix=FILL --match-full-lines -# FILL: 000000 11 22 33 44 55 66 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# FILL-NEXT: 000010 77 88 99 aa e9 e9 e9 e9 -# FILL-NEXT: 000018 +# RUN: od -v -Ax -t x1 %t-pad-fill | FileCheck %s --check-prefix=FILL --ignore-case --match-full-lines +# FILL: {{0*}}00 11 22 33 44 55 66 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# FILL-NEXT: {{0*}}10 77 88 99 aa e9 e9 e9 e9 +# FILL-NEXT: {{0*}}18 ## Remove the last section containing data and test that the padded space is gap filled. # RUN: llvm-objcopy -O binary --pad-to=0x218 --gap-fill=0xe9 --remove-section=.section2 %t %t-filled -# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-SECTION --match-full-lines -# REMOVE-SECTION: 000000 11 22 33 44 55 66 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-SECTION-NEXT: 000010 e9 e9 e9 e9 e9 e9 e9 e9 -# REMOVE-SECTION-NEXT: 000018 +# RUN: od -v -Ax -t x1 %t-filled | FileCheck %s --check-prefix=REMOVE-SECTION --ignore-case --match-full-lines +# REMOVE-SECTION: {{0*}}00 11 22 33 44 55 66 e9 e9 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-SECTION-NEXT: {{0*}}10 e9 e9 e9 e9 e9 e9 e9 e9 +# REMOVE-SECTION-NEXT: {{0*}}18 --- !ELF FileHeader: From aaa3f72c1ce6e1757df79c0d02e0675201ee07a3 Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Fri, 15 Dec 2023 17:23:16 -0500 Subject: [PATCH 022/884] [PowerPC] Emit libcall to frexpl for calls to frexp(ppcDoublDouble) (#75226) On Linux PPC call lib func ``frexpl`` for calls to ``frexp()`` for input of type PPCDoubleDouble. Fixes bug: https://github.com/llvm/llvm-project/issues/64426 --- clang/lib/CodeGen/CGBuiltin.cpp | 9 ++++++++- clang/test/CodeGen/math-builtins-long.c | 2 +- .../numerics/c.math/constexpr-cxx23-clang.pass.cpp | 6 ------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 353b7930b3c1e..3327866d2b962 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3410,9 +3410,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, { Src0->getType(), Src1->getType() }); return RValue::get(Builder.CreateCall(F, { Src0, Src1 })); } + case Builtin::BI__builtin_frexpl: { + // Linux PPC will not be adding additional PPCDoubleDouble support. + // WIP to switch default to IEEE long double. Will emit libcall for + // frexpl instead of legalizing this type in the BE. + if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble()) + break; + LLVM_FALLTHROUGH; + } case Builtin::BI__builtin_frexp: case Builtin::BI__builtin_frexpf: - case Builtin::BI__builtin_frexpl: case Builtin::BI__builtin_frexpf128: case Builtin::BI__builtin_frexpf16: return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp)); diff --git a/clang/test/CodeGen/math-builtins-long.c b/clang/test/CodeGen/math-builtins-long.c index f3c328dcbfcd7..ad0d2122b597f 100644 --- a/clang/test/CodeGen/math-builtins-long.c +++ b/clang/test/CodeGen/math-builtins-long.c @@ -35,7 +35,7 @@ void foo(long double f, long double *l, int *i, const char *c) { __builtin_fabsl(f); // F80: call { x86_fp80, i32 } @llvm.frexp.f80.i32(x86_fp80 %{{.+}}) - // PPC: call { ppc_fp128, i32 } @llvm.frexp.ppcf128.i32(ppc_fp128 %{{.+}}) + // PPC: call ppc_fp128 @frexpl(ppc_fp128 noundef %{{.+}}, ptr noundef %{{.+}}) // X86F128: call { fp128, i32 } @llvm.frexp.f128.i32(fp128 %{{.+}}) // PPCF128: call { fp128, i32 } @llvm.frexp.f128.i32(fp128 %{{.+}}) __builtin_frexpl(f,i); diff --git a/libcxx/test/libcxx/numerics/c.math/constexpr-cxx23-clang.pass.cpp b/libcxx/test/libcxx/numerics/c.math/constexpr-cxx23-clang.pass.cpp index 31511064ce7ca..a07260a34516f 100644 --- a/libcxx/test/libcxx/numerics/c.math/constexpr-cxx23-clang.pass.cpp +++ b/libcxx/test/libcxx/numerics/c.math/constexpr-cxx23-clang.pass.cpp @@ -58,15 +58,9 @@ int main(int, char**) { ASSERT_NOT_CONSTEXPR_CXX23(std::frexp(0.0f, &DummyInt) == 0.0f); ASSERT_NOT_CONSTEXPR_CXX23(std::frexp(0.0, &DummyInt) == 0.0); -//FIXME: currently linux powerpc does not support this expansion -// since 0.0L lowers to ppcf128 and special handling is required. -#if !defined(__LONG_DOUBLE_IBM128__) ASSERT_NOT_CONSTEXPR_CXX23(std::frexp(0.0L, &DummyInt) == 0.0L); -#endif ASSERT_NOT_CONSTEXPR_CXX23(std::frexpf(0.0f, &DummyInt) == 0.0f); -#if !defined(__LONG_DOUBLE_IBM128__) ASSERT_NOT_CONSTEXPR_CXX23(std::frexpl(0.0L, &DummyInt) == 0.0L); -#endif ASSERT_NOT_CONSTEXPR_CXX23(std::ilogb(1.0f) == 0); ASSERT_NOT_CONSTEXPR_CXX23(std::ilogb(1.0) == 0); From 87bd71efd0af21b6663a7729317952535446f36d Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Fri, 15 Dec 2023 14:25:30 -0800 Subject: [PATCH 023/884] Revert "[NFC] Fix the warning Wcovered-switch-default" This reverts commit e55bda06dc2bb1ef11ff4fcc43f90d8bf843f967. --- clang/include/clang/Serialization/ASTWriter.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index de69f99003d82..16ab9583f8ed8 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -763,8 +763,9 @@ class ASTWriter : public ASTDeserializationListener, return DeclDependentNonTemplateCXXMethodAbbrev; case FunctionDecl::TK_DependentFunctionTemplateSpecialization: return DeclDependentSpecializationCXXMethodAbbrev; + default: + llvm_unreachable("Unknwon Template Kind!"); } - llvm_unreachable("Unknwon Template Kind!"); } unsigned getDeclTemplateTypeParmAbbrev() const { return DeclTemplateTypeParmAbbrev; From eccc1cca71bb704e4dcaabccc993d08fd15b46a2 Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Fri, 15 Dec 2023 14:30:24 -0800 Subject: [PATCH 024/884] Revert "[NFC] [Serialization] Packing more bits and refactor AbbrevToUse" This reverts commit 9cdb825a4f1bf9e75829d03879620c6144d0b7bc. --- clang/include/clang/Serialization/ASTReader.h | 2 - clang/include/clang/Serialization/ASTWriter.h | 89 ++-- clang/lib/Serialization/ASTReaderDecl.cpp | 2 +- clang/lib/Serialization/ASTReaderStmt.cpp | 415 +++++++----------- clang/lib/Serialization/ASTWriter.cpp | 15 +- clang/lib/Serialization/ASTWriterDecl.cpp | 381 ++++------------ clang/lib/Serialization/ASTWriterStmt.cpp | 248 ++++------- 7 files changed, 364 insertions(+), 788 deletions(-) diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index a6dd779386dc1..9bb89ec941091 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -2422,8 +2422,6 @@ class BitsUnpacker { CurrentBitsIndex = 0; } - void advance(uint32_t BitsWidth) { CurrentBitsIndex += BitsWidth; } - bool getNextBit() { assert(isValid()); return Value & (1 << CurrentBitsIndex++); diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 16ab9583f8ed8..a56929ef0245e 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -564,25 +564,11 @@ class ASTWriter : public ASTDeserializationListener, unsigned DeclEnumAbbrev = 0; unsigned DeclObjCIvarAbbrev = 0; unsigned DeclCXXMethodAbbrev = 0; - unsigned DeclDependentNonTemplateCXXMethodAbbrev = 0; - unsigned DeclTemplateCXXMethodAbbrev = 0; - unsigned DeclMemberSpecializedCXXMethodAbbrev = 0; - unsigned DeclTemplateSpecializedCXXMethodAbbrev = 0; - unsigned DeclDependentSpecializationCXXMethodAbbrev = 0; - unsigned DeclTemplateTypeParmAbbrev = 0; - unsigned DeclUsingShadowAbbrev = 0; unsigned DeclRefExprAbbrev = 0; unsigned CharacterLiteralAbbrev = 0; unsigned IntegerLiteralAbbrev = 0; unsigned ExprImplicitCastAbbrev = 0; - unsigned BinaryOperatorAbbrev = 0; - unsigned CompoundAssignOperatorAbbrev = 0; - unsigned CallExprAbbrev = 0; - unsigned CXXOperatorCallExprAbbrev = 0; - unsigned CXXMemberCallExprAbbrev = 0; - - unsigned CompoundStmtAbbrev = 0; void WriteDeclAbbrevs(); void WriteDecl(ASTContext &Context, Decl *D); @@ -749,42 +735,12 @@ class ASTWriter : public ASTDeserializationListener, unsigned getDeclFieldAbbrev() const { return DeclFieldAbbrev; } unsigned getDeclEnumAbbrev() const { return DeclEnumAbbrev; } unsigned getDeclObjCIvarAbbrev() const { return DeclObjCIvarAbbrev; } - unsigned getDeclCXXMethodAbbrev(FunctionDecl::TemplatedKind Kind) const { - switch (Kind) { - case FunctionDecl::TK_NonTemplate: - return DeclCXXMethodAbbrev; - case FunctionDecl::TK_FunctionTemplate: - return DeclTemplateCXXMethodAbbrev; - case FunctionDecl::TK_MemberSpecialization: - return DeclMemberSpecializedCXXMethodAbbrev; - case FunctionDecl::TK_FunctionTemplateSpecialization: - return DeclTemplateSpecializedCXXMethodAbbrev; - case FunctionDecl::TK_DependentNonTemplate: - return DeclDependentNonTemplateCXXMethodAbbrev; - case FunctionDecl::TK_DependentFunctionTemplateSpecialization: - return DeclDependentSpecializationCXXMethodAbbrev; - default: - llvm_unreachable("Unknwon Template Kind!"); - } - } - unsigned getDeclTemplateTypeParmAbbrev() const { - return DeclTemplateTypeParmAbbrev; - } - unsigned getDeclUsingShadowAbbrev() const { return DeclUsingShadowAbbrev; } + unsigned getDeclCXXMethodAbbrev() const { return DeclCXXMethodAbbrev; } unsigned getDeclRefExprAbbrev() const { return DeclRefExprAbbrev; } unsigned getCharacterLiteralAbbrev() const { return CharacterLiteralAbbrev; } unsigned getIntegerLiteralAbbrev() const { return IntegerLiteralAbbrev; } unsigned getExprImplicitCastAbbrev() const { return ExprImplicitCastAbbrev; } - unsigned getBinaryOperatorAbbrev() const { return BinaryOperatorAbbrev; } - unsigned getCompoundAssignOperatorAbbrev() const { - return CompoundAssignOperatorAbbrev; - } - unsigned getCallExprAbbrev() const { return CallExprAbbrev; } - unsigned getCXXOperatorCallExprAbbrev() { return CXXOperatorCallExprAbbrev; } - unsigned getCXXMemberCallExprAbbrev() { return CXXMemberCallExprAbbrev; } - - unsigned getCompoundStmtAbbrev() const { return CompoundStmtAbbrev; } bool hasChain() const { return Chain; } ASTReader *getChain() const { return Chain; } @@ -885,33 +841,46 @@ class BitsPacker { BitsPacker(BitsPacker &&) = delete; BitsPacker operator=(const BitsPacker &) = delete; BitsPacker operator=(BitsPacker &&) = delete; - ~BitsPacker() = default; - - bool canWriteNextNBits(uint32_t BitsWidth) const { - return CurrentBitIndex + BitsWidth < BitIndexUpbound; - } - - void reset(uint32_t Value) { - UnderlyingValue = Value; - CurrentBitIndex = 0; + ~BitsPacker() { + assert(!hasUnconsumedValues() && "There are unprocessed bits!"); } void addBit(bool Value) { addBits(Value, 1); } void addBits(uint32_t Value, uint32_t BitsWidth) { assert(BitsWidth < BitIndexUpbound); assert((Value < (1u << BitsWidth)) && "Passing narrower bit width!"); - assert(canWriteNextNBits(BitsWidth) && - "Inserting too much bits into a value!"); - UnderlyingValue |= Value << CurrentBitIndex; + if (CurrentBitIndex + BitsWidth >= BitIndexUpbound) { + Values.push_back(0); + CurrentBitIndex = 0; + } + + assert(CurrentBitIndex < BitIndexUpbound); + Values.back() |= Value << CurrentBitIndex; CurrentBitIndex += BitsWidth; } - operator uint32_t() { return UnderlyingValue; } + bool hasUnconsumedValues() const { + return ConsumingValueIndex < Values.size(); + } + uint32_t getNextValue() { + assert(hasUnconsumedValues()); + return Values[ConsumingValueIndex++]; + } + + // We can convert the packer to an uint32_t if there is only one values. + operator uint32_t() { + assert(Values.size() == 1); + return getNextValue(); + } private: - uint32_t UnderlyingValue = 0; - uint32_t CurrentBitIndex = 0; + SmallVector Values; + uint16_t ConsumingValueIndex = 0; + // Initialize CurrentBitIndex with an invalid value + // to make it easier to update Values. See the implementation + // of `addBits` to see the details. + uint16_t CurrentBitIndex = BitIndexUpbound; }; } // namespace clang diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 209fb04342088..7140a14aefbf9 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2660,7 +2660,7 @@ void ASTDeclReader::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { D->setDeclaredWithTypename(Record.readInt()); - if (D->hasTypeConstraint()) { + if (Record.readBool()) { ConceptReference *CR = nullptr; if (Record.readBool()) CR = Record.readConceptReference(); diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index a43b1b9c0216b..b3a6f619372b4 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -73,8 +73,6 @@ namespace clang { ASTRecordReader &Record; llvm::BitstreamCursor &DeclsCursor; - std::optional CurrentUnpackingBits; - SourceLocation readSourceLocation() { return Record.readSourceLocation(); } @@ -112,9 +110,6 @@ namespace clang { /// itself. static const unsigned NumExprFields = NumStmtFields + 2; - /// The number of bits required for the packing bits for the Expr class. - static const unsigned NumExprBits = 10; - /// Read and initialize a ExplicitTemplateArgumentList structure. void ReadTemplateKWAndArgsInfo(ASTTemplateKWAndArgsInfo &Args, TemplateArgumentLoc *ArgsLocArray, @@ -152,10 +147,9 @@ void ASTStmtReader::VisitNullStmt(NullStmt *S) { void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) { VisitStmt(S); - CurrentUnpackingBits.emplace(Record.readInt()); SmallVector Stmts; - unsigned NumStmts = CurrentUnpackingBits->getNextBits(/*Width=*/20); - unsigned HasFPFeatures = CurrentUnpackingBits->getNextBit(); + unsigned NumStmts = Record.readInt(); + unsigned HasFPFeatures = Record.readInt(); assert(S->hasStoredFPFeatures() == HasFPFeatures); while (NumStmts--) Stmts.push_back(Record.readSubStmt()); @@ -220,11 +214,9 @@ void ASTStmtReader::VisitAttributedStmt(AttributedStmt *S) { void ASTStmtReader::VisitIfStmt(IfStmt *S) { VisitStmt(S); - CurrentUnpackingBits.emplace(Record.readInt()); - - bool HasElse = CurrentUnpackingBits->getNextBit(); - bool HasVar = CurrentUnpackingBits->getNextBit(); - bool HasInit = CurrentUnpackingBits->getNextBit(); + bool HasElse = Record.readInt(); + bool HasVar = Record.readInt(); + bool HasInit = Record.readInt(); S->setStatementKind(static_cast(Record.readInt())); S->setCond(Record.readSubExpr()); @@ -531,15 +523,14 @@ void ASTStmtReader::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtReader::VisitExpr(Expr *E) { VisitStmt(E); - CurrentUnpackingBits.emplace(Record.readInt()); - E->setDependence(static_cast( - CurrentUnpackingBits->getNextBits(/*Width=*/5))); - E->setValueKind(static_cast( - CurrentUnpackingBits->getNextBits(/*Width=*/2))); - E->setObjectKind(static_cast( - CurrentUnpackingBits->getNextBits(/*Width=*/3))); - E->setType(Record.readType()); + BitsUnpacker ExprBits(Record.readInt()); + E->setDependence( + static_cast(ExprBits.getNextBits(/*Width=*/5))); + E->setValueKind( + static_cast(ExprBits.getNextBits(/*Width=*/2))); + E->setObjectKind( + static_cast(ExprBits.getNextBits(/*Width=*/3))); assert(Record.getIdx() == NumExprFields && "Incorrect expression field count"); } @@ -600,20 +591,17 @@ void ASTStmtReader::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - E->DeclRefExprBits.HasQualifier = CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.HasFoundDecl = CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.HasTemplateKWAndArgsInfo = - CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.HadMultipleCandidates = CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.RefersToEnclosingVariableOrCapture = - CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.NonOdrUseReason = - CurrentUnpackingBits->getNextBits(/*Width=*/2); - E->DeclRefExprBits.IsImmediateEscalating = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasQualifier = Record.readInt(); + E->DeclRefExprBits.HasFoundDecl = Record.readInt(); + E->DeclRefExprBits.HasTemplateKWAndArgsInfo = Record.readInt(); + E->DeclRefExprBits.HadMultipleCandidates = Record.readInt(); + E->DeclRefExprBits.RefersToEnclosingVariableOrCapture = Record.readInt(); + E->DeclRefExprBits.NonOdrUseReason = Record.readInt(); + E->DeclRefExprBits.IsImmediateEscalating = Record.readInt(); E->DeclRefExprBits.CapturedByCopyInLambdaWithExplicitObjectParameter = false; unsigned NumTemplateArgs = 0; if (E->hasTemplateKWAndArgsInfo()) - NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + NumTemplateArgs = Record.readInt(); if (E->hasQualifier()) new (E->getTrailingObjects()) @@ -718,13 +706,12 @@ void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) { void ASTStmtReader::VisitUnaryOperator(UnaryOperator *E) { VisitExpr(E); - bool hasFP_Features = CurrentUnpackingBits->getNextBit(); + bool hasFP_Features = Record.readInt(); assert(hasFP_Features == E->hasStoredFPFeatures()); E->setSubExpr(Record.readSubExpr()); - E->setOpcode( - (UnaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/5)); + E->setOpcode((UnaryOperator::Opcode)Record.readInt()); E->setOperatorLoc(readSourceLocation()); - E->setCanOverflow(CurrentUnpackingBits->getNextBit()); + E->setCanOverflow(Record.readInt()); if (hasFP_Features) E->setStoredFPFeatures( FPOptionsOverride::getFromOpaqueInt(Record.readInt())); @@ -1013,10 +1000,12 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtReader::VisitCallExpr(CallExpr *E) { VisitExpr(E); - unsigned NumArgs = CurrentUnpackingBits->getNextBits(/*Width=*/13); - bool HasFPFeatures = CurrentUnpackingBits->getNextBit(); + BitsUnpacker CallExprBits = Record.readInt(); + + unsigned NumArgs = CallExprBits.getNextBits(/*Width=*/16); + bool HasFPFeatures = CallExprBits.getNextBit(); E->setADLCallKind( - static_cast(CurrentUnpackingBits->getNextBit())); + static_cast(CallExprBits.getNextBit())); assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!"); E->setRParenLoc(readSourceLocation()); E->setCallee(Record.readSubExpr()); @@ -1035,28 +1024,27 @@ void ASTStmtReader::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { void ASTStmtReader::VisitMemberExpr(MemberExpr *E) { VisitExpr(E); - bool HasQualifier = CurrentUnpackingBits->getNextBit(); - bool HasFoundDecl = CurrentUnpackingBits->getNextBit(); - bool HasTemplateInfo = CurrentUnpackingBits->getNextBit(); - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + bool HasQualifier = Record.readInt(); + bool HasFoundDecl = Record.readInt(); + bool HasTemplateInfo = Record.readInt(); + unsigned NumTemplateArgs = Record.readInt(); E->Base = Record.readSubExpr(); E->MemberDecl = Record.readDeclAs(); E->MemberDNLoc = Record.readDeclarationNameLoc(E->MemberDecl->getDeclName()); E->MemberLoc = Record.readSourceLocation(); - E->MemberExprBits.IsArrow = CurrentUnpackingBits->getNextBit(); + E->MemberExprBits.IsArrow = Record.readInt(); E->MemberExprBits.HasQualifierOrFoundDecl = HasQualifier || HasFoundDecl; E->MemberExprBits.HasTemplateKWAndArgsInfo = HasTemplateInfo; - E->MemberExprBits.HadMultipleCandidates = CurrentUnpackingBits->getNextBit(); - E->MemberExprBits.NonOdrUseReason = - CurrentUnpackingBits->getNextBits(/*Width=*/2); + E->MemberExprBits.HadMultipleCandidates = Record.readInt(); + E->MemberExprBits.NonOdrUseReason = Record.readInt(); E->MemberExprBits.OperatorLoc = Record.readSourceLocation(); if (HasQualifier || HasFoundDecl) { DeclAccessPair FoundDecl; if (HasFoundDecl) { auto *FoundD = Record.readDeclAs(); - auto AS = (AccessSpecifier)CurrentUnpackingBits->getNextBits(/*Width=*/2); + auto AS = (AccessSpecifier)Record.readInt(); FoundDecl = DeclAccessPair::make(FoundD, AS); } else { FoundDecl = DeclAccessPair::make(E->MemberDecl, @@ -1103,10 +1091,10 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { VisitExpr(E); unsigned NumBaseSpecs = Record.readInt(); assert(NumBaseSpecs == E->path_size()); - unsigned HasFPFeatures = CurrentUnpackingBits->getNextBit(); + unsigned HasFPFeatures = Record.readInt(); assert(E->hasStoredFPFeatures() == HasFPFeatures); E->setSubExpr(Record.readSubExpr()); - E->setCastKind((CastKind)CurrentUnpackingBits->getNextBits(/*Width=*/7)); + E->setCastKind((CastKind)Record.readInt()); CastExpr::path_iterator BaseI = E->path_begin(); while (NumBaseSpecs--) { auto *BaseSpec = new (Record.getContext()) CXXBaseSpecifier; @@ -1119,12 +1107,10 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { } void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) { - + bool hasFP_Features; VisitExpr(E); - bool hasFP_Features = CurrentUnpackingBits->getNextBit(); - E->setHasStoredFPFeatures(hasFP_Features); - E->setOpcode( - (BinaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/6)); + E->setHasStoredFPFeatures(hasFP_Features = Record.readInt()); + E->setOpcode((BinaryOperator::Opcode)Record.readInt()); E->setLHS(Record.readSubExpr()); E->setRHS(Record.readSubExpr()); E->setOperatorLoc(readSourceLocation()); @@ -1162,7 +1148,7 @@ ASTStmtReader::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) { void ASTStmtReader::VisitImplicitCastExpr(ImplicitCastExpr *E) { VisitCastExpr(E); - E->setIsPartOfExplicitCast(CurrentUnpackingBits->getNextBit()); + E->setIsPartOfExplicitCast(Record.readInt()); } void ASTStmtReader::VisitExplicitCastExpr(ExplicitCastExpr *E) { @@ -1700,8 +1686,7 @@ void ASTStmtReader::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) { void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { VisitCallExpr(E); - E->CXXOperatorCallExprBits.OperatorKind = - CurrentUnpackingBits->getNextBits(/*Width=*/6); + E->CXXOperatorCallExprBits.OperatorKind = Record.readInt(); E->Range = Record.readSourceRange(); } @@ -1779,8 +1764,8 @@ void ASTStmtReader::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) { SourceRange R = readSourceRange(); E->Loc = R.getBegin(); E->RParenLoc = R.getEnd(); - if (CurrentUnpackingBits->getNextBit()) - E->AngleBrackets = readSourceRange(); + R = readSourceRange(); + E->AngleBrackets = R; } void ASTStmtReader::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) { @@ -1976,9 +1961,9 @@ void ASTStmtReader::VisitCXXDependentScopeMemberExpr( CXXDependentScopeMemberExpr *E) { VisitExpr(E); - bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/16); - bool HasFirstQualifierFoundInScope = CurrentUnpackingBits->getNextBit(); + bool HasTemplateKWAndArgsInfo = Record.readInt(); + unsigned NumTemplateArgs = Record.readInt(); + bool HasFirstQualifierFoundInScope = Record.readInt(); assert((HasTemplateKWAndArgsInfo == E->hasTemplateKWAndArgsInfo()) && "Wrong HasTemplateKWAndArgsInfo!"); @@ -1994,17 +1979,11 @@ void ASTStmtReader::VisitCXXDependentScopeMemberExpr( assert((NumTemplateArgs == E->getNumTemplateArgs()) && "Wrong NumTemplateArgs!"); - E->CXXDependentScopeMemberExprBits.IsArrow = - CurrentUnpackingBits->getNextBit(); - + E->CXXDependentScopeMemberExprBits.IsArrow = Record.readInt(); + E->CXXDependentScopeMemberExprBits.OperatorLoc = readSourceLocation(); E->BaseType = Record.readType(); E->QualifierLoc = Record.readNestedNameSpecifierLoc(); - // not ImplicitAccess - if (CurrentUnpackingBits->getNextBit()) { - E->Base = Record.readSubExpr(); - E->CXXDependentScopeMemberExprBits.OperatorLoc = readSourceLocation(); - } else - E->Base = nullptr; + E->Base = Record.readSubExpr(); if (HasFirstQualifierFoundInScope) *E->getTrailingObjects() = readDeclAs(); @@ -2016,11 +1995,11 @@ void ASTStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { VisitExpr(E); - if (CurrentUnpackingBits->getNextBit()) // HasTemplateKWAndArgsInfo + if (Record.readInt()) // HasTemplateKWAndArgsInfo ReadTemplateKWAndArgsInfo( *E->getTrailingObjects(), E->getTrailingObjects(), - /*NumTemplateArgs=*/CurrentUnpackingBits->getNextBits(/*Width=*/16)); + /*NumTemplateArgs=*/Record.readInt()); E->QualifierLoc = Record.readNestedNameSpecifierLoc(); E->NameInfo = Record.readDeclarationNameInfo(); @@ -2043,15 +2022,15 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - CurrentUnpackingBits.emplace(Record.readInt()); - unsigned NumResults = CurrentUnpackingBits->getNextBits(/*Width=*/12); - bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); + BitsUnpacker OverloadExprBits = Record.readInt(); + unsigned NumResults = OverloadExprBits.getNextBits(/*Width=*/14); + bool HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); assert((E->getNumDecls() == NumResults) && "Wrong NumResults!"); assert((E->hasTemplateKWAndArgsInfo() == HasTemplateKWAndArgsInfo) && "Wrong HasTemplateKWAndArgsInfo!"); if (HasTemplateKWAndArgsInfo) { - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + unsigned NumTemplateArgs = OverloadExprBits.getNextBits(/*Width=*/14); ReadTemplateKWAndArgsInfo(*E->getTrailingASTTemplateKWAndArgsInfo(), E->getTrailingTemplateArgumentLoc(), NumTemplateArgs); @@ -2078,25 +2057,17 @@ void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) { VisitOverloadExpr(E); - E->UnresolvedMemberExprBits.IsArrow = CurrentUnpackingBits->getNextBit(); - E->UnresolvedMemberExprBits.HasUnresolvedUsing = - CurrentUnpackingBits->getNextBit(); - - if (/*!isImplicitAccess=*/CurrentUnpackingBits->getNextBit()) { - E->Base = Record.readSubExpr(); - E->OperatorLoc = readSourceLocation(); - } else { - E->Base = nullptr; - E->OperatorLoc = SourceLocation(); - } - + E->UnresolvedMemberExprBits.IsArrow = Record.readInt(); + E->UnresolvedMemberExprBits.HasUnresolvedUsing = Record.readInt(); + E->Base = Record.readSubExpr(); E->BaseType = Record.readType(); + E->OperatorLoc = readSourceLocation(); } void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) { VisitOverloadExpr(E); - E->UnresolvedLookupExprBits.RequiresADL = CurrentUnpackingBits->getNextBit(); - E->UnresolvedLookupExprBits.Overloaded = CurrentUnpackingBits->getNextBit(); + E->UnresolvedLookupExprBits.RequiresADL = Record.readInt(); + E->UnresolvedLookupExprBits.Overloaded = Record.readInt(); E->NamingClass = readDeclAs(); } @@ -2171,12 +2142,9 @@ void ASTStmtReader::VisitSubstNonTypeTemplateParmExpr( SubstNonTypeTemplateParmExpr *E) { VisitExpr(E); E->AssociatedDeclAndRef.setPointer(readDeclAs()); - E->AssociatedDeclAndRef.setInt(CurrentUnpackingBits->getNextBit()); - E->Index = CurrentUnpackingBits->getNextBits(/*Width=*/12); - if (CurrentUnpackingBits->getNextBit()) - E->PackIndex = Record.readInt(); - else - E->PackIndex = 0; + E->AssociatedDeclAndRef.setInt(Record.readInt()); + E->Index = Record.readInt(); + E->PackIndex = Record.readInt(); E->SubstNonTypeTemplateParmExprBits.NameLoc = readSourceLocation(); E->Replacement = Record.readSubExpr(); } @@ -2868,13 +2836,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) NullStmt(Empty); break; - case STMT_COMPOUND: { - BitsUnpacker StmtCompoundBits(Record[ASTStmtReader::NumStmtFields]); - unsigned NumStmts = StmtCompoundBits.getNextBits(/*Width=*/20); - bool HasFPFeatures = StmtCompoundBits.getNextBit(); - S = CompoundStmt::CreateEmpty(Context, NumStmts, HasFPFeatures); + case STMT_COMPOUND: + S = CompoundStmt::CreateEmpty( + Context, /*NumStmts=*/Record[ASTStmtReader::NumStmtFields], + /*HasFPFeatures=*/Record[ASTStmtReader::NumStmtFields + 1]); break; - } case STMT_CASE: S = CaseStmt::CreateEmpty( @@ -2896,14 +2862,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /*NumAttrs*/Record[ASTStmtReader::NumStmtFields]); break; - case STMT_IF: { - BitsUnpacker IfStmtBits(Record[ASTStmtReader::NumStmtFields]); - bool HasElse = IfStmtBits.getNextBit(); - bool HasVar = IfStmtBits.getNextBit(); - bool HasInit = IfStmtBits.getNextBit(); - S = IfStmt::CreateEmpty(Context, HasElse, HasVar, HasInit); + case STMT_IF: + S = IfStmt::CreateEmpty( + Context, + /* HasElse=*/Record[ASTStmtReader::NumStmtFields], + /* HasVar=*/Record[ASTStmtReader::NumStmtFields + 1], + /* HasInit=*/Record[ASTStmtReader::NumStmtFields + 2]); break; - } case STMT_SWITCH: S = SwitchStmt::CreateEmpty( @@ -2980,20 +2945,17 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /*HasFunctionName*/ Record[ASTStmtReader::NumExprFields]); break; - case EXPR_DECL_REF: { - BitsUnpacker DeclRefExprBits(Record[ASTStmtReader::NumStmtFields]); - DeclRefExprBits.advance(ASTStmtReader::NumExprBits); - bool HasQualifier = DeclRefExprBits.getNextBit(); - bool HasFoundDecl = DeclRefExprBits.getNextBit(); - bool HasTemplateKWAndArgsInfo = DeclRefExprBits.getNextBit(); - DeclRefExprBits.advance(5); - unsigned NumTemplateArgs = HasTemplateKWAndArgsInfo - ? DeclRefExprBits.getNextBits(/*Width=*/12) - : 0; - S = DeclRefExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, - HasTemplateKWAndArgsInfo, NumTemplateArgs); + case EXPR_DECL_REF: + S = DeclRefExpr::CreateEmpty( + Context, + /*HasQualifier=*/Record[ASTStmtReader::NumExprFields], + /*HasFoundDecl=*/Record[ASTStmtReader::NumExprFields + 1], + /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 2], + /*NumTemplateArgs=*/ + Record[ASTStmtReader::NumExprFields + 2] + ? Record[ASTStmtReader::NumExprFields + 7] + : 0); break; - } case EXPR_INTEGER_LITERAL: S = IntegerLiteral::Create(Context, Empty); @@ -3033,13 +2995,10 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /* NumExprs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_UNARY_OPERATOR: { - BitsUnpacker UnaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); - UnaryOperatorBits.advance(ASTStmtReader::NumExprBits); - bool HasFPFeatures = UnaryOperatorBits.getNextBit(); - S = UnaryOperator::CreateEmpty(Context, HasFPFeatures); + case EXPR_UNARY_OPERATOR: + S = UnaryOperator::CreateEmpty(Context, + Record[ASTStmtReader::NumExprFields]); break; - } case EXPR_OFFSETOF: S = OffsetOfExpr::CreateEmpty(Context, @@ -3074,9 +3033,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); auto HasFPFeatures = CallExprBits.getNextBit(); S = CallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); break; @@ -3087,33 +3045,22 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_MEMBER: { - BitsUnpacker ExprMemberBits(Record[ASTStmtReader::NumStmtFields]); - ExprMemberBits.advance(ASTStmtReader::NumExprBits); - bool HasQualifier = ExprMemberBits.getNextBit(); - bool HasFoundDecl = ExprMemberBits.getNextBit(); - bool HasTemplateInfo = ExprMemberBits.getNextBit(); - unsigned NumTemplateArgs = ExprMemberBits.getNextBits(/*Width=*/12); - S = MemberExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, - HasTemplateInfo, NumTemplateArgs); + case EXPR_MEMBER: + S = MemberExpr::CreateEmpty(Context, Record[ASTStmtReader::NumExprFields], + Record[ASTStmtReader::NumExprFields + 1], + Record[ASTStmtReader::NumExprFields + 2], + Record[ASTStmtReader::NumExprFields + 3]); break; - } - case EXPR_BINARY_OPERATOR: { - BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); - BinaryOperatorBits.advance(ASTStmtReader::NumExprBits); - bool HasFPFeatures = BinaryOperatorBits.getNextBit(); - S = BinaryOperator::CreateEmpty(Context, HasFPFeatures); + case EXPR_BINARY_OPERATOR: + S = BinaryOperator::CreateEmpty(Context, + Record[ASTStmtReader::NumExprFields]); break; - } - case EXPR_COMPOUND_ASSIGN_OPERATOR: { - BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); - BinaryOperatorBits.advance(ASTStmtReader::NumExprBits); - bool HasFPFeatures = BinaryOperatorBits.getNextBit(); - S = CompoundAssignOperator::CreateEmpty(Context, HasFPFeatures); + case EXPR_COMPOUND_ASSIGN_OPERATOR: + S = CompoundAssignOperator::CreateEmpty( + Context, Record[ASTStmtReader::NumExprFields]); break; - } case EXPR_CONDITIONAL_OPERATOR: S = new (Context) ConditionalOperator(Empty); @@ -3123,23 +3070,19 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) BinaryConditionalOperator(Empty); break; - case EXPR_IMPLICIT_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - bool HasFPFeatures = CastExprBits.getNextBit(); - S = ImplicitCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); + case EXPR_IMPLICIT_CAST: + S = ImplicitCastExpr::CreateEmpty( + Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields], + /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); break; - } - case EXPR_CSTYLE_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - bool HasFPFeatures = CastExprBits.getNextBit(); - S = CStyleCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); + case EXPR_CSTYLE_CAST: + S = CStyleCastExpr::CreateEmpty( + Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields], + /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); break; - } case EXPR_COMPOUND_LITERAL: S = new (Context) CompoundLiteralExpr(Empty); @@ -3834,9 +3777,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_OPERATOR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXOperatorCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3844,9 +3786,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_MEMBER_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXMemberCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3873,26 +3814,22 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { /* NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_STATIC_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - bool HasFPFeatures = CastExprBits.getNextBit(); - S = CXXStaticCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); + case EXPR_CXX_STATIC_CAST: + S = CXXStaticCastExpr::CreateEmpty( + Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields], + /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); break; - } - case EXPR_CXX_DYNAMIC_CAST: { - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - S = CXXDynamicCastExpr::CreateEmpty(Context, PathSize); + case EXPR_CXX_DYNAMIC_CAST: + S = CXXDynamicCastExpr::CreateEmpty(Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields]); break; - } - case EXPR_CXX_REINTERPRET_CAST: { - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - S = CXXReinterpretCastExpr::CreateEmpty(Context, PathSize); + case EXPR_CXX_REINTERPRET_CAST: + S = CXXReinterpretCastExpr::CreateEmpty(Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields]); break; - } case EXPR_CXX_CONST_CAST: S = CXXConstCastExpr::CreateEmpty(Context); @@ -3902,28 +3839,21 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = CXXAddrspaceCastExpr::CreateEmpty(Context); break; - case EXPR_CXX_FUNCTIONAL_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - bool HasFPFeatures = CastExprBits.getNextBit(); - S = CXXFunctionalCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); + case EXPR_CXX_FUNCTIONAL_CAST: + S = CXXFunctionalCastExpr::CreateEmpty( + Context, + /*PathSize*/ Record[ASTStmtReader::NumExprFields], + /*HasFPFeatures*/ Record[ASTStmtReader::NumExprFields + 1]); break; - } - case EXPR_BUILTIN_BIT_CAST: { -#ifndef NDEBUG - unsigned PathSize = Record[ASTStmtReader::NumExprFields]; - assert(PathSize == 0 && "Wrong PathSize!"); -#endif + case EXPR_BUILTIN_BIT_CAST: + assert(Record[ASTStmtReader::NumExprFields] == 0 && "Wrong PathSize!"); S = new (Context) BuiltinBitCastExpr(Empty); break; - } case EXPR_USER_DEFINED_LITERAL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); auto HasFPFeatures = CallExprBits.getNextBit(); S = UserDefinedLiteral::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -4014,63 +3944,47 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_DEPENDENT_SCOPE_MEMBER: { - BitsUnpacker DependentScopeMemberBits( - Record[ASTStmtReader::NumStmtFields]); - DependentScopeMemberBits.advance(ASTStmtReader::NumExprBits); - bool HasTemplateKWAndArgsInfo = DependentScopeMemberBits.getNextBit(); - unsigned NumTemplateArgs = - DependentScopeMemberBits.getNextBits(/*Width=*/16); - bool HasFirstQualifierFoundInScope = - DependentScopeMemberBits.getNextBit(); + case EXPR_CXX_DEPENDENT_SCOPE_MEMBER: S = CXXDependentScopeMemberExpr::CreateEmpty( - Context, HasTemplateKWAndArgsInfo, NumTemplateArgs, - HasFirstQualifierFoundInScope); + Context, + /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields], + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1], + /*HasFirstQualifierFoundInScope=*/ + Record[ASTStmtReader::NumExprFields + 2]); break; - } - case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF: { - BitsUnpacker DependentScopeDeclRefBits( - Record[ASTStmtReader::NumStmtFields]); - DependentScopeDeclRefBits.advance(ASTStmtReader::NumExprBits); - bool HasTemplateKWAndArgsInfo = DependentScopeDeclRefBits.getNextBit(); - unsigned NumTemplateArgs = - HasTemplateKWAndArgsInfo - ? DependentScopeDeclRefBits.getNextBits(/*Width=*/16) - : 0; - S = DependentScopeDeclRefExpr::CreateEmpty( - Context, HasTemplateKWAndArgsInfo, NumTemplateArgs); + case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF: + S = DependentScopeDeclRefExpr::CreateEmpty(Context, + /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields], + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] + ? Record[ASTStmtReader::NumExprFields + 1] + : 0); break; - } case EXPR_CXX_UNRESOLVED_CONSTRUCT: S = CXXUnresolvedConstructExpr::CreateEmpty(Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CXX_UNRESOLVED_MEMBER: { - BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumResults = OverloadExprBits.getNextBits(/*Width=*/12); - auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); - auto NumTemplateArgs = HasTemplateKWAndArgsInfo - ? OverloadExprBits.getNextBits(/*Width=*/12) - : 0; + case EXPR_CXX_UNRESOLVED_MEMBER: S = UnresolvedMemberExpr::CreateEmpty( - Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); + Context, + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; - } - case EXPR_CXX_UNRESOLVED_LOOKUP: { - BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumResults = OverloadExprBits.getNextBits(/*Width=*/12); - auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); - auto NumTemplateArgs = HasTemplateKWAndArgsInfo - ? OverloadExprBits.getNextBits(/*Width=*/12) - : 0; + case EXPR_CXX_UNRESOLVED_LOOKUP: S = UnresolvedLookupExpr::CreateEmpty( - Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); + Context, + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; - } case EXPR_TYPE_TRAIT: S = TypeTraitExpr::CreateDeserialized(Context, @@ -4130,9 +4044,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CUDA_KERNEL_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); auto HasFPFeatures = CallExprBits.getNextBit(); S = CUDAKernelCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 78939bfd533ff..91eb2af8f8ad6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -6003,17 +6003,12 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { BitsPacker DefinitionBits; -#define FIELD(Name, Width, Merge) \ - if (!DefinitionBits.canWriteNextNBits(Width)) { \ - Record->push_back(DefinitionBits); \ - DefinitionBits.reset(0); \ - } \ - DefinitionBits.addBits(Data.Name, Width); - +#define FIELD(Name, Width, Merge) DefinitionBits.addBits(Data.Name, Width); #include "clang/AST/CXXRecordDeclDefinitionBits.def" #undef FIELD - Record->push_back(DefinitionBits); + while (DefinitionBits.hasUnconsumedValues()) + Record->push_back(DefinitionBits.getNextValue()); // getODRHash will compute the ODRHash if it has not been previously computed. Record->push_back(D->getODRHash()); @@ -6052,7 +6047,7 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { LambdaBits.addBits(Lambda.CaptureDefault, /*Width=*/2); LambdaBits.addBits(Lambda.NumCaptures, /*Width=*/15); LambdaBits.addBit(Lambda.HasKnownInternalLinkage); - Record->push_back(LambdaBits); + Record->push_back(LambdaBits.getNextValue()); Record->push_back(Lambda.NumExplicitCaptures); Record->push_back(Lambda.ManglingNumber); @@ -6063,12 +6058,10 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) { for (unsigned I = 0, N = Lambda.NumCaptures; I != N; ++I) { const LambdaCapture &Capture = Lambda.Captures.front()[I]; AddSourceLocation(Capture.getLocation()); - BitsPacker CaptureBits; CaptureBits.addBit(Capture.isImplicit()); CaptureBits.addBits(Capture.getCaptureKind(), /*Width=*/3); Record->push_back(CaptureBits); - switch (Capture.getCaptureKind()) { case LCK_StarThis: case LCK_This: diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp index cc32b3ffa2627..43169b2befc68 100644 --- a/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/clang/lib/Serialization/ASTWriterDecl.cpp @@ -496,10 +496,15 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) { if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && !D->isImplicit() && + !D->isUsed(false) && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && + !D->isInvalidDecl() && + !D->isReferenced() && !D->isTopLevelDeclInObjCContainer() && + D->getAccess() == AS_none && + !D->isModulePrivate() && !CXXRecordDecl::classofKind(D->getKind()) && !D->getIntegerTypeSourceInfo() && !D->getMemberSpecializationInfo() && @@ -539,10 +544,16 @@ void ASTDeclWriter::VisitRecordDecl(RecordDecl *D) { if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isImplicit() && + !D->isUsed(false) && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && + !D->isInvalidDecl() && + !D->isReferenced() && !D->isTopLevelDeclInObjCContainer() && + D->getAccess() == AS_none && + !D->isModulePrivate() && !CXXRecordDecl::classofKind(D->getKind()) && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier) @@ -1126,7 +1137,13 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isImplicit() && + !D->isUsed(false) && + !D->isInvalidDecl() && + !D->isReferenced() && !D->isTopLevelDeclInObjCContainer() && + D->getAccess() == AS_none && + !D->isModulePrivate() && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier && !D->hasExtInfo() && @@ -1176,9 +1193,14 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) { // we dynamically check for the properties that we optimize for, but don't // know are true of all PARM_VAR_DECLs. if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && - !D->hasExtInfo() && D->getStorageClass() == 0 && + !D->hasExtInfo() && !D->isImplicit() && !D->isUsed(false) && + !D->isInvalidDecl() && !D->isReferenced() && D->getAccess() == AS_none && + !D->isModulePrivate() && D->getStorageClass() == 0 && D->getInitStyle() == VarDecl::CInit && // Can params have anything else? - D->getInit() == nullptr) // No default expr. + D->getFunctionScopeDepth() == 0 && D->getObjCDeclQualifier() == 0 && + !D->isKNRPromoted() && !D->isExplicitObjectParameter() && + !D->hasInheritedDefaultArg() && D->getInit() == nullptr && + !D->hasUninstantiatedDefaultArg()) // No default expr. AbbrevToUse = Writer.getDeclParmVarAbbrev(); // Check things we know are true of *every* PARM_VAR_DECL, which is more than @@ -1381,13 +1403,6 @@ void ASTDeclWriter::VisitUsingShadowDecl(UsingShadowDecl *D) { Record.push_back(D->getIdentifierNamespace()); Record.AddDeclRef(D->UsingOrNextShadow); Record.AddDeclRef(Context.getInstantiatedFromUsingShadowDecl(D)); - - if (D->getDeclContext() == D->getLexicalDeclContext() && - D->getFirstDecl() == D->getMostRecentDecl() && !D->hasAttrs() && - !needsAnonymousDeclarationNumber(D) && - D->getDeclName().getNameKind() == DeclarationName::Identifier) - AbbrevToUse = Writer.getDeclUsingShadowAbbrev(); - Code = serialization::DECL_USING_SHADOW; } @@ -1492,32 +1507,10 @@ void ASTDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) { D->getFirstDecl() == D->getMostRecentDecl() && !D->isInvalidDecl() && !D->hasAttrs() && !D->isTopLevelDeclInObjCContainer() && D->getDeclName().getNameKind() == DeclarationName::Identifier && - !D->hasExtInfo() && !D->isExplicitlyDefaulted()) { - if (D->getTemplatedKind() == FunctionDecl::TK_NonTemplate || - D->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate || - D->getTemplatedKind() == FunctionDecl::TK_MemberSpecialization || - D->getTemplatedKind() == FunctionDecl::TK_DependentNonTemplate) - AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); - else if (D->getTemplatedKind() == - FunctionDecl::TK_FunctionTemplateSpecialization) { - FunctionTemplateSpecializationInfo *FTSInfo = - D->getTemplateSpecializationInfo(); - - if (FTSInfo->TemplateArguments->size() == 1) { - const TemplateArgument &TA = FTSInfo->TemplateArguments->get(0); - if (TA.getKind() == TemplateArgument::Type && - !FTSInfo->TemplateArgumentsAsWritten && - !FTSInfo->getMemberSpecializationInfo()) - AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); - } - } else if (D->getTemplatedKind() == - FunctionDecl::TK_DependentFunctionTemplateSpecialization) { - DependentFunctionTemplateSpecializationInfo *DFTSInfo = - D->getDependentSpecializationInfo(); - if (!DFTSInfo->TemplateArgumentsAsWritten) - AbbrevToUse = Writer.getDeclCXXMethodAbbrev(D->getTemplatedKind()); - } - } + !D->hasExtInfo() && !D->hasInheritedPrototype() && + D->hasWrittenPrototype() && + D->getTemplatedKind() == FunctionDecl::TK_NonTemplate) + AbbrevToUse = Writer.getDeclCXXMethodAbbrev(); Code = serialization::DECL_CXX_METHOD; } @@ -1789,7 +1782,7 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { Record.push_back(D->wasDeclaredWithTypename()); const TypeConstraint *TC = D->getTypeConstraint(); - assert((bool)TC == D->hasTypeConstraint()); + Record.push_back(TC != nullptr); if (TC) { auto *CR = TC->getConceptReference(); Record.push_back(CR != nullptr); @@ -1807,13 +1800,6 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { if (OwnsDefaultArg) Record.AddTypeSourceInfo(D->getDefaultArgumentInfo()); - if (!TC && !OwnsDefaultArg && - D->getDeclContext() == D->getLexicalDeclContext() && - !D->isInvalidDecl() && !D->hasAttrs() && - !D->isTopLevelDeclInObjCContainer() && - D->getDeclName().getNameKind() == DeclarationName::Identifier) - AbbrevToUse = Writer.getDeclTemplateTypeParmAbbrev(); - Code = serialization::DECL_TEMPLATE_TYPE_PARM; } @@ -2045,104 +2031,6 @@ void ASTDeclWriter::VisitOMPCapturedExprDecl(OMPCapturedExprDecl *D) { // ASTWriter Implementation //===----------------------------------------------------------------------===// -namespace { -template -std::shared_ptr -getFunctionDeclAbbrev(serialization::DeclCode Code) { - using namespace llvm; - - auto Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(Code)); - // RedeclarableDecl - Abv->Add(BitCodeAbbrevOp(0)); // CanonicalDecl - Abv->Add(BitCodeAbbrevOp(Kind)); - if constexpr (Kind == FunctionDecl::TK_NonTemplate) { - - } else if constexpr (Kind == FunctionDecl::TK_FunctionTemplate) { - // DescribedFunctionTemplate - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - } else if constexpr (Kind == FunctionDecl::TK_DependentNonTemplate) { - // Instantiated From Decl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - } else if constexpr (Kind == FunctionDecl::TK_MemberSpecialization) { - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InstantiatedFrom - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 3)); // TemplateSpecializationKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Specialized Location - } else if constexpr (Kind == - FunctionDecl::TK_FunctionTemplateSpecialization) { - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Template - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 3)); // TemplateSpecializationKind - Abv->Add(BitCodeAbbrevOp(1)); // Template Argument Size - Abv->Add(BitCodeAbbrevOp(TemplateArgument::Type)); // Template Argument Kind - Abv->Add( - BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Template Argument Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Is Defaulted - Abv->Add(BitCodeAbbrevOp(0)); // TemplateArgumentsAsWritten - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation - Abv->Add(BitCodeAbbrevOp(0)); - Abv->Add( - BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Canonical Decl of template - } else if constexpr (Kind == FunctionDecl:: - TK_DependentFunctionTemplateSpecialization) { - // Candidates of specialization - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abv->Add(BitCodeAbbrevOp(0)); // TemplateArgumentsAsWritten - } else { - llvm_unreachable("Unknown templated kind?"); - } - // Decl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID - // NamedDecl - Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Identifier - Abv->Add(BitCodeAbbrevOp(0)); // AnonDeclNumber - // ValueDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // DeclaratorDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InnerLocStart - Abv->Add(BitCodeAbbrevOp(0)); // HasExtInfo - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TSIType - // FunctionDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 11)); // IDNS - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 27)); // Packed Function Bits: StorageClass, Inline, InlineSpecified, - // VirtualAsWritten, Pure, HasInheritedProto, HasWrittenProto, - // Deleted, Trivial, TrivialForCall, Defaulted, ExplicitlyDefaulted, - // IsIneligibleOrNotSelected, ImplicitReturnZero, Constexpr, - // UsesSEHTry, SkippedBody, MultiVersion, LateParsed, - // FriendConstraintRefersToEnclosingTemplate, Linkage - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LocEnd - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // ODRHash - // This Array slurps the rest of the record. Fortunately we want to encode - // (nearly) all the remaining (variable number of) fields in the same way. - // - // This is: - // NumParams and Params[] from FunctionDecl, and - // NumOverriddenMethods, OverriddenMethods[] from CXXMethodDecl. - // - // Add an AbbrevOp for 'size then elements' and use it here. - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - return Abv; -} - -template -std::shared_ptr getCXXMethodAbbrev() { - using namespace llvm; - auto Abv = getFunctionDeclAbbrev(serialization::DECL_CXX_METHOD); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return std::move(Abv); -} -} // namespace - void ASTWriter::WriteDeclAbbrevs() { using namespace llvm; @@ -2402,81 +2290,71 @@ void ASTWriter::WriteDeclAbbrevs() { DeclVarAbbrev = Stream.EmitAbbrev(std::move(Abv)); // Abbreviation for DECL_CXX_METHOD - DeclCXXMethodAbbrev = - Stream.EmitAbbrev(getCXXMethodAbbrev()); - DeclTemplateCXXMethodAbbrev = Stream.EmitAbbrev( - getCXXMethodAbbrev()); - DeclDependentNonTemplateCXXMethodAbbrev = Stream.EmitAbbrev( - getCXXMethodAbbrev()); - DeclMemberSpecializedCXXMethodAbbrev = Stream.EmitAbbrev( - getCXXMethodAbbrev()); - DeclTemplateSpecializedCXXMethodAbbrev = Stream.EmitAbbrev( - getCXXMethodAbbrev()); - DeclDependentSpecializationCXXMethodAbbrev = Stream.EmitAbbrev( - getCXXMethodAbbrev< - FunctionDecl::TK_DependentFunctionTemplateSpecialization>()); - - // Abbreviation for DECL_TEMPLATE_TYPE_PARM - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::DECL_TEMPLATE_TYPE_PARM)); - Abv->Add(BitCodeAbbrevOp(0)); // hasTypeConstraint - // Decl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID - // NamedDecl - Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name - Abv->Add(BitCodeAbbrevOp(0)); - // TypeDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type Ref - // TemplateTypeParmDecl - Abv->Add( - BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // wasDeclaredWithTypename - Abv->Add(BitCodeAbbrevOp(0)); // OwnsDefaultArg - DeclTemplateTypeParmAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for DECL_USING_SHADOW Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::DECL_USING_SHADOW)); - // Redeclarable - Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration + Abv->Add(BitCodeAbbrevOp(serialization::DECL_CXX_METHOD)); + // RedeclarableDecl + Abv->Add(BitCodeAbbrevOp(0)); // CanonicalDecl + // FIXME: Implement abbreviation for other template kinds. + Abv->Add(BitCodeAbbrevOp(FunctionDecl::TK_NonTemplate)); // TemplateKind // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 12)); // Packed DeclBits: HasStandaloneLexicalDC, // isInvalidDecl, HasAttrs, isImplicit, isUsed, // isReferenced, TopLevelDeclInObjCContainer, // AccessSpecifier, ModuleOwnershipKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl - Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Name - Abv->Add(BitCodeAbbrevOp(0)); - // UsingShadowDecl - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TargetDecl + Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Identifier + Abv->Add(BitCodeAbbrevOp(0)); // AnonDeclNumber + // ValueDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // DeclaratorDecl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // InnerLocStart + Abv->Add(BitCodeAbbrevOp(0)); // HasExtInfo + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TSIType + // FunctionDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 11)); // IDNS - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // UsingOrNextShadow - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, - 6)); // InstantiatedFromUsingShadowDecl - DeclUsingShadowAbbrev = Stream.EmitAbbrev(std::move(Abv)); + Abv->Add(BitCodeAbbrevOp( + BitCodeAbbrevOp::Fixed, + 27)); // Packed Function Bits: StorageClass, Inline, InlineSpecified, + // VirtualAsWritten, Pure, HasInheritedProto, HasWrittenProto, + // Deleted, Trivial, TrivialForCall, Defaulted, ExplicitlyDefaulted, + // IsIneligibleOrNotSelected, ImplicitReturnZero, Constexpr, + // UsesSEHTry, SkippedBody, MultiVersion, LateParsed, + // FriendConstraintRefersToEnclosingTemplate, Linkage + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LocEnd + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Default + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // ODRHash + // This Array slurps the rest of the record. Fortunately we want to encode + // (nearly) all the remaining (variable number of) fields in the same way. + // + // This is: + // NumParams and Params[] from FunctionDecl, and + // NumOverriddenMethods, OverriddenMethods[] from CXXMethodDecl. + // + // Add an AbbrevOp for 'size then elements' and use it here. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + DeclCXXMethodAbbrev = Stream.EmitAbbrev(std::move(Abv)); // Abbreviation for EXPR_DECL_REF Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::EXPR_DECL_REF)); - // Stmt - // Expr - // PackingBits: DependenceKind, ValueKind, ObjectKind, HasQualifier, - // GetDeclFound, ExplicitTemplateArgs, HadMultipleCandidates, - // NonOdrUseReason, RefersToEnclosingVariableOrCapture, IsImmediateEscalating - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 18)); + //Stmt + // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // DeclRefExpr + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); + //DeclRefExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HasQualifier + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //GetDeclFound + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //ExplicitTemplateArgs + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HadMultipleCandidates + Abv->Add(BitCodeAbbrevOp(0)); // RefersToEnclosingVariableOrCapture + Abv->Add(BitCodeAbbrevOp(0)); // NonOdrUseReason + Abv->Add(BitCodeAbbrevOp(0)); // IsImmediateEscalating Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclRef Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location DeclRefExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2486,10 +2364,10 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_INTEGER_LITERAL)); //Stmt // Expr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // DependenceKind, ValueKind, ObjectKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // Integer Literal + //Integer Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(32)); // Bit Width Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Value @@ -2500,10 +2378,10 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CHARACTER_LITERAL)); //Stmt // Expr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // DependenceKind, ValueKind, ObjectKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // Character Literal + //Character Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // getValue Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // getKind @@ -2514,98 +2392,17 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_IMPLICIT_CAST)); // Stmt // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, CastKind, PartOfExplicitCast - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 19)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); // CastExpr Abv->Add(BitCodeAbbrevOp(0)); // PathSize + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasFPFeatures + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // CastKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // PartOfExplicitCast // ImplicitCastExpr ExprImplicitCastAbbrev = Stream.EmitAbbrev(std::move(Abv)); - // Abbreviation for EXPR_BINARY_OPERATOR - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::EXPR_BINARY_OPERATOR)); - // Stmt - // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, OpKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 17)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // BinaryOperator - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - BinaryOperatorAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for EXPR_COMPOUND_ASSIGN_OPERATOR - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::EXPR_COMPOUND_ASSIGN_OPERATOR)); - // Stmt - // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, OpKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 17)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // BinaryOperator - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - // CompoundAssignOperator - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHSType - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Result Type - CompoundAssignOperatorAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for EXPR_CALL - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CALL)); - // Stmt - // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 25)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // CallExpr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - CallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for EXPR_CXX_OPERATOR_CALL - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CXX_OPERATOR_CALL)); - // Stmt - // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind, OperatorKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 31)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // CallExpr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - // CXXOperatorCallExpr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - CXXOperatorCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for EXPR_CXX_MEMBER_CALL - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::EXPR_CXX_MEMBER_CALL)); - // Stmt - // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 25)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - // CallExpr - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - // CXXMemberCallExpr - CXXMemberCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); - - // Abbreviation for STMT_COMPOUND - Abv = std::make_shared(); - Abv->Add(BitCodeAbbrevOp(serialization::STMT_COMPOUND)); - // Stmt - // CompoundStmt - // Packing Bits: Num Stmts, hasStoredFPFeatures - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 21)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location - CompoundStmtAbbrev = Stream.EmitAbbrev(std::move(Abv)); - Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_CONTEXT_LEXICAL)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 10139f1dc96ae..8524484ea8a0b 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -37,70 +37,15 @@ namespace clang { serialization::StmtCode Code; unsigned AbbrevToUse; - /// A helper that can help us to write a packed bit across function - /// calls. For example, we may write seperate bits in seperate functions: - /// - /// void VisitA(A* a) { - /// Record.push_back(a->isSomething()); - /// } - /// - /// void Visitb(B *b) { - /// VisitA(b); - /// Record.push_back(b->isAnother()); - /// } - /// - /// In such cases, it'll be better if we can pack these 2 bits. We achieve - /// this by writing a zero value in `VisitA` and recorded that first and add - /// the new bit to the recorded value. - class PakedBitsWriter { - public: - PakedBitsWriter(ASTRecordWriter &Record) : RecordRef(Record) {} - ~PakedBitsWriter() { assert(!CurrentIndex); } - - void addBit(bool Value) { - assert(CurrentIndex && "Writing Bits without recording first!"); - PackingBits.addBit(Value); - } - void addBits(uint32_t Value, uint32_t BitsWidth) { - assert(CurrentIndex && "Writing Bits without recording first!"); - PackingBits.addBits(Value, BitsWidth); - } - - void writeBits() { - if (!CurrentIndex) - return; - - RecordRef[*CurrentIndex] = (uint32_t)PackingBits; - CurrentIndex = std::nullopt; - PackingBits.reset(0); - } - - void updateBits() { - writeBits(); - - CurrentIndex = RecordRef.size(); - RecordRef.push_back(0); - } - - private: - BitsPacker PackingBits; - ASTRecordWriter &RecordRef; - std::optional CurrentIndex; - }; - - PakedBitsWriter CurrentPackingBits; - public: ASTStmtWriter(ASTWriter &Writer, ASTWriter::RecordData &Record) : Writer(Writer), Record(Writer, Record), - Code(serialization::STMT_NULL_PTR), AbbrevToUse(0), - CurrentPackingBits(this->Record) {} + Code(serialization::STMT_NULL_PTR), AbbrevToUse(0) {} ASTStmtWriter(const ASTStmtWriter&) = delete; ASTStmtWriter &operator=(const ASTStmtWriter &) = delete; uint64_t Emit() { - CurrentPackingBits.writeBits(); assert(Code != serialization::STMT_NULL_PTR && "unhandled sub-statement writing AST file"); return Record.EmitStmt(Code, AbbrevToUse); @@ -137,22 +82,14 @@ void ASTStmtWriter::VisitNullStmt(NullStmt *S) { void ASTStmtWriter::VisitCompoundStmt(CompoundStmt *S) { VisitStmt(S); - - CurrentPackingBits.updateBits(); - // 20 bits should be enough to store the size of stmts. - CurrentPackingBits.addBits(S->size(), /*Width=*/20); - CurrentPackingBits.addBit(S->hasStoredFPFeatures()); - + Record.push_back(S->size()); + Record.push_back(S->hasStoredFPFeatures()); for (auto *CS : S->body()) Record.AddStmt(CS); if (S->hasStoredFPFeatures()) Record.push_back(S->getStoredFPFeatures().getAsOpaqueInt()); Record.AddSourceLocation(S->getLBracLoc()); Record.AddSourceLocation(S->getRBracLoc()); - - if (!S->hasStoredFPFeatures()) - AbbrevToUse = Writer.getCompoundStmtAbbrev(); - Code = serialization::STMT_COMPOUND; } @@ -206,11 +143,9 @@ void ASTStmtWriter::VisitIfStmt(IfStmt *S) { bool HasVar = S->getConditionVariableDeclStmt() != nullptr; bool HasInit = S->getInit() != nullptr; - CurrentPackingBits.updateBits(); - - CurrentPackingBits.addBit(HasElse); - CurrentPackingBits.addBit(HasVar); - CurrentPackingBits.addBit(HasInit); + Record.push_back(HasElse); + Record.push_back(HasVar); + Record.push_back(HasInit); Record.push_back(static_cast(S->getStatementKind())); Record.AddStmt(S->getCond()); Record.AddStmt(S->getThen()); @@ -613,13 +548,15 @@ void ASTStmtWriter::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtWriter::VisitExpr(Expr *E) { VisitStmt(E); + Record.AddTypeRef(E->getType()); - CurrentPackingBits.updateBits(); - CurrentPackingBits.addBits(E->getDependence(), /*BitsWidth=*/5); - CurrentPackingBits.addBits(E->getValueKind(), /*BitsWidth=*/2); - CurrentPackingBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); + BitsPacker ExprBits; - Record.AddTypeRef(E->getType()); + ExprBits.addBits(E->getDependence(), /*BitsWidth=*/5); + ExprBits.addBits(E->getValueKind(), /*BitsWidth=*/2); + ExprBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); + + Record.push_back(ExprBits); } void ASTStmtWriter::VisitConstantExpr(ConstantExpr *E) { @@ -675,25 +612,26 @@ void ASTStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - CurrentPackingBits.addBit(E->hasQualifier()); - CurrentPackingBits.addBit(E->getDecl() != E->getFoundDecl()); - CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); - CurrentPackingBits.addBit(E->hadMultipleCandidates()); - CurrentPackingBits.addBit(E->refersToEnclosingVariableOrCapture()); - CurrentPackingBits.addBits(E->isNonOdrUse(), /*Width=*/2); - CurrentPackingBits.addBit(E->isImmediateEscalating()); + Record.push_back(E->hasQualifier()); + Record.push_back(E->getDecl() != E->getFoundDecl()); + Record.push_back(E->hasTemplateKWAndArgsInfo()); + Record.push_back(E->hadMultipleCandidates()); + Record.push_back(E->refersToEnclosingVariableOrCapture()); + Record.push_back(E->isNonOdrUse()); + Record.push_back(E->isImmediateEscalating()); if (E->hasTemplateKWAndArgsInfo()) { unsigned NumTemplateArgs = E->getNumTemplateArgs(); - // 12 bits should be sufficient to store the number of template args. - CurrentPackingBits.addBits(NumTemplateArgs, /*Width=*/12); + Record.push_back(NumTemplateArgs); } DeclarationName::NameKind nk = (E->getDecl()->getDeclName().getNameKind()); if ((!E->hasTemplateKWAndArgsInfo()) && (!E->hasQualifier()) && (E->getDecl() == E->getFoundDecl()) && - nk == DeclarationName::Identifier) { + nk == DeclarationName::Identifier && + !E->refersToEnclosingVariableOrCapture() && !E->isNonOdrUse() && + !E->isImmediateEscalating()) { AbbrevToUse = Writer.getDeclRefExprAbbrev(); } @@ -804,13 +742,11 @@ void ASTStmtWriter::VisitUnaryOperator(UnaryOperator *E) { bool HasFPFeatures = E->hasStoredFPFeatures(); // Write this first for easy access when deserializing, as they affect the // size of the UnaryOperator. - CurrentPackingBits.addBit(HasFPFeatures); + Record.push_back(HasFPFeatures); Record.AddStmt(E->getSubExpr()); - CurrentPackingBits.addBits(E->getOpcode(), - /*Width=*/5); // FIXME: stable encoding + Record.push_back(E->getOpcode()); // FIXME: stable encoding Record.AddSourceLocation(E->getOperatorLoc()); - CurrentPackingBits.addBit(E->canOverflow()); - + Record.push_back(E->canOverflow()); if (HasFPFeatures) Record.push_back(E->getStoredFPFeatures().getAsOpaqueInt()); Code = serialization::EXPR_UNARY_OPERATOR; @@ -936,10 +872,12 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtWriter::VisitCallExpr(CallExpr *E) { VisitExpr(E); - // 13 bits should be sufficient to store the number args; - CurrentPackingBits.addBits(E->getNumArgs(), /*BitsWidth=*/13); - CurrentPackingBits.addBit(E->hasStoredFPFeatures()); - CurrentPackingBits.addBit(static_cast(E->getADLCallKind())); + BitsPacker CallExprBits; + // 16 bits should be sufficient to store the number args; + CallExprBits.addBits(E->getNumArgs(), /*BitsWidth=*/16); + CallExprBits.addBit(E->hasStoredFPFeatures()); + CallExprBits.addBit(static_cast(E->getADLCallKind())); + Record.push_back(CallExprBits); Record.AddSourceLocation(E->getRParenLoc()); Record.AddStmt(E->getCallee()); @@ -949,10 +887,6 @@ void ASTStmtWriter::VisitCallExpr(CallExpr *E) { if (E->hasStoredFPFeatures()) Record.push_back(E->getFPFeatures().getAsOpaqueInt()); - - if (!E->hasStoredFPFeatures() && E->getStmtClass() == Stmt::CallExprClass) - AbbrevToUse = Writer.getCallExprAbbrev(); - Code = serialization::EXPR_CALL; } @@ -979,27 +913,25 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) { // Write these first for easy access when deserializing, as they affect the // size of the MemberExpr. - - CurrentPackingBits.addBit(HasQualifier); - CurrentPackingBits.addBit(HasFoundDecl); - CurrentPackingBits.addBit(HasTemplateInfo); - // 12 bits should be enough to store the number of args - CurrentPackingBits.addBits(NumTemplateArgs, /*Width=*/12); + Record.push_back(HasQualifier); + Record.push_back(HasFoundDecl); + Record.push_back(HasTemplateInfo); + Record.push_back(NumTemplateArgs); Record.AddStmt(E->getBase()); Record.AddDeclRef(E->getMemberDecl()); Record.AddDeclarationNameLoc(E->MemberDNLoc, E->getMemberDecl()->getDeclName()); Record.AddSourceLocation(E->getMemberLoc()); - CurrentPackingBits.addBit(E->isArrow()); - CurrentPackingBits.addBit(E->hadMultipleCandidates()); - CurrentPackingBits.addBits(E->isNonOdrUse(), /*Width=*/2); + Record.push_back(E->isArrow()); + Record.push_back(E->hadMultipleCandidates()); + Record.push_back(E->isNonOdrUse()); Record.AddSourceLocation(E->getOperatorLoc()); if (HasFoundDecl) { DeclAccessPair FoundDecl = E->getFoundDecl(); Record.AddDeclRef(FoundDecl.getDecl()); - CurrentPackingBits.addBits(FoundDecl.getAccess(), /*BitWidth=*/2); + Record.push_back(FoundDecl.getAccess()); } if (HasQualifier) @@ -1039,12 +971,10 @@ void ASTStmtWriter::VisitObjCBridgedCastExpr(ObjCBridgedCastExpr *E) { void ASTStmtWriter::VisitCastExpr(CastExpr *E) { VisitExpr(E); - Record.push_back(E->path_size()); - CurrentPackingBits.addBit(E->hasStoredFPFeatures()); - // 7 bits should be enough to store the casting kinds. - CurrentPackingBits.addBits(E->getCastKind(), /*Width=*/7); + Record.push_back(E->hasStoredFPFeatures()); Record.AddStmt(E->getSubExpr()); + Record.push_back(E->getCastKind()); // FIXME: stable encoding for (CastExpr::path_iterator PI = E->path_begin(), PE = E->path_end(); PI != PE; ++PI) @@ -1059,17 +989,13 @@ void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) { bool HasFPFeatures = E->hasStoredFPFeatures(); // Write this first for easy access when deserializing, as they affect the // size of the UnaryOperator. - CurrentPackingBits.addBit(HasFPFeatures); - CurrentPackingBits.addBits(E->getOpcode(), /*Width=*/6); + Record.push_back(HasFPFeatures); + Record.push_back(E->getOpcode()); // FIXME: stable encoding Record.AddStmt(E->getLHS()); Record.AddStmt(E->getRHS()); Record.AddSourceLocation(E->getOperatorLoc()); if (HasFPFeatures) Record.push_back(E->getStoredFPFeatures().getAsOpaqueInt()); - - if (!HasFPFeatures) - AbbrevToUse = Writer.getBinaryOperatorAbbrev(); - Code = serialization::EXPR_BINARY_OPERATOR; } @@ -1077,10 +1003,6 @@ void ASTStmtWriter::VisitCompoundAssignOperator(CompoundAssignOperator *E) { VisitBinaryOperator(E); Record.AddTypeRef(E->getComputationLHSType()); Record.AddTypeRef(E->getComputationResultType()); - - if (!E->hasStoredFPFeatures()) - AbbrevToUse = Writer.getCompoundAssignOperatorAbbrev(); - Code = serialization::EXPR_COMPOUND_ASSIGN_OPERATOR; } @@ -1109,7 +1031,7 @@ ASTStmtWriter::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) { void ASTStmtWriter::VisitImplicitCastExpr(ImplicitCastExpr *E) { VisitCastExpr(E); - CurrentPackingBits.addBit(E->isPartOfExplicitCast()); + Record.push_back(E->isPartOfExplicitCast()); if (E->path_size() == 0 && !E->hasStoredFPFeatures()) AbbrevToUse = Writer.getExprImplicitCastAbbrev(); @@ -1664,21 +1586,13 @@ void ASTStmtWriter::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) { void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { VisitCallExpr(E); - CurrentPackingBits.addBits(E->getOperator(), /*Width=*/6); + Record.push_back(E->getOperator()); Record.AddSourceRange(E->Range); - - if (!E->hasStoredFPFeatures()) - AbbrevToUse = Writer.getCXXOperatorCallExprAbbrev(); - Code = serialization::EXPR_CXX_OPERATOR_CALL; } void ASTStmtWriter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { VisitCallExpr(E); - - if (!E->hasStoredFPFeatures()) - AbbrevToUse = Writer.getCXXMemberCallExprAbbrev(); - Code = serialization::EXPR_CXX_MEMBER_CALL; } @@ -1759,9 +1673,7 @@ void ASTStmtWriter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) void ASTStmtWriter::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) { VisitExplicitCastExpr(E); Record.AddSourceRange(SourceRange(E->getOperatorLoc(), E->getRParenLoc())); - CurrentPackingBits.addBit(E->getAngleBrackets().isValid()); - if (E->getAngleBrackets().isValid()) - Record.AddSourceRange(E->getAngleBrackets()); + Record.AddSourceRange(E->getAngleBrackets()); } void ASTStmtWriter::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) { @@ -1972,10 +1884,9 @@ void ASTStmtWriter::VisitCXXDependentScopeMemberExpr( // Don't emit anything here (or if you do you will have to update // the corresponding deserialization function). - CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); - // 16 bits should be enough to store the number of template args. - CurrentPackingBits.addBits(E->getNumTemplateArgs(), /*Width=*/16); - CurrentPackingBits.addBit(E->hasFirstQualifierFoundInScope()); + Record.push_back(E->hasTemplateKWAndArgsInfo()); + Record.push_back(E->getNumTemplateArgs()); + Record.push_back(E->hasFirstQualifierFoundInScope()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = @@ -1984,15 +1895,14 @@ void ASTStmtWriter::VisitCXXDependentScopeMemberExpr( E->getTrailingObjects()); } - CurrentPackingBits.addBit(E->isArrow()); - + Record.push_back(E->isArrow()); + Record.AddSourceLocation(E->getOperatorLoc()); Record.AddTypeRef(E->getBaseType()); Record.AddNestedNameSpecifierLoc(E->getQualifierLoc()); - CurrentPackingBits.addBit(!E->isImplicitAccess()); - if (!E->isImplicitAccess()) { + if (!E->isImplicitAccess()) Record.AddStmt(E->getBase()); - Record.AddSourceLocation(E->getOperatorLoc()); - } + else + Record.AddStmt(nullptr); if (E->hasFirstQualifierFoundInScope()) Record.AddDeclRef(E->getFirstQualifierFoundInScope()); @@ -2007,14 +1917,12 @@ ASTStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { // Don't emit anything here, HasTemplateKWAndArgsInfo must be // emitted first. - CurrentPackingBits.addBit( - E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo); + Record.push_back(E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo); if (E->DependentScopeDeclRefExprBits.HasTemplateKWAndArgsInfo) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingObjects(); - // 16 bits should be enought to store the number of args - CurrentPackingBits.addBits(ArgInfo.NumTemplateArgs, /*Width=*/16); + Record.push_back(ArgInfo.NumTemplateArgs); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingObjects()); } @@ -2041,17 +1949,19 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - CurrentPackingBits.updateBits(); - // 12 Bits should enough to store the number of decls. - CurrentPackingBits.addBits(E->getNumDecls(), /*BitWidth=*/12); - CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); + BitsPacker OverloadExprBits; + // 14 Bits should enough to store the number of decls. + OverloadExprBits.addBits(E->getNumDecls(), /*BitWidth=*/14); + OverloadExprBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingASTTemplateKWAndArgsInfo(); - // 12 Bits should enough to store the number of template args. - CurrentPackingBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/12); + // 14 Bits should enough to store the number of template args. + OverloadExprBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/14); + Record.push_back(OverloadExprBits); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingTemplateArgumentLoc()); - } + } else + Record.push_back(OverloadExprBits); for (OverloadExpr::decls_iterator OvI = E->decls_begin(), OvE = E->decls_end(); @@ -2066,22 +1976,18 @@ void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { void ASTStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) { VisitOverloadExpr(E); - CurrentPackingBits.addBit(E->isArrow()); - CurrentPackingBits.addBit(E->hasUnresolvedUsing()); - CurrentPackingBits.addBit(!E->isImplicitAccess()); - if (!E->isImplicitAccess()) { - Record.AddStmt(E->getBase()); - Record.AddSourceLocation(E->getOperatorLoc()); - } - + Record.push_back(E->isArrow()); + Record.push_back(E->hasUnresolvedUsing()); + Record.AddStmt(!E->isImplicitAccess() ? E->getBase() : nullptr); Record.AddTypeRef(E->getBaseType()); + Record.AddSourceLocation(E->getOperatorLoc()); Code = serialization::EXPR_CXX_UNRESOLVED_MEMBER; } void ASTStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) { VisitOverloadExpr(E); - CurrentPackingBits.addBit(E->requiresADL()); - CurrentPackingBits.addBit(E->isOverloaded()); + Record.push_back(E->requiresADL()); + Record.push_back(E->isOverloaded()); Record.AddDeclRef(E->getNamingClass()); Code = serialization::EXPR_CXX_UNRESOLVED_LOOKUP; } @@ -2153,12 +2059,12 @@ void ASTStmtWriter::VisitSubstNonTypeTemplateParmExpr( SubstNonTypeTemplateParmExpr *E) { VisitExpr(E); Record.AddDeclRef(E->getAssociatedDecl()); - CurrentPackingBits.addBit(E->isReferenceParameter()); - CurrentPackingBits.addBits(E->getIndex(), /*Width=*/12); - CurrentPackingBits.addBit((bool)E->getPackIndex()); + Record.push_back(E->isReferenceParameter()); + Record.push_back(E->getIndex()); if (auto PackIndex = E->getPackIndex()) Record.push_back(*PackIndex + 1); - + else + Record.push_back(0); Record.AddSourceLocation(E->getNameLoc()); Record.AddStmt(E->getReplacement()); Code = serialization::EXPR_SUBST_NON_TYPE_TEMPLATE_PARM; From b3e353d263f9d6ef061f4e6d89619c72a3553002 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Fri, 15 Dec 2023 15:15:18 -0800 Subject: [PATCH 025/884] [X86] Don't use rip-relative lea to get a function address in medium static mode (#75656) This essentially reverts https://reviews.llvm.org/D140593. Somewhere along the line we properly fixed the medium code model to assume functions are small, so now we get a 32-bit movl as desired. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ------- llvm/test/CodeGen/X86/code-model-elf.ll | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index cb117475dbe4c..99c492087a458 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -18329,13 +18329,6 @@ unsigned X86TargetLowering::getGlobalWrapperKind( OpFlags == X86II::MO_DLLIMPORT)) return X86ISD::WrapperRIP; - // In the medium model, functions can always be referenced RIP-relatively, - // since they must be within 2GiB. This is also possible in non-PIC mode, and - // shorter than the 64-bit absolute immediate that would otherwise be emitted. - if (getTargetMachine().getCodeModel() == CodeModel::Medium && - isa_and_nonnull(GV)) - return X86ISD::WrapperRIP; - // GOTPCREL references must always use RIP. if (OpFlags == X86II::MO_GOTPCREL || OpFlags == X86II::MO_GOTPCREL_NORELAX) return X86ISD::WrapperRIP; diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index be93f6530ae4c..6112f2a57b82c 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -829,7 +829,7 @@ define dso_local ptr @lea_static_fn() #0 { ; ; MEDIUM-STATIC-LABEL: lea_static_fn: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: leaq static_fn(%rip), %rax +; MEDIUM-STATIC-NEXT: movl $static_fn, %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: lea_static_fn: @@ -882,7 +882,7 @@ define dso_local ptr @lea_global_fn() #0 { ; ; MEDIUM-STATIC-LABEL: lea_global_fn: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: leaq global_fn(%rip), %rax +; MEDIUM-STATIC-NEXT: movl $global_fn, %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: lea_global_fn: From 6c06bde7c4a89b4b4fa900c51049fdf34d9f042f Mon Sep 17 00:00:00 2001 From: Peiming Liu <36770114+PeimingLiu@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:33:31 -0800 Subject: [PATCH 026/884] [mlir][sparse] support loop range query using SparseTensorLevel. (#75670) --- .../Transforms/Utils/LoopEmitter.cpp | 71 +++------ .../Transforms/Utils/SparseTensorLevel.cpp | 138 +++++++++++++++++- .../Transforms/Utils/SparseTensorLevel.h | 63 ++------ .../sparse_conv_2d_slice_based.mlir | 15 +- 4 files changed, 175 insertions(+), 112 deletions(-) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp index 0ba7cf33b6cba..35faf1769746d 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp @@ -244,12 +244,12 @@ Value LoopEmitter::genAddress(OpBuilder &builder, Location loc, TensorId tid, Value LoopEmitter::genSegmentHigh(OpBuilder &builder, Location loc, TensorId tid, Level lvl, Value pLo, Value pHi) { - SparseTensorLevel &level = *lvls[tid][lvl]; - const Value sameCrd = level.peekCrdAt(builder, loc, pLo); + SparseTensorLevel &stl = *lvls[tid][lvl]; + const Value sameCrd = stl.peekCrdAt(builder, loc, pLo); auto whileOp = builder.create( loc, builder.getIndexType(), pLo, /*beforeBuilder=*/ - [pHi, &level, sameCrd](OpBuilder &builder, Location loc, ValueRange ivs) { + [pHi, &stl, sameCrd](OpBuilder &builder, Location loc, ValueRange ivs) { const auto pos = ivs[0]; Value inBound = builder.create( loc, arith::CmpIPredicate::ult, pos, pHi); @@ -260,7 +260,7 @@ Value LoopEmitter::genSegmentHigh(OpBuilder &builder, Location loc, // Load the next coordinates only when inbound (to avoid OOB // accesses). builder.setInsertionPointToStart(ifInBound.thenBlock()); - Value crd = level.peekCrdAt(builder, loc, pos); + Value crd = stl.peekCrdAt(builder, loc, pos); Value isSameCrd = builder.create( loc, arith::CmpIPredicate::eq, crd, sameCrd); YIELD(isSameCrd); @@ -1226,27 +1226,19 @@ void LoopEmitter::prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc, const Value c0 = C_IDX(0); const Value c1 = C_IDX(1); - const Value c2 = C_IDX(2); // Either the first level, or the previous level has been set. /// FIXME: See the [CLARIFY_POSITS_LVL] note in the header. assert(lvl == 0 || posits[tid][lvl - 1]); - if (isCompressedLT(lvlTp) || isLooseCompressedLT(lvlTp)) { - // TODO: eliminate the cast upon feature complete. - const Value mem = - isCompressedLT(lvlTp) - ? static_cast(*lvls[tid][lvl]).posBuffer - : static_cast(*lvls[tid][lvl]).posBuffer; - - Value pLo = lvl == 0 ? c0 : posits[tid][lvl - 1]; - if (isLooseCompressedLT(lvlTp)) - pLo = builder.create(loc, pLo, c2); - posits[tid][lvl] = genIndexLoad(builder, loc, mem, pLo); - - const Value pHi = ADDI(pLo, c1); - highs[tid][lvl] = genIndexLoad(builder, loc, mem, pHi); + if (isCompressedLT(lvlTp) || isLooseCompressedLT(lvlTp) || + is2OutOf4LT(lvlTp)) { + + Value pos = lvl == 0 ? c0 : posits[tid][lvl - 1]; + std::tie(posits[tid][lvl], highs[tid][lvl]) = + lvls[tid][lvl]->peekRangeAt(builder, loc, pos); return; } if (isSingletonLT(lvlTp)) { + // TODO: merge this as well when SparseTensorLevel support dedup. const Value pLo = lvl == 0 ? c0 : posits[tid][lvl - 1]; posits[tid][lvl] = pLo; @@ -1262,13 +1254,6 @@ void LoopEmitter::prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc, : ADDI(pLo, c1); return; } - if (is2OutOf4LT(lvlTp)) { - const Value pLo = lvl == 0 ? c0 : posits[tid][lvl - 1]; - // Each 2:4 block has exactly two specified elements. - posits[tid][lvl] = MULI(pLo, c2); - highs[tid][lvl] = ADDI(posits[tid][lvl], c2); - return; - } llvm_unreachable("Unrecognized level-type!"); } @@ -1824,18 +1809,11 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, auto [nxSz, stride] = sliceMeta[tid][lvl][1]; assert(stride == 1 && "Not yet implemented"); Value sPtrBuf = slicePosBuffer[tid][lvl][0]; - Value pHi, pLo; - if (lvl == 0) { - pLo = c0; - // TODO: eliminate the cast upon feature complete.pLo = c0; - Value pBuf = static_cast(*lvls[tid][0]).posBuffer; - pHi = genIndexLoad(builder, loc, pBuf, c1); - } else { - // TODO: eliminate the cast upon feature complete.} else { - Value pBuf = static_cast(*lvls[tid][lvl]).posBuffer; - pLo = genIndexLoad(builder, loc, pBuf, posits[tid][lvl - 1]); - pHi = genIndexLoad(builder, loc, pBuf, ADDI(posits[tid][lvl - 1], c1)); - } + const SparseTensorLevel &stl = *lvls[tid][lvl]; + + Value p = lvl == 0 ? c0 : posits[tid][lvl - 1]; + auto [pLo, pHi] = stl.peekRangeAt(builder, loc, p); + // Fills out pIdxBuffer[tid][lvl][0] with [pLo, pHi] updateSlicePos(builder, loc, sPtrBuf, pLo, c0, SlicePosKind::kLo); updateSlicePos(builder, loc, sPtrBuf, pHi, c0, SlicePosKind::kHi); @@ -1849,7 +1827,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // nonempty. though we assume that even on empty sparse tensors, a non-empty // ptr/idx buffer is allocated for each level so it would not cause OOB to // avoid generating a ifOp here. - Value minCrd = lvls[tid][lvl]->peekCrdAt(builder, loc, pLo); + Value minCrd = stl.peekCrdAt(builder, loc, pLo); // FIXME: We need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, nxSz, isNonEmpty); @@ -1879,7 +1857,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // } void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { - Value c0 = C_IDX(0), c1 = C_IDX(1); + Value c0 = C_IDX(0); unsigned depth = levelReducedDep[tid][lvl]; // The remaining slice size after reduction. Value remSz = sliceMeta[tid][lvl][depth + 1].first; @@ -1929,17 +1907,14 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, ValueRange result = genUnResolvedSliceTreeTraverse( builder, loc, tid, unResSlices, firstResLvl, reduc, - [this, c1, tid, lvl, sPtrBuf](OpBuilder &builder, Location loc, Value iv, - MutableArrayRef reduc) { + [this, tid, lvl, sPtrBuf](OpBuilder &builder, Location loc, Value iv, + MutableArrayRef reduc) { Value &nonEmpty = reduc[0]; Value &minCrd = reduc[1]; Value &curTupleCnt = reduc[2]; - Value pHi = ADDI(iv, c1); - // TODO: eliminate the cast upon feature complete. - Value pBuf = static_cast(*lvls[tid][lvl]).posBuffer; - Value sPLo = genIndexLoad(builder, loc, pBuf, iv); - Value sPHi = genIndexLoad(builder, loc, pBuf, pHi); + const SparseTensorLevel &stl = *lvls[tid][lvl]; + auto [sPLo, sPHi] = stl.peekRangeAt(builder, loc, iv); // isNonEmpty = isNonEmpty || lvlNonEmpty, i.e., as long as there is // one non-empty lvl, the slice is non-empty. @@ -1957,7 +1932,7 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, // } OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToStart(ifNonEmpty.thenBlock()); - Value curC = lvls[tid][lvl]->peekCrdAt(builder, loc, sPLo); + Value curC = stl.peekCrdAt(builder, loc, sPLo); Value isSmaller = CMPI(ult, curC, minCrd); Value newMin = SELECT(isSmaller, curC, minCrd); YIELD(newMin); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp index d9d26794d7bce..aea0910d980ab 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp @@ -13,6 +13,133 @@ using namespace mlir; using namespace mlir::sparse_tensor; +using ValuePair = std::pair; + +//===----------------------------------------------------------------------===// +// File local helper functions/macros. +//===----------------------------------------------------------------------===// +#define CMPI(p, lhs, rhs) \ + (b.create(l, arith::CmpIPredicate::p, (lhs), (rhs))) + +#define C_IDX(v) (constantIndex(b, l, (v))) +#define YIELD(vs) (b.create(l, (vs))) +#define ADDI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define ANDI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define SUBI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define MULI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define REMUI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define DIVUI(lhs, rhs) (b.create(l, (lhs), (rhs))) +#define SELECT(c, lhs, rhs) (b.create(l, (c), (lhs), (rhs))) + +static ValuePair constantRange(OpBuilder &b, Location l, Value lo, Value sz) { + return std::make_pair(lo, ADDI(lo, sz)); +} + +//===----------------------------------------------------------------------===// +// SparseTensorLevel derived classes. +//===----------------------------------------------------------------------===// + +namespace { + +class SparseLevel : public SparseTensorLevel { +public: + SparseLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseTensorLevel(lt, lvlSize), crdBuffer(crdBuffer) {} + + Value peekCrdAt(OpBuilder &b, Location l, Value pos) const override { + return genIndexLoad(b, l, crdBuffer, pos); + } + +protected: + const Value crdBuffer; +}; + +class DenseLevel : public SparseTensorLevel { +public: + DenseLevel(Value lvlSize) : SparseTensorLevel(LevelType::Dense, lvlSize) { + // Dense level, loop upper bound equals to the level size. + loopHi = lvlSize; + } + + Value peekCrdAt(OpBuilder &, Location, Value pos) const override { + return pos; + } + + ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + Value max) const override { + assert(max == nullptr && "Dense level can not be non-unique."); + return constantRange(b, l, C_IDX(0), lvlSize); + } +}; + +class CompressedLevel : public SparseLevel { +public: + CompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} + + ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + Value max) const override { + if (max == nullptr) { + Value pLo = genIndexLoad(b, l, posBuffer, p); + Value pHi = genIndexLoad(b, l, posBuffer, ADDI(p, C_IDX(1))); + return {pLo, pHi}; + } + llvm_unreachable("TODO: dedup not implemented"); + } + +private: + const Value posBuffer; +}; + +class LooseCompressedLevel : public SparseLevel { +public: + LooseCompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, + Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} + + ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + Value max) const override { + // Allows this? + assert(max == nullptr && "loss compressed level can not be non-unique."); + + p = MULI(p, C_IDX(2)); + Value pLo = genIndexLoad(b, l, posBuffer, p); + Value pHi = genIndexLoad(b, l, posBuffer, ADDI(p, C_IDX(1))); + return {pLo, pHi}; + } + +private: + const Value posBuffer; +}; + +class SingletonLevel : public SparseLevel { +public: + SingletonLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer) {} + + ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + Value max) const override { + if (max == nullptr) + return constantRange(b, l, p, C_IDX(1)); + llvm_unreachable("TODO: dedup not implemented"); + } +}; + +class TwoOutFourLevel : public SparseLevel { +public: + TwoOutFourLevel(LevelType lt, Value lvlSize, Value crdBuffer) + : SparseLevel(lt, lvlSize, crdBuffer) {} + + ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + Value max) const override { + assert(max == nullptr && "2:4 level can not be non-unique."); + // Each 2:4 block has exactly two specified elements. + Value c2 = C_IDX(2); + return constantRange(b, l, MULI(p, c2), c2); + } +}; + +} // namespace std::unique_ptr sparse_tensor::makeSparseTensorLevel(OpBuilder &builder, Location loc, Value t, @@ -49,6 +176,11 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &builder, Location loc, Value t, llvm_unreachable("unrecognizable level format"); } -Value SparseLevel::peekCrdAt(OpBuilder &b, Location l, Value pos) const { - return genIndexLoad(b, l, crdBuffer, pos); -} +#undef CMPI +#undef C_IDX +#undef YIELD +#undef ADDI +#undef ANDI +#undef SUBI +#undef MULI +#undef SELECT diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h index e10356a55cc7e..f5c29cda7c54f 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h @@ -17,6 +17,8 @@ namespace sparse_tensor { class SparseTensorLevel { SparseTensorLevel(SparseTensorLevel &&) = delete; SparseTensorLevel(const SparseTensorLevel &) = delete; + SparseTensorLevel &operator=(SparseTensorLevel &&) = delete; + SparseTensorLevel &operator=(const SparseTensorLevel &) = delete; public: SparseTensorLevel() : SparseTensorLevel(LevelType::Undef, nullptr){}; @@ -24,6 +26,13 @@ class SparseTensorLevel { virtual Value peekCrdAt(OpBuilder &b, Location l, Value p) const = 0; + /// Peeks the lower and upper bound to *fully* traverse the level with + /// the given position `p` that the immediate parent level is current at. + /// `bound` is only used when the level is `non-unique` and deduplication is + /// required. It specifies the max upper bound of the non-unique segment. + virtual std::pair peekRangeAt(OpBuilder &b, Location l, Value p, + Value bound = Value()) const = 0; + LevelType getLT() const { return lt; } Value getPos() const { return pos; } Value getCrd() const { return crd; } @@ -49,60 +58,6 @@ class SparseTensorLevel { std::unique_ptr makeSparseTensorLevel(OpBuilder &builder, Location loc, Value t, Level l); -class DenseLevel : public SparseTensorLevel { -public: - DenseLevel(Value lvlSize) : SparseTensorLevel(LevelType::Dense, lvlSize) { - // Dense level, loop upper bound equals to the level size. - loopHi = lvlSize; - } - - Value peekCrdAt(OpBuilder &, Location, Value pos) const override { - return pos; - } -}; - -class SparseLevel : public SparseTensorLevel { -public: - SparseLevel(LevelType lt, Value lvlSize, Value crdBuffer) - : SparseTensorLevel(lt, lvlSize), crdBuffer(crdBuffer) {} - - Value peekCrdAt(OpBuilder &b, Location l, Value pos) const override; - -public: // TODO: make these values private upon feature complete. - const Value crdBuffer; -}; - -class CompressedLevel : public SparseLevel { -public: - CompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, Value crdBuffer) - : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} - -public: // TODO: make these values private upon feature complete. - const Value posBuffer; -}; - -class LooseCompressedLevel : public SparseLevel { -public: - LooseCompressedLevel(LevelType lt, Value lvlSize, Value posBuffer, - Value crdBuffer) - : SparseLevel(lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} - -public: // TODO: make these values private upon feature complete. - const Value posBuffer; -}; - -class SingletonLevel : public SparseLevel { -public: - SingletonLevel(LevelType lt, Value lvlSize, Value crdBuffer) - : SparseLevel(lt, lvlSize, crdBuffer) {} -}; - -class TwoOutFourLevel : public SparseLevel { -public: - TwoOutFourLevel(LevelType lt, Value lvlSize, Value crdBuffer) - : SparseLevel(lt, lvlSize, crdBuffer) {} -}; - } // namespace sparse_tensor } // namespace mlir diff --git a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir index a3c1e76a3d09a..bf61e792ffbe0 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir @@ -27,11 +27,12 @@ // CHECK-DAG: %[[VAL_16:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x8xi32, #sparse> to memref // CHECK-DAG: %[[VAL_17:.*]] = memref.alloca() : memref<9xindex> // CHECK-DAG: %[[VAL_18:.*]] = memref.alloca() : memref<3xindex> -// CHECK-DAG: %[[VAL_19:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref -// CHECK: memref.store %[[VAL_8]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> -// CHECK: memref.store %[[VAL_19]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> -// CHECK: %[[VAL_20:.*]] = arith.cmpi ugt, %[[VAL_19]], %[[VAL_8]] : index -// CHECK: %[[VAL_21:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_8]]] : memref +// CHECK-DAG: %[[POS_LO:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_8]]] : memref +// CHECK-DAG: %[[POS_HI:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref +// CHECK: memref.store %[[POS_LO]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: memref.store %[[POS_HI]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_20:.*]] = arith.cmpi ult, %[[POS_LO]], %[[POS_HI]] : index +// CHECK: %[[VAL_21:.*]] = memref.load %[[VAL_13]]{{\[}}%[[POS_LO]]] : memref // CHECK: %[[VAL_22:.*]] = arith.cmpi uge, %[[VAL_21]], %[[VAL_6]] : index // CHECK: %[[VAL_23:.*]] = arith.andi %[[VAL_20]], %[[VAL_22]] : i1 // CHECK: %[[VAL_24:.*]] = arith.addi %[[VAL_21]], %[[VAL_3]] : index @@ -56,8 +57,8 @@ // CHECK: scf.condition(%[[VAL_44]]) %[[VAL_38]], %[[VAL_39]], %[[VAL_40]], %[[VAL_41]], %[[VAL_42]] : index, i1, index, index, index // CHECK: } do { // CHECK: ^bb0(%[[VAL_47:.*]]: index, %[[VAL_48:.*]]: i1, %[[VAL_49:.*]]: index, %[[VAL_50:.*]]: index, %[[VAL_51:.*]]: index): -// CHECK: %[[VAL_52:.*]] = arith.addi %[[VAL_47]], %[[VAL_7]] : index -// CHECK: %[[VAL_53:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_47]]] : memref +// CHECK-DAG: %[[VAL_52:.*]] = arith.addi %[[VAL_47]], %[[VAL_7]] : index +// CHECK-DAG: %[[VAL_53:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_47]]] : memref // CHECK: %[[VAL_54:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_52]]] : memref // CHECK: %[[VAL_55:.*]] = arith.cmpi ult, %[[VAL_53]], %[[VAL_54]] : index // CHECK: %[[VAL_56:.*]] = arith.ori %[[VAL_55]], %[[VAL_48]] : i1 From 9950bb994461d6dc67ac9b33e48f549edcb44739 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Fri, 15 Dec 2023 17:11:21 -0800 Subject: [PATCH 027/884] [ASTReader] Fix readability-inconsistent-declaration-parameter-name. NFC --- clang/include/clang/Serialization/ASTReader.h | 6 +-- clang/lib/Serialization/ASTReader.cpp | 38 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 9bb89ec941091..59358e77edb07 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -1424,7 +1424,7 @@ class ASTReader RecordLocation TypeCursorForIndex(unsigned Index); void LoadedDecl(unsigned Index, Decl *D); Decl *ReadDeclRecord(serialization::DeclID ID); - void markIncompleteDeclChain(Decl *Canon); + void markIncompleteDeclChain(Decl *D); /// Returns the most recent declaration of a declaration (which must be /// of a redeclarable kind) that is either local or has already been loaded @@ -2093,7 +2093,7 @@ class ASTReader SmallVectorImpl> &Sels) override; void ReadWeakUndeclaredIdentifiers( - SmallVectorImpl> &WI) override; + SmallVectorImpl> &WeakIDs) override; void ReadUsedVTables(SmallVectorImpl &VTables) override; @@ -2203,7 +2203,7 @@ class ASTReader /// Retrieve the global selector ID that corresponds to this /// the local selector ID in a given module. - serialization::SelectorID getGlobalSelectorID(ModuleFile &F, + serialization::SelectorID getGlobalSelectorID(ModuleFile &M, unsigned LocalID) const; /// Read the contents of a CXXCtorInitializer array. diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 5b51ac40000d7..9effd333daccd 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1781,26 +1781,26 @@ llvm::Error ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, } } -Token ASTReader::ReadToken(ModuleFile &F, const RecordDataImpl &Record, +Token ASTReader::ReadToken(ModuleFile &M, const RecordDataImpl &Record, unsigned &Idx) { Token Tok; Tok.startToken(); - Tok.setLocation(ReadSourceLocation(F, Record, Idx)); + Tok.setLocation(ReadSourceLocation(M, Record, Idx)); Tok.setKind((tok::TokenKind)Record[Idx++]); Tok.setFlag((Token::TokenFlags)Record[Idx++]); if (Tok.isAnnotation()) { - Tok.setAnnotationEndLoc(ReadSourceLocation(F, Record, Idx)); + Tok.setAnnotationEndLoc(ReadSourceLocation(M, Record, Idx)); switch (Tok.getKind()) { case tok::annot_pragma_loop_hint: { auto *Info = new (PP.getPreprocessorAllocator()) PragmaLoopHintInfo; - Info->PragmaName = ReadToken(F, Record, Idx); - Info->Option = ReadToken(F, Record, Idx); + Info->PragmaName = ReadToken(M, Record, Idx); + Info->Option = ReadToken(M, Record, Idx); unsigned NumTokens = Record[Idx++]; SmallVector Toks; Toks.reserve(NumTokens); for (unsigned I = 0; I < NumTokens; ++I) - Toks.push_back(ReadToken(F, Record, Idx)); + Toks.push_back(ReadToken(M, Record, Idx)); Info->Toks = llvm::ArrayRef(Toks).copy(PP.getPreprocessorAllocator()); Tok.setAnnotationValue(static_cast(Info)); break; @@ -1811,7 +1811,7 @@ Token ASTReader::ReadToken(ModuleFile &F, const RecordDataImpl &Record, auto SlotLabel = ReadString(Record, Idx); Info->SlotLabel = llvm::StringRef(SlotLabel).copy(PP.getPreprocessorAllocator()); - Info->Alignment = ReadToken(F, Record, Idx); + Info->Alignment = ReadToken(M, Record, Idx); Tok.setAnnotationValue(static_cast(Info)); break; } @@ -1827,7 +1827,7 @@ Token ASTReader::ReadToken(ModuleFile &F, const RecordDataImpl &Record, } } else { Tok.setLength(Record[Idx++]); - if (IdentifierInfo *II = getLocalIdentifier(F, Record[Idx++])) + if (IdentifierInfo *II = getLocalIdentifier(M, Record[Idx++])) Tok.setIdentifierInfo(II); } return Tok; @@ -1997,10 +1997,10 @@ unsigned HeaderFileInfoTrait::ComputeHash(internal_key_ref ikey) { } HeaderFileInfoTrait::internal_key_type -HeaderFileInfoTrait::GetInternalKey(external_key_type FE) { - internal_key_type ikey = {FE.getSize(), - M.HasTimestamps ? FE.getModificationTime() : 0, - FE.getName(), /*Imported*/ false}; +HeaderFileInfoTrait::GetInternalKey(external_key_type ekey) { + internal_key_type ikey = {ekey.getSize(), + M.HasTimestamps ? ekey.getModificationTime() : 0, + ekey.getName(), /*Imported*/ false}; return ikey; } @@ -8946,10 +8946,10 @@ Module *ASTReader::getModule(unsigned ID) { return getSubmodule(ID); } -ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &F, unsigned ID) { +ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &M, unsigned ID) { if (ID & 1) { // It's a module, look it up by submodule ID. - auto I = GlobalSubmoduleMap.find(getGlobalSubmoduleID(F, ID >> 1)); + auto I = GlobalSubmoduleMap.find(getGlobalSubmoduleID(M, ID >> 1)); return I == GlobalSubmoduleMap.end() ? nullptr : I->second; } else { // It's a prefix (preamble, PCH, ...). Look it up by index. @@ -8959,19 +8959,19 @@ ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &F, unsigned ID) { } } -unsigned ASTReader::getModuleFileID(ModuleFile *F) { - if (!F) +unsigned ASTReader::getModuleFileID(ModuleFile *M) { + if (!M) return 1; // For a file representing a module, use the submodule ID of the top-level // module as the file ID. For any other kind of file, the number of such // files loaded beforehand will be the same on reload. // FIXME: Is this true even if we have an explicit module file and a PCH? - if (F->isModule()) - return ((F->BaseSubmoduleID + NUM_PREDEF_SUBMODULE_IDS) << 1) | 1; + if (M->isModule()) + return ((M->BaseSubmoduleID + NUM_PREDEF_SUBMODULE_IDS) << 1) | 1; auto PCHModules = getModuleManager().pch_modules(); - auto I = llvm::find(PCHModules, F); + auto I = llvm::find(PCHModules, M); assert(I != PCHModules.end() && "emitting reference to unknown file"); return (I - PCHModules.end()) << 1; } From 7e4ae28645ef3ad36f1daec3fe3edc901f561bc7 Mon Sep 17 00:00:00 2001 From: Mariusz Borsa Date: Fri, 15 Dec 2023 17:33:54 -0800 Subject: [PATCH 028/884] [Sanitizers] Don't inline unpoisoning of small stacks when inlining disabled (#75555) When ASan.MaxInlinePoisoningSize == 0 , it means that no shadow memory operations should be made via inlined instrumentation code, but only via calls to shadow setting functions. This change fixes one violation of this, which happened when the function allocas count was small, i.e. less than 5 - in the code modifying the shadow just before ret instruction. We now explicitly check ASan.MaxInlinePoisoningSize , and if it's 0 then we disallow inlining. It is required for the instrumentation emitting code suitable for handling by ABI implementation. rdar://119513720 Co-authored-by: Mariusz Borsa --- .../Instrumentation/AddressSanitizer.cpp | 2 +- .../AddressSanitizer/calls-only-smallfn.ll | 28 +++++++++++++++++++ .../AddressSanitizer/calls-only.ll | 4 +-- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index b175e6f93f3e8..6468d07b4f4f4 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -3505,7 +3505,7 @@ void FunctionStackPoisoner::processStaticAllocas() { SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm); IRBuilder<> IRBPoison(ThenTerm); - if (StackMallocIdx <= 4) { + if (ASan.MaxInlinePoisoningSize != 0 && StackMallocIdx <= 4) { int ClassSize = kMinStackMallocSize << StackMallocIdx; ShadowAfterReturn.resize(ClassSize / L.Granularity, kAsanStackUseAfterReturnMagic); diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll new file mode 100644 index 0000000000000..3d67778049430 --- /dev/null +++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll @@ -0,0 +1,28 @@ +; RUN: opt < %s -passes=asan -asan-max-inline-poisoning-size=0 -asan-stack-dynamic-alloca=0 -S | FileCheck --check-prefix=OUTLINE %s +; RUN: opt < %s -passes=asan -asan-max-inline-poisoning-size=999 -asan-stack-dynamic-alloca=0 -S | FileCheck --check-prefix=INLINE %s + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx13.0.0" + +; Function Attrs: noinline nounwind optnone sanitize_address ssp uwtable(sync) +define void @foo() #0 { +entry: + %array01 = alloca [1 x i8], align 1 + %array02 = alloca [2 x i8], align 1 +; OUTLINE: call void @__asan_set_shadow_f1(i64 %23, i64 4) +; OUTLINE: call void @__asan_set_shadow_01(i64 %24, i64 1) +; OUTLINE: call void @__asan_set_shadow_f2(i64 %25, i64 1) +; OUTLINE: call void @__asan_set_shadow_02(i64 %26, i64 1) +; OUTLINE: call void @__asan_set_shadow_f3(i64 %27, i64 1) +; OUTLINE: call void @__asan_stack_free_0(i64 %7, i64 64) +; OUTLINE: call void @__asan_set_shadow_00(i64 %55, i64 8) +; INLINE: store i64 -935919682371587599, ptr %24, align 1 +; INLINE: store i64 -723401728380766731, ptr %52, align 1 + %arrayidx = getelementptr inbounds [1 x i8], ptr %array01, i64 0, i64 1 + store i8 1, ptr %arrayidx, align 1 + %arrayidx1 = getelementptr inbounds [2 x i8], ptr %array02, i64 0, i64 2 + store i8 2, ptr %arrayidx1, align 1 + ret void +} +attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+zcm,+zcz" } + diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll index 2cf0070cf0862..fa491105e017d 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll @@ -29,8 +29,8 @@ entry: ; OUTLINE: call void @__asan_set_shadow_f2(i64 %45, i64 3) ; OUTLINE: call void @__asan_set_shadow_07(i64 %46, i64 1) ; OUTLINE: call void @__asan_set_shadow_f3(i64 %47, i64 3) -; OUTLINE: call void @__asan_set_shadow_f5(i64 %134, i64 32) -; OUTLINE: call void @__asan_set_shadow_00(i64 %140, i64 24) +; OUTLINE: call void @__asan_stack_free_2(i64 %7, i64 192) +; OUTLINE: call void @__asan_set_shadow_00(i64 %135, i64 24) ; INLINE: store i64 -1007977276409515535, ptr %34, align 1 ; INLINE: store i64 -940423264817843709, ptr %36, align 1 ; INLINE: store i64 -868083087686045178, ptr %38, align 1 From 433498ce4619d0269977ad6dd273c584301d546f Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 15 Dec 2023 20:52:37 -0600 Subject: [PATCH 029/884] [llvm-readobj] Print the associated CUDA SM flags (#75664) Summary: The architecture that the CUDA application was compiled for is stored in the ELF flags. This patch just adds some simple enum values to indicate this to the user in a readable way. --- llvm/tools/llvm-readobj/ELFDumper.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index 3cf7c5a3b1895..f2851a52671bc 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1693,6 +1693,19 @@ const EnumEntry ElfHeaderAMDGPUFlagsABIVersion4[] = { LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_ON_V4), }; +const EnumEntry ElfHeaderNVPTXFlags[] = { + ENUM_ENT(EF_CUDA_SM20, "sm_20"), ENUM_ENT(EF_CUDA_SM21, "sm_21"), + ENUM_ENT(EF_CUDA_SM30, "sm_30"), ENUM_ENT(EF_CUDA_SM32, "sm_32"), + ENUM_ENT(EF_CUDA_SM35, "sm_35"), ENUM_ENT(EF_CUDA_SM37, "sm_37"), + ENUM_ENT(EF_CUDA_SM50, "sm_50"), ENUM_ENT(EF_CUDA_SM52, "sm_52"), + ENUM_ENT(EF_CUDA_SM53, "sm_53"), ENUM_ENT(EF_CUDA_SM60, "sm_60"), + ENUM_ENT(EF_CUDA_SM61, "sm_61"), ENUM_ENT(EF_CUDA_SM62, "sm_62"), + ENUM_ENT(EF_CUDA_SM70, "sm_70"), ENUM_ENT(EF_CUDA_SM72, "sm_72"), + ENUM_ENT(EF_CUDA_SM75, "sm_75"), ENUM_ENT(EF_CUDA_SM80, "sm_80"), + ENUM_ENT(EF_CUDA_SM86, "sm_86"), ENUM_ENT(EF_CUDA_SM87, "sm_87"), + ENUM_ENT(EF_CUDA_SM89, "sm_89"), ENUM_ENT(EF_CUDA_SM90, "sm_90"), +}; + const EnumEntry ElfHeaderRISCVFlags[] = { ENUM_ENT(EF_RISCV_RVC, "RVC"), ENUM_ENT(EF_RISCV_FLOAT_ABI_SINGLE, "single-float ABI"), @@ -3630,6 +3643,9 @@ template void GNUELFDumper::printFileHeaders() { else if (e.e_machine == EM_XTENSA) ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderXtensaFlags), unsigned(ELF::EF_XTENSA_MACH)); + else if (e.e_machine == EM_CUDA) + ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderNVPTXFlags), + unsigned(ELF::EF_CUDA_SM)); Str = "0x" + utohexstr(e.e_flags); if (!ElfFlags.empty()) Str = Str + ", " + ElfFlags; @@ -6912,6 +6928,9 @@ template void LLVMELFDumper::printFileHeaders() { else if (E.e_machine == EM_XTENSA) W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderXtensaFlags), unsigned(ELF::EF_XTENSA_MACH)); + else if (E.e_machine == EM_CUDA) + W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderNVPTXFlags), + unsigned(ELF::EF_CUDA_SM)); else W.printFlags("Flags", E.e_flags); W.printNumber("HeaderSize", E.e_ehsize); From 5545b2545207ae0824636665ea976e8631772c99 Mon Sep 17 00:00:00 2001 From: Yeting Kuo <46629943+yetingk@users.noreply.github.com> Date: Sat, 16 Dec 2023 11:22:07 +0800 Subject: [PATCH 030/884] [RISCV] Make Zfh imply Zfhmin. (#75576) According to spec, the Zfhmin extension is a subset of the Zfh extension. --- llvm/lib/Support/RISCVISAInfo.cpp | 2 +- llvm/test/CodeGen/RISCV/attributes.ll | 8 ++++---- llvm/test/MC/RISCV/attribute-arch.s | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index aad7ab8361af9..bbbaf26a7bd49 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -1010,7 +1010,7 @@ static const char *ImpliedExtsZcmt[] = {"zca"}; static const char *ImpliedExtsZdinx[] = {"zfinx"}; static const char *ImpliedExtsZfa[] = {"f"}; static const char *ImpliedExtsZfbfmin[] = {"f"}; -static const char *ImpliedExtsZfh[] = {"f"}; +static const char *ImpliedExtsZfh[] = {"zfhmin"}; static const char *ImpliedExtsZfhmin[] = {"f"}; static const char *ImpliedExtsZfinx[] = {"zicsr"}; static const char *ImpliedExtsZhinx[] = {"zfinx"}; diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index b34ccb3ff0f93..b3d4dc8bb638a 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -193,14 +193,14 @@ ; RV32ZIHINTPAUSE: .attribute 5, "rv32i2p1_zihintpause2p0" ; RV32ZIHINTNTL: .attribute 5, "rv32i2p1_zihintntl1p0" ; RV32ZFHMIN: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfhmin1p0" -; RV32ZFH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfh1p0" +; RV32ZFH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfh1p0_zfhmin1p0" ; RV32ZBA: .attribute 5, "rv32i2p1_zba1p0" ; RV32ZBB: .attribute 5, "rv32i2p1_zbb1p0" ; RV32ZBC: .attribute 5, "rv32i2p1_zbc1p0" ; RV32ZBS: .attribute 5, "rv32i2p1_zbs1p0" ; RV32V: .attribute 5, "rv32i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" ; RV32H: .attribute 5, "rv32i2p1_h1p0" -; RV32COMBINED: .attribute 5, "rv32i2p1_f2p2_d2p2_v1p0_zicsr2p0_zfh1p0_zbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" +; RV32COMBINED: .attribute 5, "rv32i2p1_f2p2_d2p2_v1p0_zicsr2p0_zfh1p0_zfhmin1p0_zbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" ; RV32ZBKB: .attribute 5, "rv32i2p1_zbkb1p0" ; RV32ZBKC: .attribute 5, "rv32i2p1_zbkc1p0" ; RV32ZBKX: .attribute 5, "rv32i2p1_zbkx1p0" @@ -283,14 +283,14 @@ ; RV64ZIHINTPAUSE: .attribute 5, "rv64i2p1_zihintpause2p0" ; RV64ZIHINTNTL: .attribute 5, "rv64i2p1_zihintntl1p0" ; RV64ZFHMIN: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfhmin1p0" -; RV64ZFH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfh1p0" +; RV64ZFH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfh1p0_zfhmin1p0" ; RV64ZBA: .attribute 5, "rv64i2p1_zba1p0" ; RV64ZBB: .attribute 5, "rv64i2p1_zbb1p0" ; RV64ZBC: .attribute 5, "rv64i2p1_zbc1p0" ; RV64ZBS: .attribute 5, "rv64i2p1_zbs1p0" ; RV64V: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" ; RV64H: .attribute 5, "rv64i2p1_h1p0" -; RV64COMBINED: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zfh1p0_zbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" +; RV64COMBINED: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zfh1p0_zfhmin1p0_zbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0" ; RV64ZBKB: .attribute 5, "rv64i2p1_zbkb1p0" ; RV64ZBKC: .attribute 5, "rv64i2p1_zbkc1p0" ; RV64ZBKX: .attribute 5, "rv64i2p1_zbkx1p0" diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s index 3ed48401e43fc..25f84f3cc1232 100644 --- a/llvm/test/MC/RISCV/attribute-arch.s +++ b/llvm/test/MC/RISCV/attribute-arch.s @@ -166,7 +166,7 @@ # CHECK: attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfhmin1p0" .attribute arch, "rv32ifzfh1p0" -# CHECK: attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfh1p0" +# CHECK: attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfh1p0_zfhmin1p0" .attribute arch, "rv32izfinx" # CHECK: attribute 5, "rv32i2p1_zicsr2p0_zfinx1p0" From 7a0fd97ac1094b9b1547c8d7b35e583d7387224d Mon Sep 17 00:00:00 2001 From: smanna12 Date: Fri, 15 Dec 2023 22:09:19 -0600 Subject: [PATCH 031/884] [NFC][CLANG] Rename duplicate loop attributes diagnostic functions (#75657) This patch renames CheckForDuplicateCodeAlignAttrs() to CheckForDuplicateLoopAttrs() and corresponding other functions that call it to be used for other statement attributes in future. --- clang/include/clang/Sema/Sema.h | 2 +- clang/lib/Sema/SemaStmtAttr.cpp | 21 ++++++++++----------- clang/lib/Sema/TreeTransform.h | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 1d7b4c729ce84..20228da15ade8 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -2102,7 +2102,7 @@ class Sema final { SourceLocation AttrLoc); CodeAlignAttr *BuildCodeAlignAttr(const AttributeCommonInfo &CI, Expr *E); - bool CheckRebuiltCodeAlignStmtAttributes(ArrayRef Attrs); + bool CheckRebuiltStmtAttributes(ArrayRef Attrs); bool CheckQualifiedFunctionForTypeId(QualType T, SourceLocation Loc); diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp index 0d0a7bcebab4e..e6a4d3e63e4aa 100644 --- a/clang/lib/Sema/SemaStmtAttr.cpp +++ b/clang/lib/Sema/SemaStmtAttr.cpp @@ -361,11 +361,10 @@ static Attr *handleCodeAlignAttr(Sema &S, Stmt *St, const ParsedAttr &A) { } // Diagnose non-identical duplicates as a 'conflicting' loop attributes -// and suppress duplicate errors in cases where the two match for -// [[clang::code_align()]] attribute. -static void CheckForDuplicateCodeAlignAttrs(Sema &S, - ArrayRef Attrs) { - auto FindFunc = [](const Attr *A) { return isa(A); }; +// and suppress duplicate errors in cases where the two match. +template +static void CheckForDuplicateLoopAttrs(Sema &S, ArrayRef Attrs) { + auto FindFunc = [](const Attr *A) { return isa(A); }; const auto *FirstItr = std::find_if(Attrs.begin(), Attrs.end(), FindFunc); if (FirstItr == Attrs.end()) // no attributes found @@ -375,7 +374,7 @@ static void CheckForDuplicateCodeAlignAttrs(Sema &S, std::optional FirstValue; const auto *CAFA = - dyn_cast(cast(*FirstItr)->getAlignment()); + dyn_cast(cast(*FirstItr)->getAlignment()); // Return early if first alignment expression is dependent (since we don't // know what the effective size will be), and skip the loop entirely. if (!CAFA) @@ -383,8 +382,8 @@ static void CheckForDuplicateCodeAlignAttrs(Sema &S, while (Attrs.end() != (LastFoundItr = std::find_if(LastFoundItr + 1, Attrs.end(), FindFunc))) { - const auto *CASA = dyn_cast( - cast(*LastFoundItr)->getAlignment()); + const auto *CASA = + dyn_cast(cast(*LastFoundItr)->getAlignment()); // If the value is dependent, we can not test anything. if (!CASA) return; @@ -635,10 +634,10 @@ void Sema::ProcessStmtAttributes(Stmt *S, const ParsedAttributes &InAttrs, } CheckForIncompatibleAttributes(*this, OutAttrs); - CheckForDuplicateCodeAlignAttrs(*this, OutAttrs); + CheckForDuplicateLoopAttrs(*this, OutAttrs); } -bool Sema::CheckRebuiltCodeAlignStmtAttributes(ArrayRef Attrs) { - CheckForDuplicateCodeAlignAttrs(*this, Attrs); +bool Sema::CheckRebuiltStmtAttributes(ArrayRef Attrs) { + CheckForDuplicateLoopAttrs(*this, Attrs); return false; } diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 1ad843d0bf4e0..7df5bf0cb7137 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -1378,7 +1378,7 @@ class TreeTransform { StmtResult RebuildAttributedStmt(SourceLocation AttrLoc, ArrayRef Attrs, Stmt *SubStmt) { - if (SemaRef.CheckRebuiltCodeAlignStmtAttributes(Attrs)) + if (SemaRef.CheckRebuiltStmtAttributes(Attrs)) return StmtError(); return SemaRef.BuildAttributedStmt(AttrLoc, Attrs, SubStmt); } From 4faeb7dbe9a4d35ea3556a319a814fe7c5d6c27c Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 15 Dec 2023 21:30:29 -0800 Subject: [PATCH 032/884] [InstCombine] Add test for missed opportunity to fold 'or' into 'mul' operand. NFC We are able to fold or (mul X, Y), X --> mul X, (add Y, 1) (when the multiply has no common bits with X) but we miss it if the mul operands are commuted. --- llvm/test/Transforms/InstCombine/or.ll | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/or.ll b/llvm/test/Transforms/InstCombine/or.ll index 8c8aab2bcba6a..805546099398b 100644 --- a/llvm/test/Transforms/InstCombine/or.ll +++ b/llvm/test/Transforms/InstCombine/or.ll @@ -1511,6 +1511,21 @@ define <2 x i12> @mul_no_common_bits_commute(<2 x i12> %p) { ret <2 x i12> %r } +define i32 @mul_no_common_bits_commute2(i32 %p1, i32 %p2) { +; CHECK-LABEL: @mul_no_common_bits_commute2( +; CHECK-NEXT: [[X:%.*]] = and i32 [[P1:%.*]], 7 +; CHECK-NEXT: [[Y:%.*]] = shl i32 [[P2:%.*]], 3 +; CHECK-NEXT: [[M:%.*]] = mul i32 [[Y]], [[X]] +; CHECK-NEXT: [[R:%.*]] = or disjoint i32 [[M]], [[X]] +; CHECK-NEXT: ret i32 [[R]] +; + %x = and i32 %p1, 7 + %y = shl i32 %p2, 3 + %m = mul i32 %y, %x + %r = or i32 %m, %x + ret i32 %r +} + define i32 @mul_no_common_bits_disjoint(i32 %x, i32 %y) { ; CHECK-LABEL: @mul_no_common_bits_disjoint( ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y:%.*]], 1 From c59ea32f82128f550b471ed96b7ac093ff448c60 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 16 Dec 2023 17:58:57 +0800 Subject: [PATCH 033/884] [InstCombine] Canonicalize `icmp pred (X +/- C1), C2` into `icmp pred X, C2 -/+ C1` with nowrap flag implied by with.overflow intrinsic (#75511) This patch tries to canonicalize the pattern `Overflow | icmp pred Res, C2` into `Overflow | icmp pred X, C2 +/- C1`, where `Overflow` and `Res` are return values of `xxx.with.overflow X, C1`. Alive2: https://alive2.llvm.org/ce/z/PhR_3S Fixes #75360. --- .../InstCombine/InstCombineAndOrXor.cpp | 29 ++ .../canonicalize-or-with-overflow-icmp.ll | 275 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-or-with-overflow-icmp.ll diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index 31db1d3164b77..5e362f4117d05 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -3756,6 +3756,35 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) { } } + /// Res, Overflow = xxx_with_overflow X, C1 + /// Try to canonicalize the pattern "Overflow | icmp pred Res, C2" into + /// "Overflow | icmp pred X, C2 +/- C1". + const WithOverflowInst *WO; + const Value *WOV; + const APInt *C1, *C2; + if (match(&I, m_c_Or(m_CombineAnd(m_ExtractValue<1>(m_CombineAnd( + m_WithOverflowInst(WO), m_Value(WOV))), + m_Value(Ov)), + m_OneUse(m_ICmp(Pred, m_ExtractValue<0>(m_Deferred(WOV)), + m_APInt(C2))))) && + (WO->getBinaryOp() == Instruction::Add || + WO->getBinaryOp() == Instruction::Sub) && + (ICmpInst::isEquality(Pred) || + WO->isSigned() == ICmpInst::isSigned(Pred)) && + match(WO->getRHS(), m_APInt(C1))) { + bool Overflow; + APInt NewC = WO->getBinaryOp() == Instruction::Add + ? (ICmpInst::isSigned(Pred) ? C2->ssub_ov(*C1, Overflow) + : C2->usub_ov(*C1, Overflow)) + : (ICmpInst::isSigned(Pred) ? C2->sadd_ov(*C1, Overflow) + : C2->uadd_ov(*C1, Overflow)); + if (!Overflow || ICmpInst::isEquality(Pred)) { + Value *NewCmp = Builder.CreateICmp( + Pred, WO->getLHS(), ConstantInt::get(WO->getLHS()->getType(), NewC)); + return BinaryOperator::CreateOr(Ov, NewCmp); + } + } + // (~x) | y --> ~(x & (~y)) iff that gets rid of inversions if (sinkNotIntoOtherHandOfLogicalOp(I)) return &I; diff --git a/llvm/test/Transforms/InstCombine/canonicalize-or-with-overflow-icmp.ll b/llvm/test/Transforms/InstCombine/canonicalize-or-with-overflow-icmp.ll new file mode 100644 index 0000000000000..2e801489ef4f1 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/canonicalize-or-with-overflow-icmp.ll @@ -0,0 +1,275 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) +declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32) +declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) +declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) + +declare void @use(i1) + +; Tests from PR75360 +define i1 @ckd_add_unsigned(i31 %num) { +; CHECK-LABEL: define i1 @ckd_add_unsigned( +; CHECK-SAME: i31 [[NUM:%.*]]) { +; CHECK-NEXT: [[A2:%.*]] = icmp eq i31 [[NUM]], -1 +; CHECK-NEXT: ret i1 [[A2]] +; + %a0 = zext i31 %num to i32 + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @ckd_add_unsigned_commuted(i31 %num) { +; CHECK-LABEL: define i1 @ckd_add_unsigned_commuted( +; CHECK-SAME: i31 [[NUM:%.*]]) { +; CHECK-NEXT: [[A2:%.*]] = icmp eq i31 [[NUM]], -1 +; CHECK-NEXT: ret i1 [[A2]] +; + %a0 = zext i31 %num to i32 + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + %a5 = or i1 %a4, %a2 + ret i1 %a5 +} + +define i1 @ckd_add_unsigned_imply_true(i31 %num) { +; CHECK-LABEL: define i1 @ckd_add_unsigned_imply_true( +; CHECK-SAME: i31 [[NUM:%.*]]) { +; CHECK-NEXT: ret i1 true +; + %a0 = zext i31 %num to i32 + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp sgt i32 %a3, -1 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[A0]], -2147483647 +; CHECK-NEXT: [[A5:%.*]] = icmp sgt i32 [[TMP1]], -1 +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_ssub_with_overflow_icmp(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_ssub_with_overflow_icmp( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[A0]], 1 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %a1 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_uadd_with_overflow_icmp(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_uadd_with_overflow_icmp( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[A0]], 1 +; CHECK-NEXT: [[A5:%.*]] = icmp ult i32 [[TMP1]], 10 +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp ult i32 %a3, 10 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp_eq(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_eq( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A2:%.*]] = icmp eq i32 [[A0]], 2147483647 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[A0]], 9 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[TMP1]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp eq i32 %a3, 10 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_uadd_with_overflow_icmp_ne(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_uadd_with_overflow_icmp_ne( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[A0]], 9 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %a1 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp ne i32 %a3, 10 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +; Negative tests +define i1 @canonicalize_or_sadd_with_overflow_icmp_mismatched_pred(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_mismatched_pred( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 1) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp ult i32 [[A3]], 2 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp ult i32 %a3, 2 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp_non_constant1(i32 %a0, i32 %c) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_non_constant1( +; CHECK-SAME: i32 [[A0:%.*]], i32 [[C:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 [[C]]) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], 0 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 %c) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp_non_constant2(i32 %a0, i32 %c) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_non_constant2( +; CHECK-SAME: i32 [[A0:%.*]], i32 [[C:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 1) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], [[C]] +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, %c + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp_multiuse(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_multiuse( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 1) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], 0 +; CHECK-NEXT: call void @use(i1 [[A4]]) +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 1) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 0 + call void @use(i1 %a4) + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_sadd_with_overflow_icmp_overflow(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_sadd_with_overflow_icmp_overflow( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 -2147483647) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], 2 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a0, i32 -2147483647) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 2 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_uadd_with_overflow_icmp_overflow(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_uadd_with_overflow_icmp_overflow( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A0]], i32 3) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp ult i32 [[A3]], 2 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a0, i32 3) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp ult i32 %a3, 2 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_ssub_with_overflow_icmp_overflow(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_ssub_with_overflow_icmp_overflow( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A0]], i32 -2147483648) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], -1 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a0, i32 -2147483648) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, -1 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} + +define i1 @canonicalize_or_smul_with_overflow_icmp(i32 %a0) { +; CHECK-LABEL: define i1 @canonicalize_or_smul_with_overflow_icmp( +; CHECK-SAME: i32 [[A0:%.*]]) { +; CHECK-NEXT: [[A1:%.*]] = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A0]], i32 3) +; CHECK-NEXT: [[A2:%.*]] = extractvalue { i32, i1 } [[A1]], 1 +; CHECK-NEXT: [[A3:%.*]] = extractvalue { i32, i1 } [[A1]], 0 +; CHECK-NEXT: [[A4:%.*]] = icmp slt i32 [[A3]], 10 +; CHECK-NEXT: [[A5:%.*]] = or i1 [[A2]], [[A4]] +; CHECK-NEXT: ret i1 [[A5]] +; + %a1 = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %a0, i32 3) + %a2 = extractvalue { i32, i1 } %a1, 1 + %a3 = extractvalue { i32, i1 } %a1, 0 + %a4 = icmp slt i32 %a3, 10 + %a5 = or i1 %a2, %a4 + ret i1 %a5 +} From 76041a45bbe3cd2b3b3acad46267f27815e6a652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Sat, 16 Dec 2023 11:13:20 +0000 Subject: [PATCH 034/884] [flang][nfc] Refactor linker invocation logic (#75648) Refactor how the Fortran runtime libs are added to the linker invocation. This is a non-functional change. This is an updated version of #75534. This iteration makes sure that FortranMain.a comes before FortranRuntme.a (the former depends on the latter). --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 145 ++++++++++++--------- 1 file changed, 82 insertions(+), 63 deletions(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 3d1df58190ce0..45901ee7157f7 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1116,72 +1116,91 @@ bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC, return true; } -void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args, - llvm::opt::ArgStringList &CmdArgs) { - // These are handled earlier on Windows by telling the frontend driver to add - // the correct libraries to link against as dependents in the object file. - - // if -fno-fortran-main has been passed, skip linking Fortran_main.a - bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); - if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { - if (LinkFortranMain) { - // The --whole-archive option needs to be part of the link line to - // make sure that the main() function from Fortran_main.a is pulled - // in by the linker. Determine if --whole-archive is active when - // flang will try to link Fortran_main.a. If it is, don't add the - // --whole-archive flag to the link line. If it's not, add a proper - // --whole-archive/--no-whole-archive bracket to the link line. - bool WholeArchiveActive = false; - for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) { - if (Arg) { - for (StringRef ArgValue : Arg->getValues()) { - if (ArgValue == "--whole-archive") - WholeArchiveActive = true; - if (ArgValue == "--no-whole-archive") - WholeArchiveActive = false; - } - } +/// Determines if --whole-archive is active in the list of arguments. +static bool isWholeArchivePresent(const ArgList &Args) { + bool WholeArchiveActive = false; + for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) { + if (Arg) { + for (StringRef ArgValue : Arg->getValues()) { + if (ArgValue == "--whole-archive") + WholeArchiveActive = true; + if (ArgValue == "--no-whole-archive") + WholeArchiveActive = false; } + } + } - // TODO: Find an equivalent of `--whole-archive` for Darwin. - if (!WholeArchiveActive && !TC.getTriple().isMacOSX()) { - CmdArgs.push_back("--whole-archive"); - CmdArgs.push_back("-lFortran_main"); - CmdArgs.push_back("--no-whole-archive"); - } else { - CmdArgs.push_back("-lFortran_main"); - } + return WholeArchiveActive; +} - // Perform regular linkage of the remaining runtime libraries. - CmdArgs.push_back("-lFortranRuntime"); - CmdArgs.push_back("-lFortranDecimal"); - } - } else { - if (LinkFortranMain) { - unsigned RTOptionID = options::OPT__SLASH_MT; - if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { - RTOptionID = llvm::StringSwitch(rtl->getValue()) - .Case("static", options::OPT__SLASH_MT) - .Case("static_dbg", options::OPT__SLASH_MTd) - .Case("dll", options::OPT__SLASH_MD) - .Case("dll_dbg", options::OPT__SLASH_MDd) - .Default(options::OPT__SLASH_MT); - } - switch (RTOptionID) { - case options::OPT__SLASH_MT: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); - break; - case options::OPT__SLASH_MTd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); - break; - case options::OPT__SLASH_MD: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); - break; - case options::OPT__SLASH_MDd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); - break; - } - } +/// Add Fortran runtime libs for MSVC +static void addFortranRuntimeLibsMSVC(const ArgList &Args, + llvm::opt::ArgStringList &CmdArgs) { + unsigned RTOptionID = options::OPT__SLASH_MT; + if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { + RTOptionID = llvm::StringSwitch(rtl->getValue()) + .Case("static", options::OPT__SLASH_MT) + .Case("static_dbg", options::OPT__SLASH_MTd) + .Case("dll", options::OPT__SLASH_MD) + .Case("dll_dbg", options::OPT__SLASH_MDd) + .Default(options::OPT__SLASH_MT); + } + switch (RTOptionID) { + case options::OPT__SLASH_MT: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); + break; + case options::OPT__SLASH_MTd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); + break; + case options::OPT__SLASH_MD: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); + break; + case options::OPT__SLASH_MDd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); + break; + } +} + +// Add FortranMain runtime lib +static void addFortranMain(const ToolChain &TC, const ArgList &Args, + llvm::opt::ArgStringList &CmdArgs) { + // 1. MSVC + if (TC.getTriple().isKnownWindowsMSVCEnvironment()) { + addFortranRuntimeLibsMSVC(Args, CmdArgs); + return; + } + + // 2. GNU and similar + // The --whole-archive option needs to be part of the link line to make + // sure that the main() function from Fortran_main.a is pulled in by the + // linker. However, it shouldn't be used if it's already active. + // TODO: Find an equivalent of `--whole-archive` for Darwin. + if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX()) { + CmdArgs.push_back("--whole-archive"); + CmdArgs.push_back("-lFortran_main"); + CmdArgs.push_back("--no-whole-archive"); + return; + } + + CmdArgs.push_back("-lFortran_main"); +} + +/// Add Fortran runtime libs +void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args, + llvm::opt::ArgStringList &CmdArgs) { + // 1. Link FortranMain + // FortranMain depends on FortranRuntime, so needs to be listed first. If + // -fno-fortran-main has been passed, skip linking Fortran_main.a + if (!Args.hasArg(options::OPT_no_fortran_main)) + addFortranMain(TC, Args, CmdArgs); + + // 2. Link FortranRuntime and FortranDecimal + // These are handled earlier on Windows by telling the frontend driver to + // add the correct libraries to link against as dependents in the object + // file. + if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { + CmdArgs.push_back("-lFortranRuntime"); + CmdArgs.push_back("-lFortranDecimal"); } } From 50f5b5a80bedee08fd4c46fcd171a1c85ee3834b Mon Sep 17 00:00:00 2001 From: Dinar Temirbulatov Date: Sat, 16 Dec 2023 12:03:54 +0000 Subject: [PATCH 035/884] [AArch64][SME2] Add FCLAMP, CNTP builtins for SME2 (#72487) This change enables FCLAMP, CNTP builtins for SME2 target. --- clang/include/clang/Basic/arm_sve.td | 7 +++---- .../CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c | 1 + .../CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 519438a9bc69b..55fd35c3b6c2d 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1970,8 +1970,6 @@ def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, } let TargetGuard = "sve2p1" in { -def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>; - def SVWHILEGE_COUNT : SInst<"svwhilege_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILELE_COUNT : SInst<"svwhilele_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; @@ -2071,8 +2069,6 @@ let TargetGuard = "sve2p1" in { def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; -def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone], [ImmCheck<1, ImmCheck2_4_Mul2>]>; - defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">; } @@ -2081,6 +2077,9 @@ let TargetGuard = "sve2p1|sme2" in { def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingCompatible], []>; def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; + + def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>; + def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>; } let TargetGuard = "sve2p1,b16b16" in { diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c index 18973a6467450..56b1d99262214 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c @@ -3,6 +3,7 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c index a9f482cab3969..5d8c5b7b8a18c 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c @@ -10,6 +10,8 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve \ +// RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include From e564d246b2f96d1994d952c4c6ceb578d4e78bdd Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Sat, 16 Dec 2023 15:17:28 +0200 Subject: [PATCH 036/884] [libc++][spaceship][NFC] Status page - added papers (#75043) Added C++20 papers to Spaceship status page Co-authored-by: Hristo Hristov --- libcxx/docs/Status/Cxx20Papers.csv | 4 ++-- libcxx/docs/Status/SpaceshipPapers.csv | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv index 13c126c1ba8be..d73088687975c 100644 --- a/libcxx/docs/Status/Cxx20Papers.csv +++ b/libcxx/docs/Status/Cxx20Papers.csv @@ -23,7 +23,7 @@ "`P0754R2 `__","LWG","","Jacksonville","|Complete|","7.0" "`P0809R0 `__","LWG","Comparing Unordered Containers","Jacksonville","|Nothing To Do|","" "`P0858R0 `__","LWG","Constexpr iterator requirements","Jacksonville","|Complete|","12.0" -"`P0905R1 `__","CWG","Symmetry for spaceship","Jacksonville","","" +"`P0905R1 `__","CWG","Symmetry for spaceship","Jacksonville","","","|spaceship|" "`P0966R1 `__","LWG","``string::reserve``\ Should Not Shrink","Jacksonville","|Complete| [#note-P0966]_","12.0" "","","","","","","" "`P0019R8 `__","LWG","Atomic Ref","Rapperswil","","" @@ -48,7 +48,7 @@ "`P0941R2 `__","CWG","Integrating feature-test macros into the C++ WD","Rapperswil","|In Progress|","" "`P1023R0 `__","LWG","constexpr comparison operators for std::array","Rapperswil","|Complete|","8.0" "`P1025R1 `__","CWG","Update The Reference To The Unicode Standard","Rapperswil","","" -"`P1120R0 `__","CWG","Consistency improvements for <=> and other comparison operators","Rapperswil","","" +"`P1120R0 `__","CWG","Consistency improvements for <=> and other comparison operators","Rapperswil","","","|spaceship|" "","","","","","","" "`P0318R1 `__","LWG","unwrap_ref_decay and unwrap_reference","San Diego","|Complete|","8.0" "`P0356R5 `__","LWG","Simplified partial function application","San Diego","|Complete|","13.0" diff --git a/libcxx/docs/Status/SpaceshipPapers.csv b/libcxx/docs/Status/SpaceshipPapers.csv index 26f10080af2b5..09fc67b04c91b 100644 --- a/libcxx/docs/Status/SpaceshipPapers.csv +++ b/libcxx/docs/Status/SpaceshipPapers.csv @@ -7,3 +7,5 @@ `LWG3360 `_,three_way_comparable_with is inconsistent with similar concepts,|Nothing To Do|, `LWG3380 `_,common_type and comparison categories,|Nothing To Do|, `LWG3395 `_,Definition for three-way comparison needs to be updated,|Nothing To Do|, +`P0905R1 `_,Symmetry for spaceship,, +`P1120R0 `_,Consistency improvements for <=> and other comparison operators,, From c398fa009a47eb24f88383d5e911e59e70f8db86 Mon Sep 17 00:00:00 2001 From: Stefan Pintilie Date: Sat, 16 Dec 2023 07:30:53 -0600 Subject: [PATCH 037/884] Revert "Reapply "RegisterCoalescer: Add implicit-def of super register when coalescing SUBREG_TO_REG"" This reverts commit f4b5be1ecdc85ca4257b739afb8d57e23c7a8030. The above change was breaking the clang-ppc64le-linux-test-suite bot. --- llvm/lib/CodeGen/RegisterCoalescer.cpp | 51 +-- .../AArch64/GlobalISel/arm64-pcsections.ll | 92 ++--- ...coalescer-breaks-subreg-to-reg-liveness.ll | 185 ---------- ...icit-def-regression-imp-operand-assert.mir | 4 +- ...subreg-to-reg-requires-subrange-update.mir | 47 --- .../CodeGen/X86/subreg-to-reg-coalescing.mir | 348 ------------------ 6 files changed, 57 insertions(+), 670 deletions(-) delete mode 100644 llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness.ll delete mode 100644 llvm/test/CodeGen/X86/coalescing-subreg-to-reg-requires-subrange-update.mir delete mode 100644 llvm/test/CodeGen/X86/subreg-to-reg-coalescing.mir diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp index 397fff5263426..c1af37c8510ff 100644 --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -305,11 +305,7 @@ namespace { /// number if it is not zero. If DstReg is a physical register and the /// existing subregister number of the def / use being updated is not zero, /// make sure to set it to the correct physical subregister. - /// - /// If \p IsSubregToReg, we are coalescing a DstReg = SUBREG_TO_REG - /// SrcReg. This introduces an implicit-def of DstReg on coalesced users. - void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx, - bool IsSubregToReg); + void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx); /// If the given machine operand reads only undefined lanes add an undef /// flag. @@ -1347,7 +1343,8 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP, if (DstReg.isPhysical()) { Register NewDstReg = DstReg; - unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), DefSubIdx); + unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), + DefMI->getOperand(0).getSubReg()); if (NewDstIdx) NewDstReg = TRI->getSubReg(DstReg, NewDstIdx); @@ -1496,7 +1493,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP, MRI->setRegClass(DstReg, NewRC); // Update machine operands and add flags. - updateRegDefsUses(DstReg, DstReg, DstIdx, false); + updateRegDefsUses(DstReg, DstReg, DstIdx); NewMI.getOperand(0).setSubReg(NewIdx); // updateRegDefUses can add an "undef" flag to the definition, since // it will replace DstReg with DstReg.DstIdx. If NewIdx is 0, make @@ -1817,7 +1814,7 @@ void RegisterCoalescer::addUndefFlag(const LiveInterval &Int, SlotIndex UseIdx, } void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg, - unsigned SubIdx, bool IsSubregToReg) { + unsigned SubIdx) { bool DstIsPhys = DstReg.isPhysical(); LiveInterval *DstInt = DstIsPhys ? nullptr : &LIS->getInterval(DstReg); @@ -1857,8 +1854,6 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg, if (DstInt && !Reads && SubIdx && !UseMI->isDebugInstr()) Reads = DstInt->liveAt(LIS->getInstructionIndex(*UseMI)); - bool FullDef = true; - // Replace SrcReg with DstReg in all UseMI operands. for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = UseMI->getOperand(Ops[i]); @@ -1866,13 +1861,9 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg, // Adjust flags in case of sub-register joins. We don't want to // turn a full def into a read-modify-write sub-register def and vice // versa. - if (SubIdx && MO.isDef()) { + if (SubIdx && MO.isDef()) MO.setIsUndef(!Reads); - if (!Reads) - FullDef = false; - } - // A subreg use of a partially undef (super) register may be a complete // undef use now and then has to be marked that way. if (MO.isUse() && !DstIsPhys) { @@ -1904,25 +1895,6 @@ void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg, MO.substVirtReg(DstReg, SubIdx, *TRI); } - if (IsSubregToReg && !FullDef) { - // If the coalesed instruction doesn't fully define the register, we need - // to preserve the original super register liveness for SUBREG_TO_REG. - // - // We pretended SUBREG_TO_REG was a regular copy for coalescing purposes, - // but it introduces liveness for other subregisters. Downstream users may - // have been relying on those bits, so we need to ensure their liveness is - // captured with a def of other lanes. - - // FIXME: Need to add new subrange if tracking subranges. We could also - // skip adding this if we knew the other lanes are dead, and only for - // other lanes. - - assert(!MRI->shouldTrackSubRegLiveness(DstReg) && - "this should update subranges"); - MachineInstrBuilder MIB(*MF, UseMI); - MIB.addReg(DstReg, RegState::ImplicitDefine); - } - LLVM_DEBUG({ dbgs() << "\t\tupdated: "; if (!UseMI->isDebugInstr()) @@ -2122,8 +2094,6 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) { }); } - const bool IsSubregToReg = CopyMI->isSubregToReg(); - ShrinkMask = LaneBitmask::getNone(); ShrinkMainRange = false; @@ -2191,12 +2161,9 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) { // Rewrite all SrcReg operands to DstReg. // Also update DstReg operands to include DstIdx if it is set. - if (CP.getDstIdx()) { - assert(!IsSubregToReg && "can this happen?"); - updateRegDefsUses(CP.getDstReg(), CP.getDstReg(), CP.getDstIdx(), false); - } - updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx(), - IsSubregToReg); + if (CP.getDstIdx()) + updateRegDefsUses(CP.getDstReg(), CP.getDstReg(), CP.getDstIdx()); + updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx()); // Shrink subregister ranges if necessary. if (ShrinkMask.any()) { diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll index 8529dd388ba0f..4c07081404c88 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll @@ -13,7 +13,7 @@ define i32 @val_compare_and_swap(ptr %p, i32 %cmp, i32 %new) { ; CHECK-NEXT: successors: %bb.2(0x7c000000), %bb.3(0x04000000) ; CHECK-NEXT: liveins: $w1, $w2, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s32) from %ir.p) + ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s32) from %ir.p) ; CHECK-NEXT: $wzr = SUBSWrs renamable $w8, renamable $w1, 0, implicit-def $nzcv, pcsections !0 ; CHECK-NEXT: Bcc 1, %bb.3, implicit killed $nzcv, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -47,13 +47,13 @@ define i32 @val_compare_and_swap_from_load(ptr %p, i32 %cmp, ptr %pnew) { ; CHECK-NEXT: successors: %bb.1(0x80000000) ; CHECK-NEXT: liveins: $w1, $x0, $x2 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w9 = LDRWui killed renamable $x2, 0, implicit-def renamable $x9, pcsections !0 :: (load (s32) from %ir.pnew) + ; CHECK-NEXT: renamable $w9 = LDRWui killed renamable $x2, 0, implicit-def $x9, pcsections !0 :: (load (s32) from %ir.pnew) ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1.cmpxchg.start: ; CHECK-NEXT: successors: %bb.2(0x7c000000), %bb.3(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0, $x9 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s32) from %ir.p) + ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s32) from %ir.p) ; CHECK-NEXT: $wzr = SUBSWrs renamable $w8, renamable $w1, 0, implicit-def $nzcv, pcsections !0 ; CHECK-NEXT: Bcc 1, %bb.3, implicit killed $nzcv, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -93,7 +93,7 @@ define i32 @val_compare_and_swap_rel(ptr %p, i32 %cmp, i32 %new) { ; CHECK-NEXT: successors: %bb.2(0x7c000000), %bb.3(0x04000000) ; CHECK-NEXT: liveins: $w1, $w2, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s32) from %ir.p) + ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s32) from %ir.p) ; CHECK-NEXT: $wzr = SUBSWrs renamable $w8, renamable $w1, 0, implicit-def $nzcv, pcsections !0 ; CHECK-NEXT: Bcc 1, %bb.3, implicit killed $nzcv, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -249,7 +249,7 @@ define i32 @fetch_and_nand(ptr %p) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRW renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s32) from %ir.p) + ; CHECK-NEXT: renamable $w8 = LDXRW renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s32) from %ir.p) ; CHECK-NEXT: renamable $w9 = ANDWri renamable $w8, 2, pcsections !0 ; CHECK-NEXT: $w9 = ORNWrs $wzr, killed renamable $w9, 0, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRW killed renamable $w9, renamable $x0, pcsections !0 :: (volatile store (s32) into %ir.p) @@ -302,7 +302,7 @@ define i32 @fetch_and_or(ptr %p) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w9, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s32) from %ir.p) + ; CHECK-NEXT: renamable $w8 = LDAXRW renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s32) from %ir.p) ; CHECK-NEXT: $w10 = ORRWrs renamable $w8, renamable $w9, 0, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w11 = STLXRW killed renamable $w10, renamable $x0, pcsections !0 :: (volatile store (s32) into %ir.p) ; CHECK-NEXT: CBNZW killed renamable $w11, %bb.1, pcsections !0 @@ -735,8 +735,8 @@ define i8 @atomicrmw_add_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) - ; CHECK-NEXT: $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -761,7 +761,7 @@ define i8 @atomicrmw_xchg_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $x0, $x1 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: early-clobber renamable $w9 = STXRB renamable $w1, renamable $x0, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w9, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -785,8 +785,8 @@ define i8 @atomicrmw_sub_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) - ; CHECK-NEXT: $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -810,8 +810,8 @@ define i8 @atomicrmw_and_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) - ; CHECK-NEXT: $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -835,8 +835,8 @@ define i8 @atomicrmw_or_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) - ; CHECK-NEXT: $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -860,8 +860,8 @@ define i8 @atomicrmw_xor_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) - ; CHECK-NEXT: $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -885,10 +885,10 @@ define i8 @atomicrmw_min_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = SBFMWri renamable $w8, 0, 7, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 32, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 13, implicit killed $nzcv, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 13, implicit killed $nzcv, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -912,10 +912,10 @@ define i8 @atomicrmw_max_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = SBFMWri renamable $w8, 0, 7, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 32, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRB renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -940,10 +940,10 @@ define i8 @atomicrmw_umin_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w9, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDAXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: renamable $w10 = ANDWri renamable $w8, 7 ; CHECK-NEXT: $wzr = SUBSWrs renamable $w10, renamable $w9, 0, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 9, implicit killed $nzcv, implicit-def renamable $x10, pcsections !0 + ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 9, implicit killed $nzcv, implicit-def $x10, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w11 = STLXRB renamable $w10, renamable $x0, implicit killed $x10, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w11, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -968,10 +968,10 @@ define i8 @atomicrmw_umax_i8(ptr %ptr, i8 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w9, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRB renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: renamable $w10 = ANDWri renamable $w8, 7 ; CHECK-NEXT: $wzr = SUBSWrs renamable $w10, renamable $w9, 0, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 8, implicit killed $nzcv, implicit-def renamable $x10, pcsections !0 + ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 8, implicit killed $nzcv, implicit-def $x10, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w11 = STXRB renamable $w10, renamable $x0, implicit killed $x10, pcsections !0 :: (volatile store (s8) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w11, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -995,8 +995,8 @@ define i16 @atomicrmw_add_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) - ; CHECK-NEXT: $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: $w9 = ADDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1021,7 +1021,7 @@ define i16 @atomicrmw_xchg_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $x0, $x1 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: early-clobber renamable $w9 = STXRH renamable $w1, renamable $x0, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w9, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1045,8 +1045,8 @@ define i16 @atomicrmw_sub_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) - ; CHECK-NEXT: $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: $w9 = SUBWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1070,8 +1070,8 @@ define i16 @atomicrmw_and_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) - ; CHECK-NEXT: $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: $w9 = ANDWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1095,8 +1095,8 @@ define i16 @atomicrmw_or_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) - ; CHECK-NEXT: $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: $w9 = ORRWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1120,8 +1120,8 @@ define i16 @atomicrmw_xor_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) - ; CHECK-NEXT: $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: $w9 = EORWrs renamable $w8, renamable $w1, 0, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1145,10 +1145,10 @@ define i16 @atomicrmw_min_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = SBFMWri renamable $w8, 0, 15, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 40, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 13, implicit killed $nzcv, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 13, implicit killed $nzcv, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1172,10 +1172,10 @@ define i16 @atomicrmw_max_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w1, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = SBFMWri renamable $w8, 0, 15, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 40, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def renamable $x9, pcsections !0 + ; CHECK-NEXT: renamable $w9 = CSELWr renamable $w8, renamable $w1, 12, implicit killed $nzcv, implicit-def $x9, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w10 = STLXRH renamable $w9, renamable $x0, implicit killed $x9, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w10, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1200,10 +1200,10 @@ define i16 @atomicrmw_umin_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w9, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDAXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: renamable $w10 = ANDWri renamable $w8, 15 ; CHECK-NEXT: $wzr = SUBSWrs renamable $w10, renamable $w9, 0, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 9, implicit killed $nzcv, implicit-def renamable $x10, pcsections !0 + ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 9, implicit killed $nzcv, implicit-def $x10, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w11 = STLXRH renamable $w10, renamable $x0, implicit killed $x10, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w11, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1228,10 +1228,10 @@ define i16 @atomicrmw_umax_i16(ptr %ptr, i16 %rhs) { ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000) ; CHECK-NEXT: liveins: $w9, $x0 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def renamable $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w8 = LDXRH renamable $x0, implicit-def $x8, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: renamable $w10 = ANDWri renamable $w8, 15 ; CHECK-NEXT: $wzr = SUBSWrs renamable $w10, renamable $w9, 0, implicit-def $nzcv, pcsections !0 - ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 8, implicit killed $nzcv, implicit-def renamable $x10, pcsections !0 + ; CHECK-NEXT: renamable $w10 = CSELWr killed renamable $w10, renamable $w9, 8, implicit killed $nzcv, implicit-def $x10, pcsections !0 ; CHECK-NEXT: early-clobber renamable $w11 = STXRH renamable $w10, renamable $x0, implicit killed $x10, pcsections !0 :: (volatile store (s16) into %ir.ptr) ; CHECK-NEXT: CBNZW killed renamable $w11, %bb.1, pcsections !0 ; CHECK-NEXT: {{ $}} @@ -1257,7 +1257,7 @@ define { i8, i1 } @cmpxchg_i8(ptr %ptr, i8 %desired, i8 %new) { ; CHECK-NEXT: successors: %bb.2(0x7c000000), %bb.4(0x04000000) ; CHECK-NEXT: liveins: $w1, $x2, $x8 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w0 = LDXRB renamable $x8, implicit-def renamable $x0, pcsections !0 :: (volatile load (s8) from %ir.ptr) + ; CHECK-NEXT: renamable $w0 = LDXRB renamable $x8, implicit-def $x0, pcsections !0 :: (volatile load (s8) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = ANDWri renamable $w0, 7, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 0, implicit-def $nzcv, pcsections !0 ; CHECK-NEXT: Bcc 1, %bb.4, implicit killed $nzcv, pcsections !0 @@ -1300,7 +1300,7 @@ define { i16, i1 } @cmpxchg_i16(ptr %ptr, i16 %desired, i16 %new) { ; CHECK-NEXT: successors: %bb.2(0x7c000000), %bb.4(0x04000000) ; CHECK-NEXT: liveins: $w1, $x2, $x8 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $w0 = LDXRH renamable $x8, implicit-def renamable $x0, pcsections !0 :: (volatile load (s16) from %ir.ptr) + ; CHECK-NEXT: renamable $w0 = LDXRH renamable $x8, implicit-def $x0, pcsections !0 :: (volatile load (s16) from %ir.ptr) ; CHECK-NEXT: renamable $w9 = ANDWri renamable $w0, 15, pcsections !0 ; CHECK-NEXT: dead $wzr = SUBSWrx killed renamable $w9, renamable $w1, 8, implicit-def $nzcv, pcsections !0 ; CHECK-NEXT: Bcc 1, %bb.4, implicit killed $nzcv, pcsections !0 diff --git a/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness.ll b/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness.ll deleted file mode 100644 index a3c3fc70e9761..0000000000000 --- a/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness.ll +++ /dev/null @@ -1,185 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=x86_64-grtev4-linux-gnu < %s | FileCheck %s - -%struct.wibble = type { %struct.wombat } -%struct.wombat = type { %struct.ham, [3 x i8] } -%struct.ham = type { %struct.zot } -%struct.zot = type { %struct.blam } -%struct.blam = type { %struct.ham.0 } -%struct.ham.0 = type { %struct.bar } -%struct.bar = type { %struct.bar.1 } -%struct.bar.1 = type { %struct.baz, i8 } -%struct.baz = type { %struct.snork } -%struct.snork = type <{ %struct.spam, i8, [3 x i8] }> -%struct.spam = type { %struct.snork.2, %struct.snork.2 } -%struct.snork.2 = type { i32 } -%struct.snork.3 = type { %struct.baz, i8, [3 x i8] } - -define void @foo(ptr %arg, ptr %arg1, i40 %arg2, ptr %arg3, i32 %arg4) #0 { -; CHECK-LABEL: foo: -; CHECK: # %bb.0: # %bb -; CHECK-NEXT: pushq %rbp -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset %rbp, -16 -; CHECK-NEXT: movq %rsp, %rbp -; CHECK-NEXT: .cfi_def_cfa_register %rbp -; CHECK-NEXT: pushq %r15 -; CHECK-NEXT: pushq %r14 -; CHECK-NEXT: pushq %r13 -; CHECK-NEXT: pushq %r12 -; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: subq $24, %rsp -; CHECK-NEXT: .cfi_offset %rbx, -56 -; CHECK-NEXT: .cfi_offset %r12, -48 -; CHECK-NEXT: .cfi_offset %r13, -40 -; CHECK-NEXT: .cfi_offset %r14, -32 -; CHECK-NEXT: .cfi_offset %r15, -24 -; CHECK-NEXT: movl %r8d, %r14d -; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: movq %rsi, %r13 -; CHECK-NEXT: movq %rdi, %r15 -; CHECK-NEXT: incl %r14d -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: # implicit-def: $r12 -; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: jmp .LBB0_3 -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_1: # %bb17 -; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: movq %r15, %r13 -; CHECK-NEXT: xorl %r15d, %r15d -; CHECK-NEXT: testq %rbx, %rbx -; CHECK-NEXT: sete %r15b -; CHECK-NEXT: xorl %edi, %edi -; CHECK-NEXT: callq _Znwm@PLT -; CHECK-NEXT: shll $4, %r15d -; CHECK-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload -; CHECK-NEXT: movq %r12, %rcx -; CHECK-NEXT: shrq $32, %rcx -; CHECK-NEXT: movb %cl, 12(%rax) -; CHECK-NEXT: movl %r12d, 8(%rax) -; CHECK-NEXT: movq %r15, %rbx -; CHECK-NEXT: movq %r13, %r15 -; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload -; CHECK-NEXT: decl %r14d -; CHECK-NEXT: je .LBB0_8 -; CHECK-NEXT: .LBB0_3: # %bb7 -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: callq widget@PLT -; CHECK-NEXT: cmpb $-5, (%r13) -; CHECK-NEXT: jae .LBB0_5 -; CHECK-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: movl %r12d, %r12d -; CHECK-NEXT: cmpq %r15, %rbx -; CHECK-NEXT: jbe .LBB0_1 -; CHECK-NEXT: jmp .LBB0_7 -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_5: # %bb12 -; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: movq 0, %rax -; CHECK-NEXT: movq 8, %rax -; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload -; CHECK-NEXT: cmpq %r15, %rbx -; CHECK-NEXT: jbe .LBB0_1 -; CHECK-NEXT: .LBB0_7: # in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: decl %r14d -; CHECK-NEXT: jne .LBB0_3 -; CHECK-NEXT: .LBB0_8: # %bb21 -; CHECK-NEXT: cmpb $0, 12(%rax) -; CHECK-NEXT: jne .LBB0_10 -; CHECK-NEXT: # %bb.9: # %bb26 -; CHECK-NEXT: addq $24, %rsp -; CHECK-NEXT: popq %rbx -; CHECK-NEXT: popq %r12 -; CHECK-NEXT: popq %r13 -; CHECK-NEXT: popq %r14 -; CHECK-NEXT: popq %r15 -; CHECK-NEXT: popq %rbp -; CHECK-NEXT: .cfi_def_cfa %rsp, 8 -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB0_10: # %bb25 -; CHECK-NEXT: .cfi_def_cfa %rbp, 16 -; CHECK-NEXT: movq %r15, %rdi -; CHECK-NEXT: callq pluto@PLT -bb: - br label %bb7 - -bb5: ; preds = %bb17, %bb14 - %phi = phi ptr [ %call19, %bb17 ], [ null, %bb14 ] - %phi6 = phi ptr [ %getelementptr, %bb17 ], [ null, %bb14 ] - %add = add i32 %phi9, 1 - %icmp = icmp eq i32 %phi9, %arg4 - br i1 %icmp, label %bb21, label %bb7 - -bb7: ; preds = %bb5, %bb - %phi8 = phi ptr [ null, %bb ], [ %phi6, %bb5 ] - %phi9 = phi i32 [ 0, %bb ], [ %add, %bb5 ] - %phi10 = phi i40 [ undef, %bb ], [ %phi15, %bb5 ] - %call = call ptr @widget() - %load = load i8, ptr %arg1, align 8 - %icmp11 = icmp ult i8 %load, -5 - %and = and i40 %phi10, 4294967295 - br i1 %icmp11, label %bb14, label %bb12 - -bb12: ; preds = %bb7 - %load13 = load volatile { i64, i64 }, ptr null, align 4294967296 - br label %bb14 - -bb14: ; preds = %bb12, %bb7 - %phi15 = phi i40 [ %and, %bb7 ], [ %arg2, %bb12 ] - %icmp16 = icmp ugt ptr %phi8, %arg - br i1 %icmp16, label %bb5, label %bb17 - -bb17: ; preds = %bb14 - %icmp18 = icmp eq ptr %phi8, null - %zext = zext i1 %icmp18 to i64 - %call19 = call ptr @_Znwm(i64 0) - %getelementptr = getelementptr %struct.wibble, ptr %arg3, i64 %zext - %getelementptr20 = getelementptr i8, ptr %call19, i64 8 - store i40 %phi15, ptr %getelementptr20, align 4 - br label %bb5 - -bb21: ; preds = %bb5 - %getelementptr22 = getelementptr %struct.snork.3, ptr %phi, i64 0, i32 1 - %load23 = load i8, ptr %getelementptr22, align 4 - %icmp24 = icmp eq i8 %load23, 0 - br i1 %icmp24, label %bb26, label %bb25 - -bb25: ; preds = %bb21 - call void @pluto(ptr %arg) - unreachable - -bb26: ; preds = %bb21 - ret void -} - -define void @eggs(ptr %arg, ptr %arg1) { -; CHECK-LABEL: eggs: -; CHECK: # %bb.0: # %bb -; CHECK-NEXT: pushq %rax -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: movq %rdi, %rax -; CHECK-NEXT: movq %rsi, %rdi -; CHECK-NEXT: movq %rax, %rsi -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: xorl %ecx, %ecx -; CHECK-NEXT: xorl %r8d, %r8d -; CHECK-NEXT: callq foo@PLT -; CHECK-NEXT: popq %rax -; CHECK-NEXT: .cfi_def_cfa_offset 8 -; CHECK-NEXT: retq -bb: - call void @foo(ptr %arg1, ptr %arg, i40 0, ptr null, i32 0) - ret void -} - -declare ptr @widget() - -declare void @pluto(ptr) - -declare ptr @_Znwm(i64) - -attributes #0 = { noinline "frame-pointer"="all" } diff --git a/llvm/test/CodeGen/X86/coalescer-implicit-def-regression-imp-operand-assert.mir b/llvm/test/CodeGen/X86/coalescer-implicit-def-regression-imp-operand-assert.mir index 190b14052d9b6..8241a1757af52 100644 --- a/llvm/test/CodeGen/X86/coalescer-implicit-def-regression-imp-operand-assert.mir +++ b/llvm/test/CodeGen/X86/coalescer-implicit-def-regression-imp-operand-assert.mir @@ -9,7 +9,7 @@ body: | ; CHECK-NEXT: successors: %bb.1(0x2aaaaaab), %bb.2(0x55555555) ; CHECK-NEXT: liveins: $edi ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: undef [[MOV32r0_:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def [[MOV32r0_]] + ; CHECK-NEXT: undef [[MOV32r0_:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags ; CHECK-NEXT: JCC_1 %bb.2, 5, implicit killed undef $eflags ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: @@ -28,7 +28,7 @@ body: | ; CHECK-NEXT: JCC_1 %bb.5, 5, implicit killed undef $eflags ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.4: - ; CHECK-NEXT: dead $eax = MOV32r0 implicit-def dead $eflags, implicit-def $al, implicit-def $al + ; CHECK-NEXT: dead $eax = MOV32r0 implicit-def dead $eflags, implicit-def $al ; CHECK-NEXT: RET 0, killed undef $al ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.5: diff --git a/llvm/test/CodeGen/X86/coalescing-subreg-to-reg-requires-subrange-update.mir b/llvm/test/CodeGen/X86/coalescing-subreg-to-reg-requires-subrange-update.mir deleted file mode 100644 index fe53aef86e835..0000000000000 --- a/llvm/test/CodeGen/X86/coalescing-subreg-to-reg-requires-subrange-update.mir +++ /dev/null @@ -1,47 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3 -# RUN: llc -mtriple=x86_64-- -run-pass=register-coalescer -enable-subreg-liveness -verify-coalescing -o - %s | FileCheck %s - - -# FIXME: Need to handle subrange updates when coalescing with subreg_to_reg -# This will fail if x86 enables subregister liveness. ---- -name: requires_new_subrange_coalesce_subreg_to_reg -tracksRegLiveness: true -body: | - ; CHECK-LABEL: name: requires_new_subrange_coalesce_subreg_to_reg - ; CHECK: bb.0: - ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000) - ; CHECK-NEXT: liveins: $eax - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: undef %a.sub_32bit:gr64_with_sub_8bit = COPY $eax - ; CHECK-NEXT: %b:gr32 = IMPLICIT_DEF - ; CHECK-NEXT: %c:gr64 = INSERT_SUBREG %a, %b, %subreg.sub_32bit - ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit undef $eflags - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1: - ; CHECK-NEXT: successors: %bb.2(0x80000000) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: undef %a.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags - ; CHECK-NEXT: %c.sub_32bit:gr64 = COPY %a - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: %c.sub_32bit:gr64 = SUBREG_TO_REG %a, %b, %subreg.sub_32bit - ; CHECK-NEXT: RET 0, implicit %c - bb.0: - liveins: $eax - %init_eax:gr32 = COPY $eax - %a:gr64 = SUBREG_TO_REG 0, %init_eax, %subreg.sub_32bit - %b:gr32 = IMPLICIT_DEF - %c:gr64 = INSERT_SUBREG %a, %b, %subreg.sub_32bit - JCC_1 %bb.2, 4, implicit undef $eflags - - bb.1: - %imm0:gr32 = MOV32r0 implicit-def dead $eflags - %a = SUBREG_TO_REG 0, %imm0, %subreg.sub_32bit - %c.sub_32bit = COPY %a - - bb.2: - %c.sub_32bit = SUBREG_TO_REG %a, %b, %subreg.sub_32bit - RET 0, implicit %c - -... diff --git a/llvm/test/CodeGen/X86/subreg-to-reg-coalescing.mir b/llvm/test/CodeGen/X86/subreg-to-reg-coalescing.mir deleted file mode 100644 index 6121a0bcc5641..0000000000000 --- a/llvm/test/CodeGen/X86/subreg-to-reg-coalescing.mir +++ /dev/null @@ -1,348 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2 -# RUN: llc -mtriple=x86_64-- -run-pass=register-coalescer -o - %s | FileCheck %s - -# We cannot lose the liveness of the high subregister of %1 when -# coalesced with %0, so introduce an implicit-def of the super -# register on the MOV. - ---- -name: coalesce_mov32r0_into_subreg_to_reg64 -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64 - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %1 - ; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %0:gr32 = MOV32r0 implicit-def dead $eflags - %1:gr64 = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: subreg_to_reg_folds_to_undef -tracksRegLiveness: true -body: | - bb.0: - liveins: $rax - - ; CHECK-LABEL: name: subreg_to_reg_folds_to_undef - ; CHECK: liveins: $rax - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64_with_sub_8bit = COPY $rax - ; CHECK-NEXT: undef %4.sub_32bit:gr64_with_sub_8bit = MOV32rr [[COPY]].sub_32bit, implicit-def %4 - ; CHECK-NEXT: RET 0, implicit %4 - %0:gr64 = COPY killed $rax - %1:gr32 = COPY killed %0.sub_32bit - %2:gr32 = MOV32rr killed %1 - %3:gr64 = SUBREG_TO_REG 0, killed %2, %subreg.sub_32bit - %4:gr64 = COPY killed %3 - RET 0, implicit %4 - -... - ---- -name: coalesce_mov32r0_subreg_def_into_subreg_to_reg64 -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_subreg_def_into_subreg_to_reg64 - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags - ; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - undef %0.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags - %1:gr64 = SUBREG_TO_REG 0, killed %0.sub_32bit, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: coalesce_mov32r0_into_subreg_def_with_super_def_to_reg64 -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_def_with_super_def_to_reg64 - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %1 - ; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - undef %0.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %0 - %1:gr64 = SUBREG_TO_REG 0, killed %0.sub_32bit, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_already_defs_other_subreg -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_already_defs_other_subreg - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def undef %1.sub_8bit, implicit-def %1 - ; CHECK-NEXT: INLINEASM &"", 0 /* attdialect */, implicit %1 - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit undef $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %0:gr32 = MOV32r0 implicit-def dead $eflags, implicit-def undef %0.sub_8bit - %1:gr64 = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - INLINEASM &"", 0, implicit %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit undef $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - - -# Reduced realistic case which was asserting after introducing new implicit-defs ---- -name: coalesce_needs_implicit_defs -tracksRegLiveness: true -body: | - ; CHECK-LABEL: name: coalesce_needs_implicit_defs - ; CHECK: bb.0: - ; CHECK-NEXT: successors: %bb.1(0x80000000) - ; CHECK-NEXT: liveins: $rdi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi - ; CHECK-NEXT: undef %2.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %2 - ; CHECK-NEXT: undef %3.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %3 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1: - ; CHECK-NEXT: successors: %bb.1(0x80000000) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: undef %10.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags - ; CHECK-NEXT: TEST64rr %3, %3, implicit-def $eflags - ; CHECK-NEXT: %10.sub_8bit:gr64_with_sub_8bit = SETCCr 4, implicit killed $eflags - ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi - ; CHECK-NEXT: CALL64r %2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: [[SHL64ri:%[0-9]+]]:gr64_with_sub_8bit = SHL64ri [[SHL64ri]], 4, implicit-def dead $eflags - ; CHECK-NEXT: [[ADD64rr:%[0-9]+]]:gr64_with_sub_8bit = ADD64rr [[ADD64rr]], [[COPY]], implicit-def dead $eflags - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64_with_sub_8bit = COPY [[ADD64rr]] - ; CHECK-NEXT: JMP_1 %bb.1 - bb.0: - liveins: $rdi - - %0:gr64 = COPY killed $rdi - %1:gr32 = MOV32r0 implicit-def dead $eflags - %2:gr64 = SUBREG_TO_REG 0, %1, %subreg.sub_32bit - %3:gr64 = COPY killed %2 - - bb.1: - %4:gr64 = COPY killed %3 - %5:gr32 = MOV32r0 implicit-def dead $eflags - TEST64rr killed %4, %4, implicit-def $eflags - %6:gr8 = SETCCr 4, implicit killed $eflags - %7:gr32 = COPY killed %5 - %7.sub_8bit:gr32 = COPY killed %6 - %8:gr64 = SUBREG_TO_REG 0, killed %7, %subreg.sub_32bit - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %9:gr64 = SUBREG_TO_REG 0, %1, %subreg.sub_32bit - $rdi = COPY %9 - CALL64r killed %9, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %10:gr64 = COPY killed %8 - %10:gr64 = SHL64ri %10, 4, implicit-def dead $eflags - %11:gr64 = COPY killed %10 - %11:gr64 = ADD64rr %11, %0, implicit-def dead $eflags - %3:gr64 = COPY killed %11 - JMP_1 %bb.1 - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_physreg_def -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_physreg_def - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi - ; CHECK-NEXT: CALL64r killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %0:gr32 = MOV32r0 implicit-def dead $eflags - $rdi = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - CALL64r killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_physreg_use -tracksRegLiveness: true -body: | - bb.0: - liveins: $eax - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_physreg_use - ; CHECK: liveins: $eax - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: $eax = MOV32r0 implicit-def dead $eflags - ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, $eax, %subreg.sub_32bit - ; CHECK-NEXT: $rdi = COPY [[SUBREG_TO_REG]] - ; CHECK-NEXT: CALL64r [[SUBREG_TO_REG]], csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - $eax = MOV32r0 implicit-def dead $eflags - %1:gr64 = SUBREG_TO_REG 0, killed $eax, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - -# Coalesced instruction is a copy with other implicit operands ---- -name: coalesce_copy_into_subreg_to_reg64 -tracksRegLiveness: true -body: | - bb.0: - liveins: $eax - ; CHECK-LABEL: name: coalesce_copy_into_subreg_to_reg64 - ; CHECK: liveins: $eax - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = COPY $eax, implicit-def dead $eflags, implicit-def %1 - ; CHECK-NEXT: $rdi = COPY %1 - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %0:gr32 = COPY $eax, implicit-def dead $eflags - %1:gr64 = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_multiple_redef_value -tracksRegLiveness: true -body: | - bb.0: - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_multiple_redef_value - ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: undef %1.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %1 - ; CHECK-NEXT: INLINEASM &"", 0 /* attdialect */, implicit-def %1.sub_32bit, implicit %1.sub_32bit - ; CHECK-NEXT: $rdi = COPY %1 - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - %0:gr32 = MOV32r0 implicit-def dead $eflags - INLINEASM &"", 0, implicit-def %0, implicit %0 - %1:gr64 = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - $rdi = COPY %1 - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_def_is_block_liveout -tracksRegLiveness: true -body: | - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_def_is_block_liveout - ; CHECK: bb.0: - ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM &"", 0 /* attdialect */, implicit-def undef %1.sub_32bit, implicit-def %1 - ; CHECK-NEXT: JCC_1 %bb.1, 4, implicit undef $eflags - ; CHECK-NEXT: JMP_1 %bb.2 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1: - ; CHECK-NEXT: $rdi = COPY %1 - ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: RET 0 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.2: - bb.0: - INLINEASM &"", 0, implicit-def %0:gr32 - JCC_1 %bb.1, 4, implicit undef $eflags - JMP_1 %bb.2 - - bb.1: - %1:gr64 = SUBREG_TO_REG 0, killed %0, %subreg.sub_32bit - $rdi = COPY %1 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - CALL64r killed %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - RET 0 - - bb.2: - -... - ---- -name: coalesce_mov32r0_into_subreg_to_reg64_def_is_phi_def -tracksRegLiveness: true -body: | - ; CHECK-LABEL: name: coalesce_mov32r0_into_subreg_to_reg64_def_is_phi_def - ; CHECK: bb.0: - ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM &"", 0 /* attdialect */, implicit-def undef %1.sub_32bit, implicit-def %1 - ; CHECK-NEXT: JCC_1 %bb.1, 4, implicit undef $eflags - ; CHECK-NEXT: JMP_1 %bb.2 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1: - ; CHECK-NEXT: successors: %bb.1(0x80000000) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: $rdi = COPY %1 - ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - ; CHECK-NEXT: JMP_1 %bb.1 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.2: - bb.0: - - INLINEASM &"", 0, implicit-def %0:gr32 - JCC_1 %bb.1, 4, implicit undef $eflags - JMP_1 %bb.2 - - bb.1: - %1:gr64 = SUBREG_TO_REG 0, %0, %subreg.sub_32bit - $rdi = COPY %1 - ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - CALL64r %1, csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax - ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - JMP_1 %bb.1 - - bb.2: - -... From 82ab0f7f36222a0311b5220df52f4193664569e8 Mon Sep 17 00:00:00 2001 From: Quinn Dawkins Date: Sat, 16 Dec 2023 10:08:51 -0500 Subject: [PATCH 038/884] [mlir][linalg] Fix rank-reduced cases for extract/insert slice in DropUnitDims (#74723) Inferring the reshape reassociation indices for extract/insert slice ops based on the read sizes of the original slicing op will generate an invalid expand/collapse shape op for already rank-reduced cases. Instead just infer from the shape of the slice. Ported from Differential Revision: https://reviews.llvm.org/D147488 --- .../Linalg/Transforms/DropUnitDims.cpp | 21 +++++++++------- .../Dialect/Linalg/drop-unit-extent-dims.mlir | 24 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp index 6fbf351455787..c495956fa5770 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @@ -572,13 +572,17 @@ struct RankReducedExtractSliceOp LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp, PatternRewriter &rewriter) const override { RankedTensorType resultType = sliceOp.getType(); - SmallVector offsets = sliceOp.getMixedOffsets(); - SmallVector sizes = sliceOp.getMixedSizes(); - SmallVector strides = sliceOp.getMixedStrides(); - auto reassociation = getReassociationMapForFoldingUnitDims(sizes); + SmallVector targetShape; + for (auto size : resultType.getShape()) + targetShape.push_back(rewriter.getIndexAttr(size)); + auto reassociation = getReassociationMapForFoldingUnitDims(targetShape); if (!reassociation || reassociation->size() == static_cast(resultType.getRank())) return failure(); + + SmallVector offsets = sliceOp.getMixedOffsets(); + SmallVector strides = sliceOp.getMixedStrides(); + SmallVector sizes = sliceOp.getMixedSizes(); auto rankReducedType = cast( tensor::ExtractSliceOp::inferCanonicalRankReducedResultType( reassociation->size(), sliceOp.getSourceType(), offsets, sizes, @@ -602,13 +606,14 @@ struct RankReducedInsertSliceOp : public OpRewritePattern { LogicalResult matchAndRewrite(InsertOpTy insertSliceOp, PatternRewriter &rewriter) const override { RankedTensorType sourceType = insertSliceOp.getSourceType(); - SmallVector offsets = insertSliceOp.getMixedOffsets(); - SmallVector sizes = insertSliceOp.getMixedSizes(); - SmallVector strides = insertSliceOp.getMixedStrides(); - auto reassociation = getReassociationMapForFoldingUnitDims(sizes); + SmallVector targetShape; + for (auto size : sourceType.getShape()) + targetShape.push_back(rewriter.getIndexAttr(size)); + auto reassociation = getReassociationMapForFoldingUnitDims(targetShape); if (!reassociation || reassociation->size() == static_cast(sourceType.getRank())) return failure(); + Location loc = insertSliceOp.getLoc(); tensor::CollapseShapeOp reshapedSource; { diff --git a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir index 795e9ee528717..0c51a032df901 100644 --- a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir +++ b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir @@ -489,6 +489,18 @@ func.func @slice_unit_dims(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { // ----- +func.func @rank_reduced_extract_slice(%arg0: tensor<1x1x3x1x3xf32>) -> tensor<1x3x3xf32> { + %0 = tensor.extract_slice %arg0[0, 0, 0, 0, 0] [1, 1, 3, 1, 3] [1, 1, 1, 1, 1] : tensor<1x1x3x1x3xf32> to tensor<1x3x3xf32> + return %0 : tensor<1x3x3xf32> +} +// CHECK-LABEL: func @rank_reduced_extract_slice +// CHECK: %[[SLICE:.+]] = tensor.extract_slice +// CHECK-SAME: tensor<1x1x3x1x3xf32> to tensor<3x3xf32> +// CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[SLICE]] {{\[}}[0, 1], [2]] +// CHECK: return %[[RESULT]] + +// ----- + func.func @insert_slice_unit_dims(%arg0: tensor<1x3xf32>, %arg1: tensor<1x1xf32>) -> tensor<1x3xf32> { %0 = tensor.insert_slice %arg1 into %arg0[0, 2] [1, 1] [1, 1] : tensor<1x1xf32> into tensor<1x3xf32> return %0 : tensor<1x3xf32> @@ -501,6 +513,18 @@ func.func @insert_slice_unit_dims(%arg0: tensor<1x3xf32>, %arg1: tensor<1x1xf32> // ----- +func.func @rank_reduced_insert_slice(%arg0: tensor<1x1x3x1x3xf32>, %arg1: tensor<1x3x3xf32>) -> tensor<1x1x3x1x3xf32> { + %0 = tensor.insert_slice %arg1 into %arg0[0, 0, 0, 0, 0] [1, 1, 3, 1, 3] [1, 1, 1, 1, 1] : tensor<1x3x3xf32> into tensor<1x1x3x1x3xf32> + return %0 : tensor<1x1x3x1x3xf32> +} +// CHECK-LABEL: func @rank_reduced_insert_slice +// CHECK: %[[RESHAPE:.+]] = tensor.collapse_shape %{{.+}} {{\[}}[0, 1], [2]] +// CHECK: %[[RESULT:.+]] = tensor.insert_slice %[[RESHAPE]] +// CHECK-SAME: tensor<3x3xf32> into tensor<1x1x3x1x3xf32> +// CHECK: return %[[RESULT]] + +// ----- + #accesses = [ affine_map<(i, j, k, l, m) -> (i, k, m)>, affine_map<(i, j, k, l, m) -> ()>, From f49e2b05bf3ececa2fe20c5d658ab92ab974dc36 Mon Sep 17 00:00:00 2001 From: Youngsuk Kim Date: Sat, 16 Dec 2023 11:47:37 -0500 Subject: [PATCH 039/884] [clang][CGCUDANV] Unify PointerType members of CGNVCUDARuntime (NFC) (#75668) Unify 3 `Pointertype *` members which all refer to the same llvm type. Opaque pointer clean-up effort. --- clang/lib/CodeGen/CGCUDANV.cpp | 88 +++++++++++++++------------------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index 520b0c4f11767..353370f1d761b 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -39,7 +39,7 @@ class CGNVCUDARuntime : public CGCUDARuntime { private: llvm::IntegerType *IntTy, *SizeTy; llvm::Type *VoidTy; - llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy; + llvm::PointerType *PtrTy; /// Convenience reference to LLVM Context llvm::LLVMContext &Context; @@ -232,15 +232,12 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) VoidTy = CGM.VoidTy; Zeros[0] = llvm::ConstantInt::get(SizeTy, 0); Zeros[1] = Zeros[0]; - - CharPtrTy = CGM.UnqualPtrTy; - VoidPtrTy = CGM.UnqualPtrTy; - VoidPtrPtrTy = CGM.UnqualPtrTy; + PtrTy = CGM.UnqualPtrTy; } llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const { // cudaError_t cudaSetupArgument(void *, size_t, size_t) - llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy}; + llvm::Type *Params[] = {PtrTy, SizeTy, SizeTy}; return CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, Params, false), addPrefixToName("SetupArgument")); @@ -250,24 +247,24 @@ llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const { if (CGM.getLangOpts().HIP) { // hipError_t hipLaunchByPtr(char *); return CGM.CreateRuntimeFunction( - llvm::FunctionType::get(IntTy, CharPtrTy, false), "hipLaunchByPtr"); + llvm::FunctionType::get(IntTy, PtrTy, false), "hipLaunchByPtr"); } // cudaError_t cudaLaunch(char *); - return CGM.CreateRuntimeFunction( - llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch"); + return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy, PtrTy, false), + "cudaLaunch"); } llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const { - return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false); + return llvm::FunctionType::get(VoidTy, PtrTy, false); } llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const { - return llvm::FunctionType::get(VoidTy, VoidPtrTy, false); + return llvm::FunctionType::get(VoidTy, PtrTy, false); } llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const { - llvm::Type *Params[] = {llvm::PointerType::getUnqual(Context), VoidPtrTy, - VoidPtrTy, llvm::PointerType::getUnqual(Context)}; + llvm::Type *Params[] = {llvm::PointerType::getUnqual(Context), PtrTy, PtrTy, + llvm::PointerType::getUnqual(Context)}; return llvm::FunctionType::get(VoidTy, Params, false); } @@ -330,15 +327,15 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, // args, allocate a single pointer so we still have a valid pointer to the // argument array that we can pass to runtime, even if it will be unused. Address KernelArgs = CGF.CreateTempAlloca( - VoidPtrTy, CharUnits::fromQuantity(16), "kernel_args", + PtrTy, CharUnits::fromQuantity(16), "kernel_args", llvm::ConstantInt::get(SizeTy, std::max(1, Args.size()))); // Store pointers to the arguments in a locally allocated launch_args. for (unsigned i = 0; i < Args.size(); ++i) { llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); - llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, VoidPtrTy); + llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); CGF.Builder.CreateDefaultAlignedStore( VoidVarPtr, - CGF.Builder.CreateConstGEP1_32(VoidPtrTy, KernelArgs.getPointer(), i)); + CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i)); } llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -386,8 +383,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, CGF.CreateMemTemp(Dim3Ty, CharUnits::fromQuantity(8), "block_dim"); Address ShmemSize = CGF.CreateTempAlloca(SizeTy, CGM.getSizeAlign(), "shmem_size"); - Address Stream = - CGF.CreateTempAlloca(VoidPtrTy, CGM.getPointerAlign(), "stream"); + Address Stream = CGF.CreateTempAlloca(PtrTy, CGM.getPointerAlign(), "stream"); llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, {/*gridDim=*/GridDim.getType(), @@ -402,8 +398,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, ShmemSize.getPointer(), Stream.getPointer()}); // Emit the call to cudaLaunch - llvm::Value *Kernel = CGF.Builder.CreatePointerCast( - KernelHandles[CGF.CurFn->getName()], VoidPtrTy); + llvm::Value *Kernel = + CGF.Builder.CreatePointerCast(KernelHandles[CGF.CurFn->getName()], PtrTy); CallArgList LaunchKernelArgs; LaunchKernelArgs.add(RValue::get(Kernel), cudaLaunchKernelFD->getParamDecl(0)->getType()); @@ -443,7 +439,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, Offset = Offset.alignTo(TInfo.Align); llvm::Value *Args[] = { CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), - VoidPtrTy), + PtrTy), llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), }; @@ -458,8 +454,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, // Emit the call to cudaLaunch llvm::FunctionCallee cudaLaunchFn = getLaunchFn(); - llvm::Value *Arg = CGF.Builder.CreatePointerCast( - KernelHandles[CGF.CurFn->getName()], CharPtrTy); + llvm::Value *Arg = + CGF.Builder.CreatePointerCast(KernelHandles[CGF.CurFn->getName()], PtrTy); CGF.EmitRuntimeCallOrInvoke(cudaLaunchFn, Arg); CGF.EmitBranch(EndBlock); @@ -537,11 +533,8 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { // void __cudaRegisterFunction(void **, const char *, char *, const char *, // int, uint3*, uint3*, dim3*, dim3*, int*) llvm::Type *RegisterFuncParams[] = { - VoidPtrPtrTy, CharPtrTy, - CharPtrTy, CharPtrTy, - IntTy, VoidPtrTy, - VoidPtrTy, VoidPtrTy, - VoidPtrTy, llvm::PointerType::getUnqual(Context)}; + PtrTy, PtrTy, PtrTy, PtrTy, IntTy, + PtrTy, PtrTy, PtrTy, PtrTy, llvm::PointerType::getUnqual(Context)}; llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, RegisterFuncParams, false), addUnderscoredPrefixToName("RegisterFunction")); @@ -553,7 +546,7 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { for (auto &&I : EmittedKernels) { llvm::Constant *KernelName = makeConstantString(getDeviceSideName(cast(I.D))); - llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy); + llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(PtrTy); llvm::Value *Args[] = { &GpuBinaryHandlePtr, KernelHandles[I.Kernel->getName()], @@ -576,16 +569,15 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { // void __cudaRegisterVar(void **, char *, char *, const char *, // int, int, int, int) - llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, - CharPtrTy, IntTy, VarSizeTy, - IntTy, IntTy}; + llvm::Type *RegisterVarParams[] = {PtrTy, PtrTy, PtrTy, PtrTy, + IntTy, VarSizeTy, IntTy, IntTy}; llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction( llvm::FunctionType::get(VoidTy, RegisterVarParams, false), addUnderscoredPrefixToName("RegisterVar")); // void __hipRegisterManagedVar(void **, char *, char *, const char *, // size_t, unsigned) - llvm::Type *RegisterManagedVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, - CharPtrTy, VarSizeTy, IntTy}; + llvm::Type *RegisterManagedVarParams[] = {PtrTy, PtrTy, PtrTy, + PtrTy, VarSizeTy, IntTy}; llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction( llvm::FunctionType::get(VoidTy, RegisterManagedVarParams, false), addUnderscoredPrefixToName("RegisterManagedVar")); @@ -593,16 +585,13 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { // const void **, const char *, int, int); llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction( llvm::FunctionType::get( - VoidTy, {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy}, - false), + VoidTy, {PtrTy, PtrTy, PtrTy, PtrTy, IntTy, IntTy}, false), addUnderscoredPrefixToName("RegisterSurface")); // void __cudaRegisterTexture(void **, const struct textureReference *, // const void **, const char *, int, int, int) llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction( llvm::FunctionType::get( - VoidTy, - {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy, IntTy}, - false), + VoidTy, {PtrTy, PtrTy, PtrTy, PtrTy, IntTy, IntTy, IntTy}, false), addUnderscoredPrefixToName("RegisterTexture")); for (auto &&Info : DeviceVars) { llvm::GlobalVariable *Var = Info.Var; @@ -713,11 +702,11 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // void ** __{cuda|hip}RegisterFatBinary(void *); llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction( - llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false), + llvm::FunctionType::get(PtrTy, PtrTy, false), addUnderscoredPrefixToName("RegisterFatBinary")); // struct { int magic, int version, void * gpu_binary, void * dont_care }; llvm::StructType *FatbinWrapperTy = - llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy); + llvm::StructType::get(IntTy, IntTy, PtrTy, PtrTy); // Register GPU binary with the CUDA runtime, store returned handle in a // global variable and save a reference in GpuBinaryHandle to be cleaned up @@ -813,7 +802,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // Data. Values.add(FatBinStr); // Unused in fatbin v1. - Values.add(llvm::ConstantPointerNull::get(VoidPtrTy)); + Values.add(llvm::ConstantPointerNull::get(PtrTy)); llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal( addUnderscoredPrefixToName("_fatbin_wrapper"), CGM.getPointerAlign(), /*constant*/ true); @@ -836,9 +825,8 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // The name, size, and initialization pattern of this variable is part // of HIP ABI. GpuBinaryHandle = new llvm::GlobalVariable( - TheModule, VoidPtrPtrTy, /*isConstant=*/false, - Linkage, - /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy), + TheModule, PtrTy, /*isConstant=*/false, Linkage, + /*Initializer=*/llvm::ConstantPointerNull::get(PtrTy), "__hip_gpubin_handle"); if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage) GpuBinaryHandle->setComdat( @@ -848,7 +836,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { if (Linkage != llvm::GlobalValue::InternalLinkage) GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility); Address GpuBinaryAddr( - GpuBinaryHandle, VoidPtrPtrTy, + GpuBinaryHandle, PtrTy, CharUnits::fromQuantity(GpuBinaryHandle->getAlignment())); { auto *HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr); @@ -880,8 +868,8 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(RegisterFatbinFunc, FatbinWrapper); GpuBinaryHandle = new llvm::GlobalVariable( - TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, - llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); + TheModule, PtrTy, false, llvm::GlobalValue::InternalLinkage, + llvm::ConstantPointerNull::get(PtrTy), "__cuda_gpubin_handle"); GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign()); CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle, CGM.getPointerAlign()); @@ -895,7 +883,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { CudaFeature::CUDA_USES_FATBIN_REGISTER_END)) { // void __cudaRegisterFatBinaryEnd(void **); llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction( - llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + llvm::FunctionType::get(VoidTy, PtrTy, false), "__cudaRegisterFatBinaryEnd"); CtorBuilder.CreateCall(RegisterFatbinEndFunc, RegisterFatbinCall); } @@ -967,7 +955,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { // void __cudaUnregisterFatBinary(void ** handle); llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction( - llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + llvm::FunctionType::get(VoidTy, PtrTy, false), addUnderscoredPrefixToName("UnregisterFatBinary")); llvm::Function *ModuleDtorFunc = llvm::Function::Create( From 395f9ce30e884cb6ce02f7a3bdd0dd1f72ea9033 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 10:14:44 -0800 Subject: [PATCH 040/884] Use StringRef::{starts,ends}_with (NFC) This patch replaces uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- clang/lib/Lex/HeaderSearch.cpp | 2 +- llvm/lib/IR/AutoUpgrade.cpp | 2 +- llvm/lib/Target/DirectX/DXILResource.cpp | 2 +- .../SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp | 2 +- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 16 ++++----- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 34 +++++++++---------- llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp | 4 +-- llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 8 ++--- .../Xtensa/AsmParser/XtensaAsmParser.cpp | 8 ++--- 9 files changed, 39 insertions(+), 39 deletions(-) diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp index a0ac0eaf2f900..0f1090187734f 100644 --- a/clang/lib/Lex/HeaderSearch.cpp +++ b/clang/lib/Lex/HeaderSearch.cpp @@ -1671,7 +1671,7 @@ static OptionalFileEntryRef getPrivateModuleMap(FileEntryRef File, if (Filename == "module.map") Diags.Report(diag::warn_deprecated_module_dot_map) << PrivateFilename << 1 - << File.getDir().getName().endswith(".framework"); + << File.getDir().getName().ends_with(".framework"); } return PMMFile; } diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 738ec301d1479..6b54047020a05 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -5220,7 +5220,7 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { Res.append("-p7:160:256:256:32"); if (!DL.contains("-p8") && !DL.starts_with("p8")) Res.append("-p8:128:128"); - if (!DL.contains("-p9") && !DL.startswith("p9")) + if (!DL.contains("-p9") && !DL.starts_with("p9")) Res.append("-p9:192:256:256:32"); return Res; diff --git a/llvm/lib/Target/DirectX/DXILResource.cpp b/llvm/lib/Target/DirectX/DXILResource.cpp index 92306d907e054..d3ff12a1f7b34 100644 --- a/llvm/lib/Target/DirectX/DXILResource.cpp +++ b/llvm/lib/Target/DirectX/DXILResource.cpp @@ -261,7 +261,7 @@ void UAVResource::parseSourceType(StringRef S) { S = S.substr(S.find("<") + 1); constexpr size_t PrefixLen = StringRef("vector<").size(); - if (S.startswith("vector<")) + if (S.starts_with("vector<")) S = S.substr(PrefixLen, S.find(",") - PrefixLen); else S = S.substr(0, S.find(">")); diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp index 1af7b7a5d7845..b69031adb1673 100644 --- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp +++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp @@ -177,7 +177,7 @@ std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue) { bool getSpirvBuiltInIdByName(llvm::StringRef Name, SPIRV::BuiltIn::BuiltIn &BI) { const std::string Prefix = "__spirv_BuiltIn"; - if (!Name.startswith(Prefix)) + if (!Name.starts_with(Prefix)) return false; const SPIRV::SymbolicOperand *Lookup = diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index ae9e801f8f50b..5ac45079bd002 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -872,8 +872,8 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call, std::tie(GroupResultRegister, GroupResultType) = buildBoolRegister(MIRBuilder, Call->ReturnType, GR); - auto Scope = Builtin->Name.startswith("sub_group") ? SPIRV::Scope::Subgroup - : SPIRV::Scope::Workgroup; + auto Scope = Builtin->Name.starts_with("sub_group") ? SPIRV::Scope::Subgroup + : SPIRV::Scope::Workgroup; Register ScopeRegister = buildConstantIntReg(Scope, MIRBuilder, GR); // Build work/sub group instruction. @@ -1999,13 +1999,13 @@ struct OpenCLType { //===----------------------------------------------------------------------===// static Type *parseTypeString(const StringRef Name, LLVMContext &Context) { - if (Name.startswith("void")) + if (Name.starts_with("void")) return Type::getVoidTy(Context); - else if (Name.startswith("int") || Name.startswith("uint")) + else if (Name.starts_with("int") || Name.starts_with("uint")) return Type::getInt32Ty(Context); - else if (Name.startswith("float")) + else if (Name.starts_with("float")) return Type::getFloatTy(Context); - else if (Name.startswith("half")) + else if (Name.starts_with("half")) return Type::getHalfTy(Context); llvm_unreachable("Unable to recognize type!"); } @@ -2081,7 +2081,7 @@ parseBuiltinTypeNameToTargetExtType(std::string TypeName, // Pointers-to-opaque-structs representing OpenCL types are first translated // to equivalent SPIR-V types. OpenCL builtin type names should have the // following format: e.g. %opencl.event_t - if (NameWithParameters.startswith("opencl.")) { + if (NameWithParameters.starts_with("opencl.")) { const SPIRV::OpenCLType *OCLTypeRecord = SPIRV::lookupOpenCLType(NameWithParameters); if (!OCLTypeRecord) @@ -2093,7 +2093,7 @@ parseBuiltinTypeNameToTargetExtType(std::string TypeName, // Names of the opaque structs representing a SPIR-V builtins without // parameters should have the following format: e.g. %spirv.Event - assert(NameWithParameters.startswith("spirv.") && + assert(NameWithParameters.starts_with("spirv.") && "Unknown builtin opaque type!"); // Parameterized SPIR-V builtins names follow this format: diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index d450613196f3e..b8a6784ff3c62 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -962,35 +962,35 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName( if (TypeStr.starts_with("atomic_")) TypeStr = TypeStr.substr(strlen("atomic_")); - if (TypeStr.startswith("void")) { + if (TypeStr.starts_with("void")) { Ty = Type::getVoidTy(Ctx); TypeStr = TypeStr.substr(strlen("void")); - } else if (TypeStr.startswith("bool")) { + } else if (TypeStr.starts_with("bool")) { Ty = Type::getIntNTy(Ctx, 1); TypeStr = TypeStr.substr(strlen("bool")); - } else if (TypeStr.startswith("char") || TypeStr.startswith("uchar")) { + } else if (TypeStr.starts_with("char") || TypeStr.starts_with("uchar")) { Ty = Type::getInt8Ty(Ctx); - TypeStr = TypeStr.startswith("char") ? TypeStr.substr(strlen("char")) - : TypeStr.substr(strlen("uchar")); - } else if (TypeStr.startswith("short") || TypeStr.startswith("ushort")) { + TypeStr = TypeStr.starts_with("char") ? TypeStr.substr(strlen("char")) + : TypeStr.substr(strlen("uchar")); + } else if (TypeStr.starts_with("short") || TypeStr.starts_with("ushort")) { Ty = Type::getInt16Ty(Ctx); - TypeStr = TypeStr.startswith("short") ? TypeStr.substr(strlen("short")) - : TypeStr.substr(strlen("ushort")); - } else if (TypeStr.startswith("int") || TypeStr.startswith("uint")) { + TypeStr = TypeStr.starts_with("short") ? TypeStr.substr(strlen("short")) + : TypeStr.substr(strlen("ushort")); + } else if (TypeStr.starts_with("int") || TypeStr.starts_with("uint")) { Ty = Type::getInt32Ty(Ctx); - TypeStr = TypeStr.startswith("int") ? TypeStr.substr(strlen("int")) - : TypeStr.substr(strlen("uint")); + TypeStr = TypeStr.starts_with("int") ? TypeStr.substr(strlen("int")) + : TypeStr.substr(strlen("uint")); } else if (TypeStr.starts_with("long") || TypeStr.starts_with("ulong")) { Ty = Type::getInt64Ty(Ctx); - TypeStr = TypeStr.startswith("long") ? TypeStr.substr(strlen("long")) - : TypeStr.substr(strlen("ulong")); - } else if (TypeStr.startswith("half")) { + TypeStr = TypeStr.starts_with("long") ? TypeStr.substr(strlen("long")) + : TypeStr.substr(strlen("ulong")); + } else if (TypeStr.starts_with("half")) { Ty = Type::getHalfTy(Ctx); TypeStr = TypeStr.substr(strlen("half")); - } else if (TypeStr.startswith("float")) { + } else if (TypeStr.starts_with("float")) { Ty = Type::getFloatTy(Ctx); TypeStr = TypeStr.substr(strlen("float")); - } else if (TypeStr.startswith("double")) { + } else if (TypeStr.starts_with("double")) { Ty = Type::getDoubleTy(Ctx); TypeStr = TypeStr.substr(strlen("double")); } else @@ -1007,7 +1007,7 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName( // Handle "typeN*" or "type vector[N]*". bool IsPtrToVec = TypeStr.consume_back("*"); - if (TypeStr.startswith(" vector[")) { + if (TypeStr.starts_with(" vector[")) { TypeStr = TypeStr.substr(strlen(" vector[")); TypeStr = TypeStr.substr(0, TypeStr.find(']')); } diff --git a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp index 3a51e29dcf161..322e051a87db1 100644 --- a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp @@ -177,8 +177,8 @@ void SPIRVRegularizer::visitCallInst(CallInst &CI) { StringRef DemangledName(NameStr); // TODO: add support for other builtins. - if (DemangledName.startswith("fmin") || DemangledName.startswith("fmax") || - DemangledName.startswith("min") || DemangledName.startswith("max")) + if (DemangledName.starts_with("fmin") || DemangledName.starts_with("fmax") || + DemangledName.starts_with("min") || DemangledName.starts_with("max")) visitCallScalToVec(&CI, MangledName, DemangledName); free(NameStr); } diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index 0bd51436082d5..1c0e8d84e2fd1 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -279,7 +279,7 @@ static bool isKernelQueryBI(const StringRef MangledName) { } static bool isNonMangledOCLBuiltin(StringRef Name) { - if (!Name.startswith("__")) + if (!Name.starts_with("__")) return false; return isEnqueueKernelBI(Name) || isKernelQueryBI(Name) || @@ -289,8 +289,8 @@ static bool isNonMangledOCLBuiltin(StringRef Name) { std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name); - bool IsNonMangledSPIRV = Name.startswith("__spirv_"); - bool IsMangled = Name.startswith("_Z"); + bool IsNonMangledSPIRV = Name.starts_with("__spirv_"); + bool IsMangled = Name.starts_with("_Z"); if (!IsNonMangledOCL && !IsNonMangledSPIRV && !IsMangled) return std::string(); @@ -311,7 +311,7 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { // Similar to ::std:: in C++. size_t Start, Len = 0; size_t DemangledNameLenStart = 2; - if (Name.startswith("_ZN")) { + if (Name.starts_with("_ZN")) { // Skip CV and ref qualifiers. size_t NameSpaceStart = Name.find_first_not_of("rVKRO", 3); // All built-ins are in the ::cl:: namespace. diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp index 76c7328b547cd..3f808298527f8 100644 --- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp +++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp @@ -598,8 +598,8 @@ bool XtensaAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic, bool XtensaAsmParser::ParseInstructionWithSR(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, OperandVector &Operands) { - if ((Name.startswith("wsr.") || Name.startswith("rsr.") || - Name.startswith("xsr.")) && + if ((Name.starts_with("wsr.") || Name.starts_with("rsr.") || + Name.starts_with("xsr.")) && (Name.size() > 4)) { // Parse case when instruction name is concatenated with SR register // name, like "wsr.sar a1" @@ -655,8 +655,8 @@ bool XtensaAsmParser::ParseInstructionWithSR(ParseInstructionInfo &Info, bool XtensaAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, OperandVector &Operands) { - if (Name.startswith("wsr") || Name.startswith("rsr") || - Name.startswith("xsr")) { + if (Name.starts_with("wsr") || Name.starts_with("rsr") || + Name.starts_with("xsr")) { return ParseInstructionWithSR(Info, Name, NameLoc, Operands); } From 01c8af573961c54f0d922c3f3acffa880a0a459c Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 16 Dec 2023 13:12:17 -0800 Subject: [PATCH 041/884] [ELF,test] Improve duplicate "symbol not found" error tests --- .../linkerscript/copy-rel-symbol-value-err.s | 4 ++-- lld/test/ELF/linkerscript/sections-padding.s | 16 ++++++++----- lld/test/ELF/linkerscript/symbol-assignexpr.s | 24 ++++++++++++++++--- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/lld/test/ELF/linkerscript/copy-rel-symbol-value-err.s b/lld/test/ELF/linkerscript/copy-rel-symbol-value-err.s index cd5262b142f5f..70ca582affc46 100644 --- a/lld/test/ELF/linkerscript/copy-rel-symbol-value-err.s +++ b/lld/test/ELF/linkerscript/copy-rel-symbol-value-err.s @@ -3,9 +3,9 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/copy-rel-symbol-value.s -o %t2.o # RUN: ld.lld %t2.o -o %t2.so -shared # RUN: echo "SECTIONS { . = . + SIZEOF_HEADERS; foo = bar; }" > %t.script -# RUN: not ld.lld %t.o %t2.so --script %t.script -o /dev/null 2>&1 | FileCheck %s +# RUN: not ld.lld %t.o %t2.so --script %t.script -o /dev/null 2>&1 | FileCheck %s --implicit-check-not=error: -# CHECK: symbol not found: bar +# CHECK-COUNT-2: error: {{.*}}.script:1: symbol not found: bar .global _start _start: diff --git a/lld/test/ELF/linkerscript/sections-padding.s b/lld/test/ELF/linkerscript/sections-padding.s index 4d147d79c63e6..5b5767498b09c 100644 --- a/lld/test/ELF/linkerscript/sections-padding.s +++ b/lld/test/ELF/linkerscript/sections-padding.s @@ -33,8 +33,12 @@ ## Invalid hex value: # RUN: echo "SECTIONS { .mysec : { *(.mysec*) } =0x99XX }" > %t.script # RUN: not ld.lld -o /dev/null --script %t.script %t 2>&1 \ -# RUN: | FileCheck --check-prefix=ERR2 %s -# ERR2: malformed number: 0x99XX +# RUN: | FileCheck --check-prefix=ERR2 %s --implicit-check-not=error: +# ERR2: error: {{.*}}.script:1: malformed number: 0x99XX +# ERR2-NEXT: >>> SECTIONS { .mysec : { *(.mysec*) } =0x99XX } +# ERR2-NEXT: >>> ^ +# ERR2-EMPTY: +# ERR2-NEXT: error: {{.*}}.script:1: symbol not found: 0x99XX ## Check case with space between '=' and a value: # RUN: echo "SECTIONS { .mysec : { *(.mysec*) } = 0x1122 }" > %t.script @@ -58,13 +62,13 @@ ## Check we report an error if expression value is larger than 32-bits. # RUN: echo "SECTIONS { .mysec : { *(.mysec*) } =(0x11 << 32) }" > %t.script -# RUN: not ld.lld -o /dev/null --script %t.script %t 2>&1 | FileCheck --check-prefix=ERR3 %s -# ERR3: filler expression result does not fit 32-bit: 0x1100000000 +# RUN: not ld.lld -o /dev/null --script %t.script %t 2>&1 | FileCheck --check-prefix=ERR3 %s --implicit-check-not=error: +# ERR3: error: {{.*}}.script:1: filler expression result does not fit 32-bit: 0x1100000000 ## Check we report an error if an expression use a symbol. # RUN: echo "SECTIONS { foo = 0x11; .mysec : { *(.mysec*) } = foo }" > %t.script -# RUN: not ld.lld -o /dev/null %t --script %t.script 2>&1 | FileCheck --check-prefix=ERR4 %s -# ERR4: symbol not found: foo +# RUN: not ld.lld -o /dev/null %t --script %t.script 2>&1 | FileCheck --check-prefix=ERR4 %s --implicit-check-not=error: +# ERR4: error: {{.*}}.script:1: symbol not found: foo ## Check we are able to parse scripts where "/DISCARD/" follows a section fill expression. # RUN: echo "SECTIONS { .mysec : { *(.mysec*) } =0x1122 /DISCARD/ : { *(.text) } }" > %t.script diff --git a/lld/test/ELF/linkerscript/symbol-assignexpr.s b/lld/test/ELF/linkerscript/symbol-assignexpr.s index ed6d746b3e2d1..7f1d9b702d057 100644 --- a/lld/test/ELF/linkerscript/symbol-assignexpr.s +++ b/lld/test/ELF/linkerscript/symbol-assignexpr.s @@ -2,9 +2,27 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o # RUN: echo "SECTIONS { symbol2 = symbol; }" > %t2.script -# RUN: not ld.lld -o /dev/null -T %t2.script %t.o 2>&1 \ -# RUN: | FileCheck -check-prefix=ERR %s -# ERR: {{.*}}.script:1: symbol not found: symbol +# RUN: not ld.lld -o /dev/null -T %t2.script %t.o -Map=%t.map 2>&1 \ +# RUN: | FileCheck --check-prefix=ERR %s --implicit-check-not=error: +# RUN: FileCheck --input-file=%t.map %s --check-prefix=MAP +# RUN: not ld.lld -o /dev/null --noinhibit-exec -T %t2.script %t.o 2>&1 \ +# RUN: | FileCheck --check-prefix=ERR %s --implicit-check-not=error: + +# ERR-COUNT-3: {{.*}}.script:1: symbol not found: symbol + +# MAP: VMA LMA Size Align Out In Symbol +# MAP-NEXT: 0 0 0 1 symbol2 = symbol +# MAP-NEXT: 0 0 1 4 .text +# MAP-NEXT: 0 0 1 4 {{.*}}.o:(.text) +# MAP-NEXT: 0 0 0 1 _start +# MAP-NEXT: 0 0 8 1 .comment +# MAP-NEXT: 0 0 8 1 :(.comment) +# MAP-NEXT: 0 0 60 8 .symtab +# MAP-NEXT: 0 0 60 8 :(.symtab) +# MAP-NEXT: 0 0 2a 1 .shstrtab +# MAP-NEXT: 0 0 2a 1 :(.shstrtab) +# MAP-NEXT: 0 0 17 1 .strtab +# MAP-NEXT: 0 0 17 1 :(.strtab) .global _start _start: From 744f38913fa380580431df0ae89ef5fb3df30240 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 14:39:37 -0800 Subject: [PATCH 042/884] [lldb] Use StringRef::{starts,ends}_with (NFC) This patch replaces uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- lldb/include/lldb/Utility/CompletionRequest.h | 2 +- .../Breakpoint/BreakpointResolverFileLine.cpp | 2 +- lldb/source/Commands/CommandCompletions.cpp | 4 +-- .../source/Commands/CommandObjectCommands.cpp | 2 +- lldb/source/Commands/CommandObjectMemory.cpp | 2 +- lldb/source/Commands/CommandObjectThread.cpp | 6 ++-- lldb/source/Commands/CommandObjectType.cpp | 2 +- lldb/source/Core/IOHandlerCursesGUI.cpp | 2 +- lldb/source/Core/Mangled.cpp | 20 ++++++------- lldb/source/Core/Module.cpp | 2 +- lldb/source/Core/PluginManager.cpp | 4 +-- lldb/source/Core/RichManglingContext.cpp | 2 +- lldb/source/Core/ValueObject.cpp | 2 +- lldb/source/Expression/IRExecutionUnit.cpp | 4 +-- lldb/source/Expression/REPL.cpp | 2 +- lldb/source/Interpreter/CommandAlias.cpp | 2 +- lldb/source/Interpreter/OptionArgParser.cpp | 2 +- lldb/source/Interpreter/Options.cpp | 8 ++--- .../ExpressionParser/Clang/ClangASTSource.cpp | 4 +-- .../Clang/ClangExpressionDeclMap.cpp | 6 ++-- .../Clang/ClangExpressionParser.cpp | 10 +++---- .../Clang/CppModuleConfiguration.cpp | 2 +- .../ExpressionParser/Clang/IRForTarget.cpp | 16 +++++----- .../Language/CPlusPlus/CPlusPlusLanguage.cpp | 2 +- .../Language/CPlusPlus/LibCxxUnorderedMap.cpp | 2 +- .../Language/CPlusPlus/LibStdcppTuple.cpp | 4 +-- .../Plugins/Language/ObjC/NSDictionary.cpp | 2 +- .../Plugins/Language/ObjC/ObjCLanguage.cpp | 4 +-- .../CPlusPlus/CPPLanguageRuntime.cpp | 8 ++--- .../ItaniumABI/ItaniumABILanguageRuntime.cpp | 4 +-- .../AppleObjCRuntime/AppleObjCRuntimeV2.cpp | 6 ++-- .../ObjectFile/Mach-O/ObjectFileMachO.cpp | 29 ++++++++++--------- .../Linux/IntelPTSingleBufferTrace.cpp | 2 +- .../Process/elf-core/ProcessElfCore.cpp | 2 +- .../GDBRemoteCommunicationClient.cpp | 2 +- .../GDBRemoteCommunicationServerLLGS.cpp | 6 ++-- .../Process/gdb-remote/ProcessGDBRemote.cpp | 4 +-- .../Plugins/SymbolFile/DWARF/DWARFUnit.cpp | 2 +- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 2 +- .../SymbolFile/NativePDB/CompileUnitIndex.cpp | 2 +- .../SymbolFile/NativePDB/PdbAstBuilder.cpp | 4 +-- .../NativePDB/SymbolFileNativePDB.cpp | 2 +- .../TypeSystem/Clang/TypeSystemClang.cpp | 2 +- lldb/source/Symbol/ObjectFile.cpp | 10 +++---- lldb/source/Symbol/Symbol.cpp | 2 +- lldb/source/Symbol/Symtab.cpp | 2 +- lldb/source/Symbol/Variable.cpp | 4 +-- lldb/source/Target/TargetList.cpp | 2 +- lldb/source/Utility/Args.cpp | 2 +- lldb/source/Utility/CompletionRequest.cpp | 4 +-- lldb/source/Utility/FileSpec.cpp | 4 +-- lldb/source/Utility/FileSpecList.cpp | 2 +- lldb/source/Utility/NameMatches.cpp | 4 +-- lldb/source/Utility/StringExtractor.cpp | 2 +- .../Utility/TildeExpressionResolver.cpp | 4 +-- lldb/source/Utility/XcodeSDK.cpp | 4 +-- lldb/tools/lldb-dap/IOStream.cpp | 2 +- lldb/tools/lldb-dap/JSONUtils.cpp | 5 ++-- lldb/tools/lldb-dap/lldb-dap.cpp | 2 +- .../Expression/ClangExpressionDeclMapTest.cpp | 2 +- .../minidump/RegisterContextMinidumpTest.cpp | 2 +- .../MockTildeExpressionResolver.cpp | 2 +- 62 files changed, 132 insertions(+), 128 deletions(-) diff --git a/lldb/include/lldb/Utility/CompletionRequest.h b/lldb/include/lldb/Utility/CompletionRequest.h index 1fbc96944e82d..1a2b1d639950f 100644 --- a/lldb/include/lldb/Utility/CompletionRequest.h +++ b/lldb/include/lldb/Utility/CompletionRequest.h @@ -184,7 +184,7 @@ class CompletionRequest { // this can be a static_assert. static_assert(M != CompletionMode::RewriteLine, "Shouldn't rewrite line with this function"); - if (completion.startswith(GetCursorArgumentPrefix())) + if (completion.starts_with(GetCursorArgumentPrefix())) AddCompletion(completion, description, M); } diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp index 61bef498438bd..cc4e1d26724f0 100644 --- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp @@ -206,7 +206,7 @@ void BreakpointResolverFileLine::DeduceSourceMapping( [path_separator](llvm::StringRef a, llvm::StringRef b, bool case_sensitive) -> std::optional { if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b)) { - if (a.empty() || a.endswith(path_separator)) { + if (a.empty() || a.ends_with(path_separator)) { return a; } } diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp index e766a6c8c10bc..16078a92ab5fe 100644 --- a/lldb/source/Commands/CommandCompletions.cpp +++ b/lldb/source/Commands/CommandCompletions.cpp @@ -424,7 +424,7 @@ static void DiskFilesOrDirectories(const llvm::Twine &partial_name, auto Name = path::filename(Entry.path()); // Omit ".", ".." - if (Name == "." || Name == ".." || !Name.startswith(PartialItem)) + if (Name == "." || Name == ".." || !Name.starts_with(PartialItem)) continue; bool is_dir = Status->isDirectory(); @@ -608,7 +608,7 @@ void CommandCompletions::Registers(CommandInterpreter &interpreter, CompletionRequest &request, SearchFilter *searcher) { std::string reg_prefix; - if (request.GetCursorArgumentPrefix().startswith("$")) + if (request.GetCursorArgumentPrefix().starts_with("$")) reg_prefix = "$"; RegisterContext *reg_ctx = diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp index 74d97b0db16cb..5b9af4a3e1b88 100644 --- a/lldb/source/Commands/CommandObjectCommands.cpp +++ b/lldb/source/Commands/CommandObjectCommands.cpp @@ -411,7 +411,7 @@ rather than using a positional placeholder:" // Get the alias command. auto alias_command = args[0].ref(); - if (alias_command.startswith("-")) { + if (alias_command.starts_with("-")) { result.AppendError("aliases starting with a dash are not supported"); if (alias_command == "--help" || alias_command == "--long-help") { result.AppendWarning("if trying to pass options to 'command alias' add " diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp index 4ecac732d0dca..b78a0492cca55 100644 --- a/lldb/source/Commands/CommandObjectMemory.cpp +++ b/lldb/source/Commands/CommandObjectMemory.cpp @@ -1421,7 +1421,7 @@ class CommandObjectMemoryWrite : public CommandObjectParsed { // Be careful, getAsInteger with a radix of 16 rejects "0xab" so we // have to special case that: bool success = false; - if (entry.ref().startswith("0x")) + if (entry.ref().starts_with("0x")) success = !entry.ref().getAsInteger(0, uval64); if (!success) success = !entry.ref().getAsInteger(16, uval64); diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp index dd9fab4bbddab..a1e7e3f11361e 100644 --- a/lldb/source/Commands/CommandObjectThread.cpp +++ b/lldb/source/Commands/CommandObjectThread.cpp @@ -145,14 +145,14 @@ class CommandObjectThreadBacktrace : public CommandObjectIterateOverThreads { for (size_t idx = 0; idx < num_entries; idx++) { llvm::StringRef arg_string = copy_args[idx].ref(); - if (arg_string.equals("-c") || count_opt.startswith(arg_string)) { + if (arg_string.equals("-c") || count_opt.starts_with(arg_string)) { idx++; if (idx == num_entries) return std::nullopt; count_idx = idx; if (copy_args[idx].ref().getAsInteger(0, count_val)) return std::nullopt; - } else if (arg_string.equals("-s") || start_opt.startswith(arg_string)) { + } else if (arg_string.equals("-s") || start_opt.starts_with(arg_string)) { idx++; if (idx == num_entries) return std::nullopt; @@ -1575,7 +1575,7 @@ class CommandObjectThreadReturn : public CommandObjectRaw { // I am going to handle this by hand, because I don't want you to have to // say: // "thread return -- -5". - if (command.startswith("-x")) { + if (command.starts_with("-x")) { if (command.size() != 2U) result.AppendWarning("Return values ignored when returning from user " "called expressions"); diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp index 411dc2fb723ce..f76420f3cc683 100644 --- a/lldb/source/Commands/CommandObjectType.cpp +++ b/lldb/source/Commands/CommandObjectType.cpp @@ -1570,7 +1570,7 @@ void CommandObjectTypeSummaryAdd::DoExecute(Args &command, static bool FixArrayTypeNameWithRegex(ConstString &type_name) { llvm::StringRef type_name_ref(type_name.GetStringRef()); - if (type_name_ref.endswith("[]")) { + if (type_name_ref.ends_with("[]")) { std::string type_name_str(type_name.GetCString()); type_name_str.resize(type_name_str.length() - 2); if (type_name_str.back() != ' ') diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp index abf0b6b801f37..620e68a28510e 100644 --- a/lldb/source/Core/IOHandlerCursesGUI.cpp +++ b/lldb/source/Core/IOHandlerCursesGUI.cpp @@ -7052,7 +7052,7 @@ class SourceFileWindowDelegate : public WindowDelegate { m_file_sp->DisplaySourceLines(curr_line + 1, column, 0, 0, &lineStream); StringRef line = lineStream.GetString(); - if (line.endswith("\n")) + if (line.ends_with("\n")) line = line.drop_back(); bool wasWritten = window.OutputColoredStringTruncated( 1, line, m_first_visible_column, is_pc_line); diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp index 4587119519e98..23ae3913093fa 100644 --- a/lldb/source/Core/Mangled.cpp +++ b/lldb/source/Core/Mangled.cpp @@ -42,20 +42,20 @@ Mangled::ManglingScheme Mangled::GetManglingScheme(llvm::StringRef const name) { if (name.empty()) return Mangled::eManglingSchemeNone; - if (name.startswith("?")) + if (name.starts_with("?")) return Mangled::eManglingSchemeMSVC; - if (name.startswith("_R")) + if (name.starts_with("_R")) return Mangled::eManglingSchemeRustV0; - if (name.startswith("_D")) + if (name.starts_with("_D")) return Mangled::eManglingSchemeD; - if (name.startswith("_Z")) + if (name.starts_with("_Z")) return Mangled::eManglingSchemeItanium; // ___Z is a clang extension of block invocations - if (name.startswith("___Z")) + if (name.starts_with("___Z")) return Mangled::eManglingSchemeItanium; // Swift's older style of mangling used "_T" as a mangling prefix. This can @@ -64,16 +64,16 @@ Mangled::ManglingScheme Mangled::GetManglingScheme(llvm::StringRef const name) { // for select old-style swift mangled names. The known cases are ObjC classes // and protocols. Classes are either prefixed with "_TtC" or "_TtGC". // Protocols are prefixed with "_TtP". - if (name.startswith("_TtC") || name.startswith("_TtGC") || - name.startswith("_TtP")) + if (name.starts_with("_TtC") || name.starts_with("_TtGC") || + name.starts_with("_TtP")) return Mangled::eManglingSchemeSwift; // Swift 4.2 used "$S" and "_$S". // Swift 5 and onward uses "$s" and "_$s". // Swift also uses "@__swiftmacro_" as a prefix for mangling filenames. - if (name.startswith("$S") || name.startswith("_$S") || - name.startswith("$s") || name.startswith("_$s") || - name.startswith("@__swiftmacro_")) + if (name.starts_with("$S") || name.starts_with("_$S") || + name.starts_with("$s") || name.starts_with("_$s") || + name.starts_with("@__swiftmacro_")) return Mangled::eManglingSchemeSwift; return Mangled::eManglingSchemeNone; diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp index 65a65c455efa7..c0574b724ace7 100644 --- a/lldb/source/Core/Module.cpp +++ b/lldb/source/Core/Module.cpp @@ -1354,7 +1354,7 @@ void Module::SetSymbolFileFileSpec(const FileSpec &file) { if (FileSystem::Instance().IsDirectory(file)) { std::string new_path(file.GetPath()); std::string old_path(obj_file->GetFileSpec().GetPath()); - if (llvm::StringRef(old_path).startswith(new_path)) { + if (llvm::StringRef(old_path).starts_with(new_path)) { // We specified the same bundle as the symbol file that we already // have return; diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp index dea380e47f4ee..b428370d7f333 100644 --- a/lldb/source/Core/PluginManager.cpp +++ b/lldb/source/Core/PluginManager.cpp @@ -821,7 +821,7 @@ PluginManager::GetPlatformCreateCallbackForPluginName(llvm::StringRef name) { void PluginManager::AutoCompletePlatformName(llvm::StringRef name, CompletionRequest &request) { for (const auto &instance : GetPlatformInstances().GetInstances()) { - if (instance.name.startswith(name)) + if (instance.name.starts_with(name)) request.AddCompletion(instance.name); } } @@ -869,7 +869,7 @@ PluginManager::GetProcessCreateCallbackForPluginName(llvm::StringRef name) { void PluginManager::AutoCompleteProcessName(llvm::StringRef name, CompletionRequest &request) { for (const auto &instance : GetProcessInstances().GetInstances()) { - if (instance.name.startswith(name)) + if (instance.name.starts_with(name)) request.AddCompletion(instance.name, instance.description); } } diff --git a/lldb/source/Core/RichManglingContext.cpp b/lldb/source/Core/RichManglingContext.cpp index 08c9b280b8ccb..b68c9e11581b4 100644 --- a/lldb/source/Core/RichManglingContext.cpp +++ b/lldb/source/Core/RichManglingContext.cpp @@ -72,7 +72,7 @@ bool RichManglingContext::IsCtorOrDtor() const { // We can only check for destructors here. auto base_name = get(m_cxx_method_parser)->GetBasename(); - return base_name.startswith("~"); + return base_name.starts_with("~"); } case None: return false; diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp index b13bffa0ca809..b82e6082eebdd 100644 --- a/lldb/source/Core/ValueObject.cpp +++ b/lldb/source/Core/ValueObject.cpp @@ -2112,7 +2112,7 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl( *final_result = ValueObject::eExpressionPathEndResultTypeInvalid; return ValueObjectSP(); } - if (!temp_expression.startswith(">")) { + if (!temp_expression.starts_with(">")) { *reason_to_stop = ValueObject::eExpressionPathScanEndReasonUnexpectedSymbol; *final_result = ValueObject::eExpressionPathEndResultTypeInvalid; diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp index 562bf3cdd2ed0..0682746e448e3 100644 --- a/lldb/source/Expression/IRExecutionUnit.cpp +++ b/lldb/source/Expression/IRExecutionUnit.cpp @@ -537,7 +537,7 @@ lldb::SectionType IRExecutionUnit::GetSectionTypeFromSectionName( sect_type = lldb::eSectionTypeCode; else if (name.equals("__data") || name.equals(".data")) sect_type = lldb::eSectionTypeCode; - else if (name.startswith("__debug_") || name.startswith(".debug_")) { + else if (name.starts_with("__debug_") || name.starts_with(".debug_")) { const uint32_t name_idx = name[0] == '_' ? 8 : 7; llvm::StringRef dwarf_name(name.substr(name_idx)); switch (dwarf_name[0]) { @@ -596,7 +596,7 @@ lldb::SectionType IRExecutionUnit::GetSectionTypeFromSectionName( default: break; } - } else if (name.startswith("__apple_") || name.startswith(".apple_")) + } else if (name.starts_with("__apple_") || name.starts_with(".apple_")) sect_type = lldb::eSectionTypeInvalid; else if (name.equals("__objc_imageinfo")) sect_type = lldb::eSectionTypeOther; diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp index 07d5b5b3dd934..a6a4ffb5e0af9 100644 --- a/lldb/source/Expression/REPL.cpp +++ b/lldb/source/Expression/REPL.cpp @@ -497,7 +497,7 @@ void REPL::IOHandlerInputComplete(IOHandler &io_handler, std::string &code) { void REPL::IOHandlerComplete(IOHandler &io_handler, CompletionRequest &request) { // Complete an LLDB command if the first character is a colon... - if (request.GetRawLine().startswith(":")) { + if (request.GetRawLine().starts_with(":")) { Debugger &debugger = m_target.GetDebugger(); // auto complete LLDB commands diff --git a/lldb/source/Interpreter/CommandAlias.cpp b/lldb/source/Interpreter/CommandAlias.cpp index b95d3c91fcbc2..c5971b52f837f 100644 --- a/lldb/source/Interpreter/CommandAlias.cpp +++ b/lldb/source/Interpreter/CommandAlias.cpp @@ -182,7 +182,7 @@ bool CommandAlias::IsDashDashCommand() { for (const auto &opt_entry : *GetOptionArguments()) { std::tie(opt, std::ignore, value) = opt_entry; if (opt == CommandInterpreter::g_argument && !value.empty() && - llvm::StringRef(value).endswith("--")) { + llvm::StringRef(value).ends_with("--")) { m_is_dashdash_alias = eLazyBoolYes; break; } diff --git a/lldb/source/Interpreter/OptionArgParser.cpp b/lldb/source/Interpreter/OptionArgParser.cpp index 8a92c7d08c476..d13805a75ffbf 100644 --- a/lldb/source/Interpreter/OptionArgParser.cpp +++ b/lldb/source/Interpreter/OptionArgParser.cpp @@ -61,7 +61,7 @@ int64_t OptionArgParser::ToOptionEnum(llvm::StringRef s, for (const auto &enum_value : enum_values) { llvm::StringRef this_enum(enum_value.string_value); - if (this_enum.startswith(s)) + if (this_enum.starts_with(s)) return enum_value.value; } diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp index acbde7660440b..89fe69009d903 100644 --- a/lldb/source/Interpreter/Options.cpp +++ b/lldb/source/Interpreter/Options.cpp @@ -636,7 +636,7 @@ bool Options::HandleOptionCompletion(CompletionRequest &request, // upper level code will know this is a full match and add the " ". const OptionDefinition &opt = opt_defs[opt_defs_index]; llvm::StringRef long_option = opt.long_option; - if (cur_opt_str.startswith("--") && cur_opt_str != long_option) { + if (cur_opt_str.starts_with("--") && cur_opt_str != long_option) { request.AddCompletion("--" + long_option.str(), opt.usage_text); return true; } else @@ -652,7 +652,7 @@ bool Options::HandleOptionCompletion(CompletionRequest &request, if (cur_opt_str.consume_front("--")) { for (auto &def : opt_defs) { llvm::StringRef long_option(def.long_option); - if (long_option.startswith(cur_opt_str)) + if (long_option.starts_with(cur_opt_str)) request.AddCompletion("--" + long_option.str(), def.usage_text); } } @@ -890,8 +890,8 @@ static size_t FindArgumentIndexForOption(const Args &args, std::string long_opt = std::string(llvm::formatv("--{0}", long_option.definition->long_option)); for (const auto &entry : llvm::enumerate(args)) { - if (entry.value().ref().startswith(short_opt) || - entry.value().ref().startswith(long_opt)) + if (entry.value().ref().starts_with(short_opt) || + entry.value().ref().starts_with(long_opt)) return entry.index(); } diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp index 00ab6a04bd323..79dd306f7627f 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp @@ -571,8 +571,8 @@ bool ClangASTSource::IgnoreName(const ConstString name, // The ClangASTSource is not responsible for finding $-names. return name_string_ref.empty() || - (ignore_all_dollar_names && name_string_ref.startswith("$")) || - name_string_ref.startswith("_$"); + (ignore_all_dollar_names && name_string_ref.starts_with("$")) || + name_string_ref.starts_with("_$"); } void ClangASTSource::FindExternalVisibleDecls( diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp index 6fbc0bb22f82b..2d306b42760b1 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp @@ -1369,7 +1369,7 @@ void ClangExpressionDeclMap::FindExternalVisibleDecls( if (!namespace_decl) SearchPersistenDecls(context, name); - if (name.GetStringRef().startswith("$") && !namespace_decl) { + if (name.GetStringRef().starts_with("$") && !namespace_decl) { if (name == "$__lldb_class") { LookUpLldbClass(context); return; @@ -1385,7 +1385,7 @@ void ClangExpressionDeclMap::FindExternalVisibleDecls( } // any other $__lldb names should be weeded out now - if (name.GetStringRef().startswith("$__lldb")) + if (name.GetStringRef().starts_with("$__lldb")) return; // No ParserVars means we can't do register or variable lookup. @@ -1400,7 +1400,7 @@ void ClangExpressionDeclMap::FindExternalVisibleDecls( return; } - assert(name.GetStringRef().startswith("$")); + assert(name.GetStringRef().starts_with("$")); llvm::StringRef reg_name = name.GetStringRef().substr(1); if (m_parser_vars->m_exe_ctx.GetRegisterContext()) { diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp index f6856b1a2558c..574d661e2a215 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp @@ -834,13 +834,13 @@ class CodeComplete : public CodeCompleteConsumer { case CodeCompletionResult::RK_Declaration: return !( Result.Declaration->getIdentifier() && - Result.Declaration->getIdentifier()->getName().startswith(Filter)); + Result.Declaration->getIdentifier()->getName().starts_with(Filter)); case CodeCompletionResult::RK_Keyword: - return !StringRef(Result.Keyword).startswith(Filter); + return !StringRef(Result.Keyword).starts_with(Filter); case CodeCompletionResult::RK_Macro: - return !Result.Macro->getName().startswith(Filter); + return !Result.Macro->getName().starts_with(Filter); case CodeCompletionResult::RK_Pattern: - return !StringRef(Result.Pattern->getAsString()).startswith(Filter); + return !StringRef(Result.Pattern->getAsString()).starts_with(Filter); } // If we trigger this assert or the above switch yields a warning, then // CodeCompletionResult has been enhanced with more kinds of completion @@ -904,7 +904,7 @@ class CodeComplete : public CodeCompleteConsumer { } // We also filter some internal lldb identifiers here. The user // shouldn't see these. - if (llvm::StringRef(ToInsert).startswith("$__lldb_")) + if (llvm::StringRef(ToInsert).starts_with("$__lldb_")) return std::nullopt; if (ToInsert.empty()) return std::nullopt; diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp index 847dab6592b88..62443d1290dc7 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp @@ -72,7 +72,7 @@ bool CppModuleConfiguration::analyzeFile(const FileSpec &f, // path. Ignore subdirectories such as /c++/v1/experimental as those don't // need to be specified in the header search. if (libcpp_regex.match(f.GetPath()) && - parent_path(posix_dir, Style::posix).endswith("c++")) { + parent_path(posix_dir, Style::posix).ends_with("c++")) { if (!m_std_inc.TrySet(posix_dir)) return false; if (triple.str().empty()) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp index 33e5dd0015aeb..597873af8b2ae 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp @@ -154,9 +154,10 @@ clang::NamedDecl *IRForTarget::DeclForGlobal(GlobalValue *global_val) { /// Returns true iff the mangled symbol is for a static guard variable. static bool isGuardVariableSymbol(llvm::StringRef mangled_symbol, bool check_ms_abi = true) { - bool result = mangled_symbol.startswith("_ZGV"); // Itanium ABI guard variable + bool result = + mangled_symbol.starts_with("_ZGV"); // Itanium ABI guard variable if (check_ms_abi) - result |= mangled_symbol.endswith("@4IA"); // Microsoft ABI + result |= mangled_symbol.ends_with("@4IA"); // Microsoft ABI return result; } @@ -720,8 +721,9 @@ bool IRForTarget::RewriteObjCConstStrings() { static bool IsObjCSelectorRef(Value *value) { GlobalVariable *global_variable = dyn_cast(value); - return !(!global_variable || !global_variable->hasName() || - !global_variable->getName().startswith("OBJC_SELECTOR_REFERENCES_")); + return !( + !global_variable || !global_variable->hasName() || + !global_variable->getName().starts_with("OBJC_SELECTOR_REFERENCES_")); } // This function does not report errors; its callers are responsible. @@ -940,7 +942,7 @@ bool IRForTarget::RewritePersistentAllocs(llvm::BasicBlock &basic_block) { if (AllocaInst *alloc = dyn_cast(&inst)) { llvm::StringRef alloc_name = alloc->getName(); - if (alloc_name.startswith("$") && !alloc_name.startswith("$__lldb")) { + if (alloc_name.starts_with("$") && !alloc_name.starts_with("$__lldb")) { if (alloc_name.find_first_of("0123456789") == 1) { LLDB_LOG(log, "Rejecting a numeric persistent variable."); @@ -1017,7 +1019,7 @@ bool IRForTarget::MaybeHandleVariable(Value *llvm_value_ptr) { const Type *value_type = nullptr; - if (name.startswith("$")) { + if (name.starts_with("$")) { // The $__lldb_expr_result name indicates the return value has allocated // as a static variable. Per the comment at // ASTResultSynthesizer::SynthesizeBodyResult, accesses to this static @@ -1223,7 +1225,7 @@ bool IRForTarget::ResolveExternals(Function &llvm_function) { LLDB_LOG(log, "Examining {0}, DeclForGlobalValue returns {1}", global_name, static_cast(DeclForGlobal(&global_var))); - if (global_name.startswith("OBJC_IVAR")) { + if (global_name.starts_with("OBJC_IVAR")) { if (!HandleSymbol(&global_var)) { m_error_stream.Format("Error [IRForTarget]: Couldn't find Objective-C " "indirect ivar symbol {0}\n", diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp index f4537b4133b93..586cc08a6f123 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp @@ -466,7 +466,7 @@ class ManglingSubstitutor } void trySubstitute(llvm::StringRef From, llvm::StringRef To) { - if (!llvm::StringRef(currentParserPos(), this->numLeft()).startswith(From)) + if (!llvm::StringRef(currentParserPos(), this->numLeft()).starts_with(From)) return; // We found a match. Append unmodified input up to this point. diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp index 2e8da396a4a7b..ff7043bdf97ff 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp @@ -84,7 +84,7 @@ static bool isStdTemplate(ConstString type_name, llvm::StringRef type) { // The type name may be prefixed with `std::__::`. if (name.consume_front("std::")) consumeInlineNamespace(name); - return name.consume_front(type) && name.startswith("<"); + return name.consume_front(type) && name.starts_with("<"); } static bool isUnorderedMap(ConstString type_name) { diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp index aef7cbac603f2..f1bfeae5099b7 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp @@ -69,9 +69,9 @@ bool LibStdcppTupleSyntheticFrontEnd::Update() { for (size_t i = 0; i < child_count; ++i) { ValueObjectSP child_sp = current_child->GetChildAtIndex(i); llvm::StringRef name_str = child_sp->GetName().GetStringRef(); - if (name_str.startswith("std::_Tuple_impl<")) { + if (name_str.starts_with("std::_Tuple_impl<")) { next_child_sp = child_sp; - } else if (name_str.startswith("std::_Head_base<")) { + } else if (name_str.starts_with("std::_Head_base<")) { ValueObjectSP value_sp = child_sp->GetChildMemberWithName("_M_head_impl"); if (value_sp) { diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp index 605c79cbd9b55..5ae0751cb065f 100644 --- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp @@ -37,7 +37,7 @@ NSDictionary_Additionals::AdditionalFormatterMatching::Prefix::Prefix( bool NSDictionary_Additionals::AdditionalFormatterMatching::Prefix::Match( ConstString class_name) { - return class_name.GetStringRef().startswith(m_prefix.GetStringRef()); + return class_name.GetStringRef().starts_with(m_prefix.GetStringRef()); } NSDictionary_Additionals::AdditionalFormatterMatching::Full::Full(ConstString n) diff --git a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp index 82b037129c244..742ae7b149454 100644 --- a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp +++ b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp @@ -81,9 +81,9 @@ ObjCLanguage::MethodName::Create(llvm::StringRef name, bool strict) { // Figure out type Type type = eTypeUnspecified; - if (name.startswith("+[")) + if (name.starts_with("+[")) type = eTypeClassMethod; - else if (name.startswith("-[")) + else if (name.starts_with("-[")) type = eTypeInstanceMethod; // If there's no type and it's strict, this is invalid diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp index e65b99f44be6d..300ecc8e8ed58 100644 --- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.cpp @@ -220,7 +220,7 @@ CPPLanguageRuntime::FindLibCppStdFunctionCallableInfo( llvm::StringRef vtable_name(symbol->GetName().GetStringRef()); bool found_expected_start_string = - vtable_name.startswith("vtable for std::__1::__function::__func<"); + vtable_name.starts_with("vtable for std::__1::__function::__func<"); if (!found_expected_start_string) return optional_info; @@ -277,7 +277,7 @@ CPPLanguageRuntime::FindLibCppStdFunctionCallableInfo( } // Case 4 or 5 - if (symbol && !symbol->GetName().GetStringRef().startswith("vtable for") && + if (symbol && !symbol->GetName().GetStringRef().starts_with("vtable for") && !contains_lambda_identifier(first_template_parameter) && !has_invoke) { optional_info.callable_case = LibCppStdFunctionCallableCase::FreeOrMemberFunction; @@ -312,7 +312,7 @@ CPPLanguageRuntime::FindLibCppStdFunctionCallableInfo( lldb::FunctionSP func_sp = vtable_cu->FindFunction([name_to_use](const FunctionSP &f) { auto name = f->GetName().GetStringRef(); - if (name.startswith(name_to_use) && name.contains("operator")) + if (name.starts_with(name_to_use) && name.contains("operator")) return true; return false; @@ -373,7 +373,7 @@ CPPLanguageRuntime::GetStepThroughTrampolinePlan(Thread &thread, // step into the wrapped callable. // bool found_expected_start_string = - function_name.startswith("std::__1::function<"); + function_name.starts_with("std::__1::function<"); if (!found_expected_start_string) return ret_plan_sp; diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp index 0ea9201901ab3..47b1db16f1e90 100644 --- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp @@ -274,7 +274,7 @@ llvm::Expected "no symbol found for 0x%" PRIx64, vtable_load_addr); llvm::StringRef name = symbol->GetMangled().GetDemangledName().GetStringRef(); - if (name.startswith(vtable_demangled_prefix)) { + if (name.starts_with(vtable_demangled_prefix)) { VTableInfo info = {vtable_addr, symbol}; std::lock_guard locker(m_mutex); auto pos = m_vtable_info_map[vtable_addr] = info; @@ -450,7 +450,7 @@ class CommandObjectMultiwordItaniumABI_Demangle : public CommandObjectParsed { // on behalf of the user. This is the moral equivalent of the -_/-n // options to c++filt auto name = entry.ref(); - if (name.startswith("__Z")) + if (name.starts_with("__Z")) name = name.drop_front(); Mangled mangled(name); diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp index 1fd7d027731de..dc492ac0f06d3 100644 --- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp +++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp @@ -2641,7 +2641,7 @@ static bool DoesProcessHaveSharedCache(Process &process) { return true; // this should not happen llvm::StringRef platform_plugin_name_sr = platform_sp->GetPluginName(); - if (platform_plugin_name_sr.endswith("-simulator")) + if (platform_plugin_name_sr.ends_with("-simulator")) return false; return true; @@ -2731,7 +2731,7 @@ lldb::addr_t AppleObjCRuntimeV2::LookupRuntimeSymbol(ConstString name) { llvm::StringRef ivar_prefix("OBJC_IVAR_$_"); llvm::StringRef class_prefix("OBJC_CLASS_$_"); - if (name_strref.startswith(ivar_prefix)) { + if (name_strref.starts_with(ivar_prefix)) { llvm::StringRef ivar_skipped_prefix = name_strref.substr(ivar_prefix.size()); std::pair class_and_ivar = @@ -2764,7 +2764,7 @@ lldb::addr_t AppleObjCRuntimeV2::LookupRuntimeSymbol(ConstString name) { ivar_func); } } - } else if (name_strref.startswith(class_prefix)) { + } else if (name_strref.starts_with(class_prefix)) { llvm::StringRef class_skipped_prefix = name_strref.substr(class_prefix.size()); const ConstString class_name_cs(class_skipped_prefix); diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp index 24f3939a8f2ba..58275c052f74e 100644 --- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp +++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp @@ -2164,20 +2164,20 @@ static SymbolType GetSymbolType(const char *&symbol_name, if (symbol_name) { llvm::StringRef symbol_name_ref(symbol_name); - if (symbol_name_ref.startswith("OBJC_")) { + if (symbol_name_ref.starts_with("OBJC_")) { static const llvm::StringRef g_objc_v2_prefix_class("OBJC_CLASS_$_"); static const llvm::StringRef g_objc_v2_prefix_metaclass( "OBJC_METACLASS_$_"); static const llvm::StringRef g_objc_v2_prefix_ivar("OBJC_IVAR_$_"); - if (symbol_name_ref.startswith(g_objc_v2_prefix_class)) { + if (symbol_name_ref.starts_with(g_objc_v2_prefix_class)) { symbol_name = symbol_name + g_objc_v2_prefix_class.size(); type = eSymbolTypeObjCClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith(g_objc_v2_prefix_metaclass)) { + } else if (symbol_name_ref.starts_with(g_objc_v2_prefix_metaclass)) { symbol_name = symbol_name + g_objc_v2_prefix_metaclass.size(); type = eSymbolTypeObjCMetaClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith(g_objc_v2_prefix_ivar)) { + } else if (symbol_name_ref.starts_with(g_objc_v2_prefix_ivar)) { symbol_name = symbol_name + g_objc_v2_prefix_ivar.size(); type = eSymbolTypeObjCIVar; demangled_is_synthesized = true; @@ -3789,18 +3789,19 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { if (symbol_name && symbol_name[0] == '_' && symbol_name[1] == 'O') { llvm::StringRef symbol_name_ref(symbol_name); - if (symbol_name_ref.startswith(g_objc_v2_prefix_class)) { + if (symbol_name_ref.starts_with(g_objc_v2_prefix_class)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_class.size(); type = eSymbolTypeObjCClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith(g_objc_v2_prefix_metaclass)) { + } else if (symbol_name_ref.starts_with( + g_objc_v2_prefix_metaclass)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_metaclass.size(); type = eSymbolTypeObjCMetaClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith(g_objc_v2_prefix_ivar)) { + } else if (symbol_name_ref.starts_with(g_objc_v2_prefix_ivar)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_ivar.size(); type = eSymbolTypeObjCIVar; @@ -4250,27 +4251,27 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { if (symbol_name) { llvm::StringRef symbol_name_ref(symbol_name); - if (symbol_name_ref.startswith("_OBJC_")) { + if (symbol_name_ref.starts_with("_OBJC_")) { llvm::StringRef g_objc_v2_prefix_class( "_OBJC_CLASS_$_"); llvm::StringRef g_objc_v2_prefix_metaclass( "_OBJC_METACLASS_$_"); llvm::StringRef g_objc_v2_prefix_ivar( "_OBJC_IVAR_$_"); - if (symbol_name_ref.startswith(g_objc_v2_prefix_class)) { + if (symbol_name_ref.starts_with(g_objc_v2_prefix_class)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_class.size(); type = eSymbolTypeObjCClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith( + } else if (symbol_name_ref.starts_with( g_objc_v2_prefix_metaclass)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_metaclass.size(); type = eSymbolTypeObjCMetaClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith( + } else if (symbol_name_ref.starts_with( g_objc_v2_prefix_ivar)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = @@ -4297,7 +4298,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { llvm::StringRef symbol_name_ref(symbol_name); llvm::StringRef g_objc_v1_prefix_class( ".objc_class_name_"); - if (symbol_name_ref.startswith(g_objc_v1_prefix_class)) { + if (symbol_name_ref.starts_with(g_objc_v1_prefix_class)) { symbol_name_non_abi_mangled = symbol_name; symbol_name = symbol_name + g_objc_v1_prefix_class.size(); type = eSymbolTypeObjCClass; @@ -5163,10 +5164,10 @@ uint32_t ObjectFileMachO::GetDependentModules(FileSpecList &files) { std::string loader_path("@loader_path"); std::string executable_path("@executable_path"); for (auto &rpath : rpath_paths) { - if (llvm::StringRef(rpath).startswith(loader_path)) { + if (llvm::StringRef(rpath).starts_with(loader_path)) { rpath.erase(0, loader_path.size()); rpath.insert(0, this_file_spec.GetDirectory().GetCString()); - } else if (llvm::StringRef(rpath).startswith(executable_path)) { + } else if (llvm::StringRef(rpath).starts_with(executable_path)) { rpath.erase(0, executable_path.size()); rpath.insert(0, this_file_spec.GetDirectory().GetCString()); } diff --git a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp index 8b64412ddd98e..8c69989702c2a 100644 --- a/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp +++ b/lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp @@ -62,7 +62,7 @@ static Expected ReadIntelPTConfigFile(const char *file, if (type == BitOffset) { const char *prefix = "config:"; - if (!text_buffer.startswith(prefix)) + if (!text_buffer.starts_with(prefix)) return createStringError(inconvertibleErrorCode(), "The file '%s' contents doesn't start with '%s'", file, prefix); diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp index a4540de4acc45..7723009787f7f 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp @@ -832,7 +832,7 @@ llvm::Error ProcessElfCore::parseOpenBSDNotes(llvm::ArrayRef notes) { for (const auto ¬e : notes) { // OpenBSD per-thread information is stored in notes named "OpenBSD@nnn" so // match on the initial part of the string. - if (!llvm::StringRef(note.info.n_name).startswith("OpenBSD")) + if (!llvm::StringRef(note.info.n_name).starts_with("OpenBSD")) continue; switch (note.info.n_type) { diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp index 2cf8c29bf9d2f..ad72b3d121e67 100644 --- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp @@ -4042,7 +4042,7 @@ void GDBRemoteCommunicationClient::ServeSymbolLookups( return; } else { llvm::StringRef response_str(response.GetStringRef()); - if (response_str.startswith("qSymbol:")) { + if (response_str.starts_with("qSymbol:")) { response.SetFilePos(strlen("qSymbol:")); std::string symbol_name; if (response.GetHexByteString(symbol_name)) { diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp index 187c23a206094..3d37bb226a65f 100644 --- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp @@ -3921,7 +3921,7 @@ GDBRemoteCommunicationServerLLGS::Handle_qSaveCore( std::string path_hint; StringRef packet_str{packet.GetStringRef()}; - assert(packet_str.startswith("qSaveCore")); + assert(packet_str.starts_with("qSaveCore")); if (packet_str.consume_front("qSaveCore;")) { for (auto x : llvm::split(packet_str, ';')) { if (x.consume_front("path-hint:")) @@ -3947,7 +3947,7 @@ GDBRemoteCommunicationServerLLGS::Handle_QNonStop( Log *log = GetLog(LLDBLog::Process); StringRef packet_str{packet.GetStringRef()}; - assert(packet_str.startswith("QNonStop:")); + assert(packet_str.starts_with("QNonStop:")); packet_str.consume_front("QNonStop:"); if (packet_str == "0") { if (m_non_stop) @@ -4306,7 +4306,7 @@ lldb_private::process_gdb_remote::LLGSArgToURL(llvm::StringRef url_arg, std::string host_port = url_arg.str(); // If host_and_port starts with ':', default the host to be "localhost" and // expect the remainder to be the port. - if (url_arg.startswith(":")) + if (url_arg.starts_with(":")) host_port.insert(0, "localhost"); // Try parsing the (preprocessed) argument as host:port pair. diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index d5e557b4b88c0..316be471df929 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -4472,7 +4472,7 @@ bool ParseRegisters( // and a simple type. Just in case, look for that too (setting both // does no harm). if (!gdb_type.empty() && !(encoding_set || format_set)) { - if (llvm::StringRef(gdb_type).startswith("int")) { + if (llvm::StringRef(gdb_type).starts_with("int")) { reg_info.format = eFormatHex; reg_info.encoding = eEncodingUint; } else if (gdb_type == "data_ptr" || gdb_type == "code_ptr") { @@ -4482,7 +4482,7 @@ bool ParseRegisters( reg_info.format = eFormatFloat; reg_info.encoding = eEncodingIEEE754; } else if (gdb_type == "aarch64v" || - llvm::StringRef(gdb_type).startswith("vec") || + llvm::StringRef(gdb_type).starts_with("vec") || gdb_type == "i387_ext" || gdb_type == "uint128") { // lldb doesn't handle 128-bit uints correctly (for ymm*h), so // treat them as vector (similarly to xmm/ymm) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp index 6f771c66a725c..0e2f4d45543bb 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp @@ -809,7 +809,7 @@ removeHostnameFromPathname(llvm::StringRef path_from_dwarf) { // check whether we have a windows path, and so the first character is a // drive-letter not a hostname. if (host.size() == 1 && llvm::isAlpha(host[0]) && - (path.startswith("\\") || path.startswith("/"))) + (path.starts_with("\\") || path.starts_with("/"))) return path_from_dwarf; return path; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 7eddc5074eff1..505ea29ca4d4f 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -1982,7 +1982,7 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() { // (corresponding to .dwo) so we simply skip it. if (m_objfile_sp->GetFileSpec().GetFileNameExtension() == ".dwo" && llvm::StringRef(m_objfile_sp->GetFileSpec().GetPath()) - .endswith(dwo_module_spec.GetFileSpec().GetPath())) { + .ends_with(dwo_module_spec.GetFileSpec().GetPath())) { continue; } diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/CompileUnitIndex.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/CompileUnitIndex.cpp index 06cb720b1e9f7..25d04f999ad67 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/CompileUnitIndex.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/CompileUnitIndex.cpp @@ -236,7 +236,7 @@ CompileUnitIndex::GetMainSourceFile(const CompilandIndexItem &item) const { llvm::cantFail( TypeDeserializer::deserializeAs(file_cvt, file_name)); - llvm::sys::path::Style style = working_dir.String.startswith("/") + llvm::sys::path::Style style = working_dir.String.starts_with("/") ? llvm::sys::path::Style::posix : llvm::sys::path::Style::windows; if (llvm::sys::path::is_absolute(file_name.String, style)) diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp index 5b690ead1e8de..b79d3e63f72b1 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp @@ -1264,9 +1264,9 @@ void PdbAstBuilder::ParseNamespace(clang::DeclContext &context) { clang::NamespaceDecl *ns = llvm::cast(context); llvm::StringRef ns_name = ns->getName(); - if (ns_name.startswith(qname)) { + if (ns_name.starts_with(qname)) { ns_name = ns_name.drop_front(qname.size()); - if (ns_name.startswith("::")) + if (ns_name.starts_with("::")) GetOrCreateType(tid); } } diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp index 35c2575028d85..ad08013399369 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp @@ -1379,7 +1379,7 @@ bool SymbolFileNativePDB::ParseSupportFiles(CompileUnit &comp_unit, for (llvm::StringRef f : cci->m_file_list) { FileSpec::Style style = - f.startswith("/") ? FileSpec::Style::posix : FileSpec::Style::windows; + f.starts_with("/") ? FileSpec::Style::posix : FileSpec::Style::windows; FileSpec spec(f, style); support_files.Append(spec); } diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index 47024cd03536a..797df8c098af1 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -7593,7 +7593,7 @@ clang::CXXMethodDecl *TypeSystemClang::AddMethodToCXXRecordType( nullptr /*expr*/, is_explicit ? clang::ExplicitSpecKind::ResolvedTrue : clang::ExplicitSpecKind::ResolvedFalse); - if (name.startswith("~")) { + if (name.starts_with("~")) { cxx_dtor_decl = clang::CXXDestructorDecl::CreateDeserialized(getASTContext(), 0); cxx_dtor_decl->setDeclContext(cxx_record_decl); diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp index 07f79aaedab82..d890ad92e8312 100644 --- a/lldb/source/Symbol/ObjectFile.cpp +++ b/lldb/source/Symbol/ObjectFile.cpp @@ -607,15 +607,15 @@ lldb::SymbolType ObjectFile::GetSymbolTypeFromName(llvm::StringRef name, lldb::SymbolType symbol_type_hint) { if (!name.empty()) { - if (name.startswith("_OBJC_")) { + if (name.starts_with("_OBJC_")) { // ObjC - if (name.startswith("_OBJC_CLASS_$_")) + if (name.starts_with("_OBJC_CLASS_$_")) return lldb::eSymbolTypeObjCClass; - if (name.startswith("_OBJC_METACLASS_$_")) + if (name.starts_with("_OBJC_METACLASS_$_")) return lldb::eSymbolTypeObjCMetaClass; - if (name.startswith("_OBJC_IVAR_$_")) + if (name.starts_with("_OBJC_IVAR_$_")) return lldb::eSymbolTypeObjCIVar; - } else if (name.startswith(".objc_class_name_")) { + } else if (name.starts_with(".objc_class_name_")) { // ObjC v1 return lldb::eSymbolTypeObjCClass; } diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp index fcc45f861c225..08900a3ef3491 100644 --- a/lldb/source/Symbol/Symbol.cpp +++ b/lldb/source/Symbol/Symbol.cpp @@ -634,7 +634,7 @@ bool Symbol::IsSyntheticWithAutoGeneratedName() const { if (!m_mangled) return true; ConstString demangled = m_mangled.GetDemangledName(); - return demangled.GetStringRef().startswith(GetSyntheticSymbolPrefix()); + return demangled.GetStringRef().starts_with(GetSyntheticSymbolPrefix()); } void Symbol::SynthesizeNameIfNeeded() const { diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp index 1aebe198f9e78..564a3a94cfa20 100644 --- a/lldb/source/Symbol/Symtab.cpp +++ b/lldb/source/Symbol/Symtab.cpp @@ -233,7 +233,7 @@ static bool lldb_skip_name(llvm::StringRef mangled, Mangled::ManglingScheme scheme) { switch (scheme) { case Mangled::eManglingSchemeItanium: { - if (mangled.size() < 3 || !mangled.startswith("_Z")) + if (mangled.size() < 3 || !mangled.starts_with("_Z")) return true; // Avoid the following types of symbols in the index. diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp index db740cb7cb6e4..2bb2ff7db4b72 100644 --- a/lldb/source/Symbol/Variable.cpp +++ b/lldb/source/Symbol/Variable.cpp @@ -510,7 +510,7 @@ static void PrivateAutoCompleteMembers( i, member_name, nullptr, nullptr, nullptr); if (partial_member_name.empty() || - llvm::StringRef(member_name).startswith(partial_member_name)) { + llvm::StringRef(member_name).starts_with(partial_member_name)) { if (member_name == partial_member_name) { PrivateAutoComplete( frame, partial_path, @@ -685,7 +685,7 @@ static void PrivateAutoComplete( continue; llvm::StringRef variable_name = var_sp->GetName().GetStringRef(); - if (variable_name.startswith(token)) { + if (variable_name.starts_with(token)) { if (variable_name == token) { Type *variable_type = var_sp->GetType(); if (variable_type) { diff --git a/lldb/source/Target/TargetList.cpp b/lldb/source/Target/TargetList.cpp index 3ec523b941010..121b6253d2a59 100644 --- a/lldb/source/Target/TargetList.cpp +++ b/lldb/source/Target/TargetList.cpp @@ -271,7 +271,7 @@ Status TargetList::CreateTargetInternal(Debugger &debugger, arch = specified_arch; FileSpec file(user_exe_path); - if (!FileSystem::Instance().Exists(file) && user_exe_path.startswith("~")) { + if (!FileSystem::Instance().Exists(file) && user_exe_path.starts_with("~")) { // we want to expand the tilde but we don't want to resolve any symbolic // links so we can't use the FileSpec constructor's resolve flag llvm::SmallString<64> unglobbed_path; diff --git a/lldb/source/Utility/Args.cpp b/lldb/source/Utility/Args.cpp index 152be96a22128..13b993bc74c9f 100644 --- a/lldb/source/Utility/Args.cpp +++ b/lldb/source/Utility/Args.cpp @@ -641,7 +641,7 @@ void OptionsWithRaw::SetFromString(llvm::StringRef arg_string) { // If the string doesn't start with a dash, we just have no options and just // a raw part. - if (!arg_string.startswith("-")) { + if (!arg_string.starts_with("-")) { m_suffix = std::string(original_args); return; } diff --git a/lldb/source/Utility/CompletionRequest.cpp b/lldb/source/Utility/CompletionRequest.cpp index 8f9dbb79d37bc..e12609ca75e7d 100644 --- a/lldb/source/Utility/CompletionRequest.cpp +++ b/lldb/source/Utility/CompletionRequest.cpp @@ -36,8 +36,8 @@ CompletionRequest::CompletionRequest(llvm::StringRef command_line, // The cursor is after a space but the space is not part of the argument. // Let's add an empty fake argument to the end to make sure the completion // code. Note: The space could be part of the last argument when it's quoted. - if (partial_command.endswith(" ") && - !GetCursorArgumentPrefix().endswith(" ")) + if (partial_command.ends_with(" ") && + !GetCursorArgumentPrefix().ends_with(" ")) AppendEmptyArgument(); } diff --git a/lldb/source/Utility/FileSpec.cpp b/lldb/source/Utility/FileSpec.cpp index 4bbfbb7c1fab5..5387be9a681f6 100644 --- a/lldb/source/Utility/FileSpec.cpp +++ b/lldb/source/Utility/FileSpec.cpp @@ -311,9 +311,9 @@ bool FileSpec::Match(const FileSpec &pattern, const FileSpec &file) { std::optional FileSpec::GuessPathStyle(llvm::StringRef absolute_path) { - if (absolute_path.startswith("/")) + if (absolute_path.starts_with("/")) return Style::posix; - if (absolute_path.startswith(R"(\\)")) + if (absolute_path.starts_with(R"(\\)")) return Style::windows; if (absolute_path.size() >= 3 && llvm::isAlpha(absolute_path[0]) && (absolute_path.substr(1, 2) == R"(:\)" || diff --git a/lldb/source/Utility/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp index d5369ac4bbe51..e3d8ea650c75d 100644 --- a/lldb/source/Utility/FileSpecList.cpp +++ b/lldb/source/Utility/FileSpecList.cpp @@ -117,7 +117,7 @@ size_t FileSpecList::FindCompatibleIndex(size_t start_idx, auto is_suffix = [](llvm::StringRef a, llvm::StringRef b, bool case_sensitive) -> bool { if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b)) - return a.empty() || a.endswith("/"); + return a.empty() || a.ends_with("/"); return false; }; const bool case_sensitive = diff --git a/lldb/source/Utility/NameMatches.cpp b/lldb/source/Utility/NameMatches.cpp index 1c8cd6a0ca31d..f002b86f163bf 100644 --- a/lldb/source/Utility/NameMatches.cpp +++ b/lldb/source/Utility/NameMatches.cpp @@ -22,9 +22,9 @@ bool lldb_private::NameMatches(llvm::StringRef name, NameMatch match_type, case NameMatch::Contains: return name.contains(match); case NameMatch::StartsWith: - return name.startswith(match); + return name.starts_with(match); case NameMatch::EndsWith: - return name.endswith(match); + return name.ends_with(match); case NameMatch::RegularExpression: { RegularExpression regex(match); return regex.Execute(name); diff --git a/lldb/source/Utility/StringExtractor.cpp b/lldb/source/Utility/StringExtractor.cpp index c7e4ac7942842..579faa3da42f3 100644 --- a/lldb/source/Utility/StringExtractor.cpp +++ b/lldb/source/Utility/StringExtractor.cpp @@ -254,7 +254,7 @@ uint64_t StringExtractor::GetHexMaxU64(bool little_endian, bool StringExtractor::ConsumeFront(const llvm::StringRef &str) { llvm::StringRef S = GetStringRef(); - if (!S.startswith(str)) + if (!S.starts_with(str)) return false; else m_index += str.size(); diff --git a/lldb/source/Utility/TildeExpressionResolver.cpp b/lldb/source/Utility/TildeExpressionResolver.cpp index 6311ae062f1f5..2e334b2aae540 100644 --- a/lldb/source/Utility/TildeExpressionResolver.cpp +++ b/lldb/source/Utility/TildeExpressionResolver.cpp @@ -60,7 +60,7 @@ bool StandardTildeExpressionResolver::ResolvePartial(StringRef Expr, while ((user_entry = getpwent()) != nullptr) { StringRef ThisName(user_entry->pw_name); - if (!ThisName.startswith(Expr)) + if (!ThisName.starts_with(Expr)) continue; Buffer.resize(1); @@ -75,7 +75,7 @@ bool StandardTildeExpressionResolver::ResolvePartial(StringRef Expr, bool TildeExpressionResolver::ResolveFullPath( StringRef Expr, llvm::SmallVectorImpl &Output) { - if (!Expr.startswith("~")) { + if (!Expr.starts_with("~")) { Output.assign(Expr.begin(), Expr.end()); return false; } diff --git a/lldb/source/Utility/XcodeSDK.cpp b/lldb/source/Utility/XcodeSDK.cpp index 154ddbebe8b30..d744336373b23 100644 --- a/lldb/source/Utility/XcodeSDK.cpp +++ b/lldb/source/Utility/XcodeSDK.cpp @@ -152,7 +152,7 @@ void XcodeSDK::Merge(const XcodeSDK &other) { *this = other; else { // The Internal flag always wins. - if (llvm::StringRef(m_name).endswith(".sdk")) + if (llvm::StringRef(m_name).ends_with(".sdk")) if (!l.internal && r.internal) m_name = m_name.substr(0, m_name.size() - 3) + std::string("Internal.sdk"); @@ -291,7 +291,7 @@ std::string XcodeSDK::FindXcodeContentsDirectoryInPath(llvm::StringRef path) { // .app. If the next component is Contents then we've found the Contents // directory. for (auto it = begin; it != end; ++it) { - if (it->endswith(".app")) { + if (it->ends_with(".app")) { auto next = it; if (++next != end && *next == "Contents") { llvm::SmallString<128> buffer; diff --git a/lldb/tools/lldb-dap/IOStream.cpp b/lldb/tools/lldb-dap/IOStream.cpp index 897ab791ed062..96e9a1ed49532 100644 --- a/lldb/tools/lldb-dap/IOStream.cpp +++ b/lldb/tools/lldb-dap/IOStream.cpp @@ -138,7 +138,7 @@ bool InputStream::read_line(std::ofstream *log, std::string &line) { if (!read_full(log, 1, line)) return false; - if (llvm::StringRef(line).endswith("\r\n")) + if (llvm::StringRef(line).ends_with("\r\n")) break; } line.erase(line.size() - 2); diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp index c8e5304ecec81..a0a175f960bcf 100644 --- a/lldb/tools/lldb-dap/JSONUtils.cpp +++ b/lldb/tools/lldb-dap/JSONUtils.cpp @@ -455,8 +455,9 @@ llvm::json::Value CreateBreakpoint(lldb::SBBreakpoint &bp, static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) { uint64_t debug_info_size = 0; llvm::StringRef section_name(section.GetName()); - if (section_name.startswith(".debug") || section_name.startswith("__debug") || - section_name.startswith(".apple") || section_name.startswith("__apple")) + if (section_name.starts_with(".debug") || + section_name.starts_with("__debug") || + section_name.starts_with(".apple") || section_name.starts_with("__apple")) debug_info_size += section.GetFileByteSize(); size_t num_sub_sections = section.GetNumSubSections(); for (size_t i = 0; i < num_sub_sections; i++) { diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp index d36e9b4d1b098..75b3948b5efb7 100644 --- a/lldb/tools/lldb-dap/lldb-dap.cpp +++ b/lldb/tools/lldb-dap/lldb-dap.cpp @@ -3094,7 +3094,7 @@ void request_setVariable(const llvm::json::Object &request) { lldb::SBValue container = g_dap.variables.GetVariable(variablesReference); variable = container.GetChildMemberWithName(name.data()); if (!variable.IsValid()) { - if (name.startswith("[")) { + if (name.starts_with("[")) { llvm::StringRef index_str(name.drop_front(1)); uint64_t index = 0; if (!index_str.consumeInteger(0, index)) { diff --git a/lldb/unittests/Expression/ClangExpressionDeclMapTest.cpp b/lldb/unittests/Expression/ClangExpressionDeclMapTest.cpp index 77bd85bcb942c..1c07119d4497f 100644 --- a/lldb/unittests/Expression/ClangExpressionDeclMapTest.cpp +++ b/lldb/unittests/Expression/ClangExpressionDeclMapTest.cpp @@ -35,7 +35,7 @@ struct FakeClangExpressionDeclMap : public ClangExpressionDeclMap { // The declaration needs to have '$' prefix in its name like every // persistent declaration and must be inside the scratch AST context. assert(d); - assert(d->getName().startswith("$")); + assert(d->getName().starts_with("$")); assert(&d->getASTContext() == &m_scratch_context->getASTContext()); m_persistent_decls[d->getName()] = d; } diff --git a/lldb/unittests/Process/minidump/RegisterContextMinidumpTest.cpp b/lldb/unittests/Process/minidump/RegisterContextMinidumpTest.cpp index 0bacd882d8e89..f179ded7fa6b6 100644 --- a/lldb/unittests/Process/minidump/RegisterContextMinidumpTest.cpp +++ b/lldb/unittests/Process/minidump/RegisterContextMinidumpTest.cpp @@ -152,7 +152,7 @@ static void TestARMRegInfo(const lldb_private::RegisterInfo *info) { // correctly when using this information. llvm::StringRef name(info->name); llvm::StringRef alt_name(info->alt_name); - if (name.startswith("r") || alt_name.startswith("r")) { + if (name.starts_with("r") || alt_name.starts_with("r")) { EXPECT_NE(info->kinds[lldb::eRegisterKindEHFrame], LLDB_INVALID_REGNUM); EXPECT_NE(info->kinds[lldb::eRegisterKindDWARF], LLDB_INVALID_REGNUM); } diff --git a/lldb/unittests/TestingSupport/MockTildeExpressionResolver.cpp b/lldb/unittests/TestingSupport/MockTildeExpressionResolver.cpp index 0e64e1977ca44..08381b34e6769 100644 --- a/lldb/unittests/TestingSupport/MockTildeExpressionResolver.cpp +++ b/lldb/unittests/TestingSupport/MockTildeExpressionResolver.cpp @@ -69,7 +69,7 @@ bool MockTildeExpressionResolver::ResolvePartial(StringRef Expr, SmallString<16> QualifiedName("~"); for (const auto &User : UserDirectories) { - if (!User.getKey().startswith(Expr)) + if (!User.getKey().starts_with(Expr)) continue; QualifiedName.resize(1); QualifiedName.append(User.getKey().begin(), User.getKey().end()); From 038871ae684b9cf47222bc0de666847f7294cab5 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 14:47:16 -0800 Subject: [PATCH 043/884] [llvm] Use StringRef::{starts,ends}_with (NFC) This patch replaces uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- llvm/lib/TargetParser/Host.cpp | 4 ++-- llvm/unittests/Support/ProgramTest.cpp | 2 +- llvm/unittests/Support/TypeNameTest.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index e822da11ee85a..e61fcb248faec 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -1536,7 +1536,7 @@ StringRef sys::detail::getHostCPUNameForSPARC(StringRef ProcCpuinfoContent) { // Look for cpu line to determine cpu name StringRef Cpu; for (unsigned I = 0, E = Lines.size(); I != E; ++I) { - if (Lines[I].startswith("cpu")) { + if (Lines[I].starts_with("cpu")) { Cpu = Lines[I].substr(5).ltrim("\t :"); break; } @@ -1853,7 +1853,7 @@ bool sys::getHostCPUFeatures(StringMap &Features) { // Look for the CPU features. for (unsigned I = 0, E = Lines.size(); I != E; ++I) - if (Lines[I].startswith("Features")) { + if (Lines[I].starts_with("Features")) { Lines[I].split(CPUFeatures, ' '); break; } diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp index 91dbb57fbdeca..2e2b1958b9ac9 100644 --- a/llvm/unittests/Support/ProgramTest.cpp +++ b/llvm/unittests/Support/ProgramTest.cpp @@ -130,7 +130,7 @@ TEST_F(ProgramEnvTest, CreateProcessLongPath) { // prefix. sys::path::native(MyAbsExe, sys::path::Style::windows_backslash); std::string MyExe; - if (!StringRef(MyAbsExe).startswith("\\\\?\\")) + if (!StringRef(MyAbsExe).starts_with("\\\\?\\")) MyExe.append("\\\\?\\"); MyExe.append(std::string(MyAbsExe.begin(), MyAbsExe.end())); diff --git a/llvm/unittests/Support/TypeNameTest.cpp b/llvm/unittests/Support/TypeNameTest.cpp index e0d9ac5ee7e38..3550c626e9e67 100644 --- a/llvm/unittests/Support/TypeNameTest.cpp +++ b/llvm/unittests/Support/TypeNameTest.cpp @@ -35,7 +35,7 @@ TEST(TypeNameTest, Names) { #ifdef __clang__ EXPECT_TRUE(S2Name.ends_with("S2")) << S2Name.str(); #else - EXPECT_TRUE(S2Name.endswith("::S2")) << S2Name.str(); + EXPECT_TRUE(S2Name.ends_with("::S2")) << S2Name.str(); #endif #else EXPECT_EQ("UNKNOWN_TYPE", S1Name); From c26510a2bf369a0031a6757dedc1fe9f901b3975 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sat, 16 Dec 2023 14:54:01 -0800 Subject: [PATCH 044/884] [RISCV] Fix intrinsic names in sf_vfwmacc_4x4x4.ll. NFC The type strings in the intrinsic name were using f16 instead of bf16 for float types. Nothing really checks these strings so everything still worked. --- .../CodeGen/RISCV/rvv/sf_vfwmacc_4x4x4.ll | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vfwmacc_4x4x4.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vfwmacc_4x4x4.ll index 0ba92c5f70e6e..180155139b57b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/sf_vfwmacc_4x4x4.ll +++ b/llvm/test/CodeGen/RISCV/rvv/sf_vfwmacc_4x4x4.ll @@ -4,7 +4,7 @@ ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+experimental-zvfbfmin,+xsfvfwmaccqqq \ ; RUN: -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK -declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4f16.nxv1f16( +declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4bf16.nxv1bf16.iXLen( , , , @@ -17,7 +17,7 @@ define @intrinsic_vfwmacc_4x4x4_tu_f32mf2( @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4f16.nxv1f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4bf16.nxv1bf16.iXLen( %0, %1, %2, @@ -33,7 +33,7 @@ define @intrinsic_vfwmacc_4x4x4_ta_f32mf2( @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4f16.nxv1f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv1f32.nxv4bf16.nxv1bf16.iXLen( %0, %1, %2, @@ -42,7 +42,7 @@ entry: ret %a } -declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4f16.nxv2f16( +declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4bf16.nxv2bf16.iXLen( , , , @@ -55,7 +55,7 @@ define @intrinsic_vfwmacc_4x4x4_tu_f32m1( @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4f16.nxv2f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4bf16.nxv2bf16.iXLen( %0, %1, %2, @@ -71,7 +71,7 @@ define @intrinsic_vfwmacc_4x4x4_ta_f32m1( @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4f16.nxv2f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv2f32.nxv4bf16.nxv2bf16.iXLen( %0, %1, %2, @@ -80,7 +80,7 @@ entry: ret %a } -declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4f16.nxv4f16( +declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4bf16.nxv4bf16.iXLen( , , , @@ -93,7 +93,7 @@ define @intrinsic_vfwmacc_4x4x4_tu_f32m2( @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4f16.nxv4f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4bf16.nxv4bf16.iXLen( %0, %1, %2, @@ -109,7 +109,7 @@ define @intrinsic_vfwmacc_4x4x4_ta_f32m2( @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4f16.nxv4f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv4f32.nxv4bf16.nxv4bf16.iXLen( %0, %1, %2, @@ -118,7 +118,7 @@ entry: ret %a } -declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4f16.nxv8f16( +declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4bf16.nxv8bf16.iXLen( , , , @@ -131,7 +131,7 @@ define @intrinsic_vfwmacc_4x4x4_tu_f32m4( @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4f16.nxv8f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4bf16.nxv8bf16.iXLen( %0, %1, %2, @@ -147,7 +147,7 @@ define @intrinsic_vfwmacc_4x4x4_ta_f32m4( @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4f16.nxv8f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv8f32.nxv4bf16.nxv8bf16.iXLen( %0, %1, %2, @@ -156,7 +156,7 @@ entry: ret %a } -declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4f16.nxv16f16( +declare @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4bf16.nxv16bf16.iXLen( , , , @@ -169,7 +169,7 @@ define @intrinsic_vfwmacc_4x4x4_tu_f32m8( @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4f16.nxv16f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4bf16.nxv16bf16.iXLen( %0, %1, %2, @@ -185,7 +185,7 @@ define @intrinsic_vfwmacc_4x4x4_ta_f32m8( @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4f16.nxv16f16( + %a = call @llvm.riscv.sf.vfwmacc.4x4x4.nxv16f32.nxv4bf16.nxv16bf16.iXLen( %0, %1, %2, From ee667db4b83eb6171bbceca1010cddd0da6f17ca Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 15:02:15 -0800 Subject: [PATCH 045/884] [lldb] Use StringRef::{starts,ends}_with (NFC) This patch replaces uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- lldb/source/Host/common/Host.cpp | 2 +- .../ObjectFile/Mach-O/ObjectFileMachO.cpp | 16 ++++++++-------- .../Platform/MacOSX/PlatformDarwinKernel.cpp | 4 ++-- .../Process/Windows/Common/DebuggerThread.cpp | 2 +- .../Lua/ScriptInterpreterTests.cpp | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp index 8314d3581f6a4..f4cec97f5af63 100644 --- a/lldb/source/Host/common/Host.cpp +++ b/lldb/source/Host/common/Host.cpp @@ -554,7 +554,7 @@ bool Host::IsInteractiveGraphicSession() { return false; } std::unique_ptr Host::CreateDefaultConnection(llvm::StringRef url) { #if defined(_WIN32) - if (url.startswith("file://")) + if (url.starts_with("file://")) return std::unique_ptr(new ConnectionGenericFile()); #endif return std::unique_ptr(new ConnectionFileDescriptor()); diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp index 58275c052f74e..182a9f2afaeb2 100644 --- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp +++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp @@ -2868,7 +2868,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { if (symbol_name && symbol_name[0] == '_' && symbol_name[1] == 'O') { llvm::StringRef symbol_name_ref(symbol_name); - if (symbol_name_ref.startswith( + if (symbol_name_ref.starts_with( g_objc_v2_prefix_class)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = @@ -2876,14 +2876,14 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { type = eSymbolTypeObjCClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith( + } else if (symbol_name_ref.starts_with( g_objc_v2_prefix_metaclass)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = symbol_name + g_objc_v2_prefix_metaclass.size(); type = eSymbolTypeObjCMetaClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith( + } else if (symbol_name_ref.starts_with( g_objc_v2_prefix_ivar)) { symbol_name_non_abi_mangled = symbol_name + 1; symbol_name = @@ -3382,7 +3382,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { if (symbol_name) { llvm::StringRef symbol_name_ref(symbol_name); - if (symbol_name_ref.startswith("_OBJC_")) { + if (symbol_name_ref.starts_with("_OBJC_")) { llvm::StringRef g_objc_v2_prefix_class( "_OBJC_CLASS_$_"); @@ -3391,7 +3391,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { "_OBJC_METACLASS_$_"); llvm::StringRef g_objc_v2_prefix_ivar("_OBJC_IVAR_$_"); - if (symbol_name_ref.startswith( + if (symbol_name_ref.starts_with( g_objc_v2_prefix_class)) { symbol_name_non_abi_mangled = symbol_name + 1; @@ -3401,7 +3401,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { type = eSymbolTypeObjCClass; demangled_is_synthesized = true; } else if ( - symbol_name_ref.startswith( + symbol_name_ref.starts_with( g_objc_v2_prefix_metaclass)) { symbol_name_non_abi_mangled = symbol_name + 1; @@ -3410,7 +3410,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { g_objc_v2_prefix_metaclass.size(); type = eSymbolTypeObjCMetaClass; demangled_is_synthesized = true; - } else if (symbol_name_ref.startswith( + } else if (symbol_name_ref.starts_with( g_objc_v2_prefix_ivar)) { symbol_name_non_abi_mangled = symbol_name + 1; @@ -3441,7 +3441,7 @@ void ObjectFileMachO::ParseSymtab(Symtab &symtab) { llvm::StringRef symbol_name_ref(symbol_name); llvm::StringRef g_objc_v1_prefix_class(".objc_class_name_"); - if (symbol_name_ref.startswith( + if (symbol_name_ref.starts_with( g_objc_v1_prefix_class)) { symbol_name_non_abi_mangled = symbol_name; symbol_name = symbol_name + diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp index 4d485e75adcd8..e2839f3285cce 100644 --- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp +++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp @@ -487,8 +487,8 @@ PlatformDarwinKernel::GetKernelsAndKextsInDirectoryHelper( llvm::StringRef filename = file_spec.GetFilename().GetStringRef(); bool is_kernel_filename = - filename.startswith("kernel") || filename.startswith("mach"); - bool is_dsym_yaa = filename.endswith(".dSYM.yaa"); + filename.starts_with("kernel") || filename.starts_with("mach"); + bool is_dsym_yaa = filename.ends_with(".dSYM.yaa"); if (ft == llvm::sys::fs::file_type::regular_file || ft == llvm::sys::fs::file_type::symlink_file) { diff --git a/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp b/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp index 93227c5524143..e7fee41239da7 100644 --- a/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp +++ b/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp @@ -501,7 +501,7 @@ DebuggerThread::HandleLoadDllEvent(const LOAD_DLL_DEBUG_INFO &info, llvm::convertWideToUTF8(buffer.data(), path_str_utf8); llvm::StringRef path_str = path_str_utf8; const char *path = path_str.data(); - if (path_str.startswith("\\\\?\\")) + if (path_str.starts_with("\\\\?\\")) path += 4; on_load_dll(path); diff --git a/lldb/unittests/ScriptInterpreter/Lua/ScriptInterpreterTests.cpp b/lldb/unittests/ScriptInterpreter/Lua/ScriptInterpreterTests.cpp index ce6dc586cc401..2693bef3f5fbd 100644 --- a/lldb/unittests/ScriptInterpreter/Lua/ScriptInterpreterTests.cpp +++ b/lldb/unittests/ScriptInterpreter/Lua/ScriptInterpreterTests.cpp @@ -48,6 +48,6 @@ TEST_F(ScriptInterpreterTest, ExecuteOneLine) { CommandReturnObject result(/*colors*/ false); EXPECT_TRUE(script_interpreter.ExecuteOneLine("foo = 1", &result)); EXPECT_FALSE(script_interpreter.ExecuteOneLine("nil = foo", &result)); - EXPECT_TRUE(result.GetErrorData().startswith( + EXPECT_TRUE(result.GetErrorData().starts_with( "error: lua failed attempting to evaluate 'nil = foo'")); } From b8f89b84bc26c46a5a10d01eb5414fbde3c8700a Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 15:02:17 -0800 Subject: [PATCH 046/884] Use StringRef::{starts,ends}_with (NFC) This patch replaces uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- clang-tools-extra/clangd/index/remote/server/Server.cpp | 2 +- clang/test/Index/recursive-cxx-member-calls.cpp | 2 +- libc/utils/HdrGen/Generator.cpp | 6 +++--- mlir/examples/toy/Ch2/toyc.cpp | 2 +- mlir/examples/toy/Ch3/toyc.cpp | 2 +- mlir/examples/toy/Ch4/toyc.cpp | 2 +- mlir/examples/toy/Ch5/toyc.cpp | 2 +- mlir/examples/toy/Ch6/toyc.cpp | 2 +- mlir/examples/toy/Ch7/toyc.cpp | 2 +- .../plugins-nextgen/common/include/PluginInterface.h | 2 +- openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp index e108d4d0b057b..4ef3ab6f9af9c 100644 --- a/clang-tools-extra/clangd/index/remote/server/Server.cpp +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -451,7 +451,7 @@ std::unique_ptr makeLogger(llvm::StringRef LogPrefix, void log(Level L, const char *Fmt, const llvm::formatv_object_base &Message) override { if (Context::current().get(CurrentRequest) == nullptr || - llvm::StringRef(Fmt).startswith("[public]")) + llvm::StringRef(Fmt).starts_with("[public]")) return StreamLogger::log(L, Fmt, Message); if (L >= Error) return StreamLogger::log(L, Fmt, diff --git a/clang/test/Index/recursive-cxx-member-calls.cpp b/clang/test/Index/recursive-cxx-member-calls.cpp index be908c506e747..48fc8f14544c4 100644 --- a/clang/test/Index/recursive-cxx-member-calls.cpp +++ b/clang/test/Index/recursive-cxx-member-calls.cpp @@ -99,7 +99,7 @@ using namespace clang; AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) { llvm::StringRef AttrName = Name->getName(); - if (AttrName.startswith("__") && AttrName.endswith("__")) + if (AttrName.starts_with("__") && AttrName.ends_with("__")) AttrName = AttrName.substr(2, AttrName.size() - 4); return llvm::StringSwitch < AttributeList::Kind > (AttrName) diff --git a/libc/utils/HdrGen/Generator.cpp b/libc/utils/HdrGen/Generator.cpp index 24d22680fe525..3bcf005adda74 100644 --- a/libc/utils/HdrGen/Generator.cpp +++ b/libc/utils/HdrGen/Generator.cpp @@ -57,7 +57,7 @@ void Generator::parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args) { ArgStr.split(Args, ","); for (llvm::StringRef &A : Args) { A = A.trim(' '); - if (A.startswith(ParamNamePrefix) && A.endswith(ParamNameSuffix)) { + if (A.starts_with(ParamNamePrefix) && A.ends_with(ParamNameSuffix)) { A = A.drop_front(ParamNamePrefixSize).drop_back(ParamNameSuffixSize); A = ArgMap[std::string(A)]; } @@ -80,7 +80,7 @@ void Generator::generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) { Content = P.second; llvm::StringRef Line = P.first.trim(' '); - if (Line.startswith(CommandPrefix)) { + if (Line.starts_with(CommandPrefix)) { Line = Line.drop_front(CommandPrefixSize); P = Line.split("("); @@ -107,7 +107,7 @@ void Generator::generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) { Command::ErrorReporter Reporter( llvm::SMLoc::getFromPointer(CommandName.data()), SrcMgr); Cmd->run(OS, Args, StdHeader, Records, Reporter); - } else if (!Line.startswith(CommentPrefix)) { + } else if (!Line.starts_with(CommentPrefix)) { // There is no comment or command on this line so we just write it as is. OS << P.first << "\n"; } diff --git a/mlir/examples/toy/Ch2/toyc.cpp b/mlir/examples/toy/Ch2/toyc.cpp index fa431972e211e..e33b49b41c5a1 100644 --- a/mlir/examples/toy/Ch2/toyc.cpp +++ b/mlir/examples/toy/Ch2/toyc.cpp @@ -78,7 +78,7 @@ int dumpMLIR() { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/mlir/examples/toy/Ch3/toyc.cpp b/mlir/examples/toy/Ch3/toyc.cpp index 8c27a7af97a00..c2c5f1fe1ba17 100644 --- a/mlir/examples/toy/Ch3/toyc.cpp +++ b/mlir/examples/toy/Ch3/toyc.cpp @@ -82,7 +82,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context, mlir::OwningOpRef &module) { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/mlir/examples/toy/Ch4/toyc.cpp b/mlir/examples/toy/Ch4/toyc.cpp index d35e07cf3f20b..a1534ed30341c 100644 --- a/mlir/examples/toy/Ch4/toyc.cpp +++ b/mlir/examples/toy/Ch4/toyc.cpp @@ -83,7 +83,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context, mlir::OwningOpRef &module) { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/mlir/examples/toy/Ch5/toyc.cpp b/mlir/examples/toy/Ch5/toyc.cpp index e0742b8e992b3..4eb6fdeeafa1d 100644 --- a/mlir/examples/toy/Ch5/toyc.cpp +++ b/mlir/examples/toy/Ch5/toyc.cpp @@ -88,7 +88,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context, mlir::OwningOpRef &module) { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/mlir/examples/toy/Ch6/toyc.cpp b/mlir/examples/toy/Ch6/toyc.cpp index fe2137cfdfbfc..534f0d60e8009 100644 --- a/mlir/examples/toy/Ch6/toyc.cpp +++ b/mlir/examples/toy/Ch6/toyc.cpp @@ -112,7 +112,7 @@ int loadMLIR(mlir::MLIRContext &context, mlir::OwningOpRef &module) { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/mlir/examples/toy/Ch7/toyc.cpp b/mlir/examples/toy/Ch7/toyc.cpp index d4cc8e7279d3b..e4af0a3a3dce1 100644 --- a/mlir/examples/toy/Ch7/toyc.cpp +++ b/mlir/examples/toy/Ch7/toyc.cpp @@ -112,7 +112,7 @@ int loadMLIR(mlir::MLIRContext &context, mlir::OwningOpRef &module) { // Handle '.toy' input to the compiler. if (inputType != InputType::MLIR && - !llvm::StringRef(inputFilename).endswith(".mlir")) { + !llvm::StringRef(inputFilename).ends_with(".mlir")) { auto moduleAST = parseInputFile(inputFilename); if (!moduleAST) return 6; diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h index ab6c457fba786..716b0ad784331 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h @@ -290,7 +290,7 @@ struct GenericKernelTy { /// Return true if this kernel is a constructor or destructor. bool isCtorOrDtor() const { // TODO: This is not a great solution and should be revisited. - return StringRef(Name).endswith("tor"); + return StringRef(Name).ends_with("tor"); } /// Get the kernel image. diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp index 7bad411b9d8e3..a2ccf8446ba77 100644 --- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp @@ -1304,7 +1304,7 @@ struct CUDAPluginTy final : public GenericPluginTy { StringRef ArchStr(Info->Arch); StringRef PrefixStr("sm_"); - if (!ArchStr.startswith(PrefixStr)) + if (!ArchStr.starts_with(PrefixStr)) return Plugin::error("Unrecognized image arch %s", ArchStr.data()); int32_t ImageMajor = ArchStr[PrefixStr.size() + 0] - '0'; From 0ca95b269ff90afb706e2cf4c4a59d7c3afe6c65 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 16 Dec 2023 15:43:41 -0800 Subject: [PATCH 047/884] [llvm-objdump,test] Improve zero dumping and inline relocs tests --- .../X86/disassemble-zeroes-relocations.test | 93 ++++++----- .../X86/elf-disassemble-relocs.test | 152 ++++++++---------- 2 files changed, 122 insertions(+), 123 deletions(-) diff --git a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test index c48e2d8c4f613..d61d7455570b1 100644 --- a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test +++ b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test @@ -1,54 +1,67 @@ -# RUN: yaml2obj %s -o %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t ## Check we do not skip zeroes blocks if have relocations pointed to these places. -# RUN: llvm-objdump -D --reloc %t +# RUN: llvm-objdump -D --reloc -j .rodata %t | FileCheck %s -# CHECK: 0000000000000000 .rodata: +# CHECK: 0000000000000000 : # CHECK-NEXT: 0: 00 00 addb %al, (%rax) -# CHECK-NEXT: 0000000000000000: R_X86_64_64 *ABS* +# CHECK-NEXT: 0000000000000000: R_X86_64_64 x0 # CHECK-NEXT: 2: 00 00 addb %al, (%rax) # CHECK-NEXT: 4: 00 00 addb %al, (%rax) # CHECK-NEXT: 6: 00 00 addb %al, (%rax) # CHECK-NEXT: 8: 00 00 addb %al, (%rax) -# CHECK-NEXT: 0000000000000008: R_X86_64_64 *ABS* -# CHECK-NEXT: a: 00 00 addb %al, (%rax) -# CHECK-NEXT: c: 00 00 addb %al, (%rax) -# CHECK-NEXT: e: 00 00 addb %al, (%rax) +# CHECK-NEXT: 0000000000000008: R_X86_64_64 x1 +# CHECK-NEXT: ... +# CHECK-NEXT: 16: 00 00 addb %al, (%rax) +# CHECK-NEXT: 18: 00 00 addb %al, (%rax) +# CHECK-NEXT: 0000000000000018: R_X86_64_64 x2 +# CHECK-NEXT: 1a: 00 00 addb %al, (%rax) +# CHECK-NEXT: 1c: 00 00 addb %al, (%rax) +# CHECK-NEXT: 1e: 00 00 addb %al, (%rax) +# CHECK-EMPTY: +# CHECK-NEXT: 000000000000001f : +# CHECK-NEXT: ... +# CHECK-EMPTY: +# CHECK-NEXT: 0000000000000037 : +# CHECK-NEXT: ... +# CHECK-NEXT: 3f: 00 00 addb %al, (%rax) +# CHECK-NEXT: 000000000000003f: R_X86_64_64 x3 +# CHECK-NEXT: 41: 00 00 addb %al, (%rax) +# CHECK-NEXT: 43: 00 00 addb %al, (%rax) +# CHECK-NEXT: 45: 00 00 addb %al, (%rax) ## Check that without -reloc all zeroes would be omitted. # RUN: llvm-objdump -D %t | FileCheck %s --check-prefix=SKIP -# SKIP: 0000000000000000 <.rodata>: -# SKIP-NEXT: ... +# SKIP: Disassembly of section .rodata: # SKIP-EMPTY: -# SKIP-NEXT: Disassembly of section .rela.rodata: +# SKIP-NEXT: 0000000000000000 : +# SKIP-NEXT: ... +# SKIP-NEXT: 1c: 00 00 addb %al, (%rax) +# SKIP-NEXT: 1e: 00 00 addb %al, (%rax) +# SKIP-EMPTY: +# SKIP-NEXT: 000000000000001f : +# SKIP-NEXT: ... +# SKIP-EMPTY: +# SKIP-NEXT: 0000000000000037 : +# SKIP-NEXT: ... +# SKIP-EMPTY: +# SKIP-NEXT: Disassembly of section .rela.rodata: + +.rodata +.globl rodata1, rodata2 +rodata1: + .reloc ., BFD_RELOC_64, x0 + .space 8 + .reloc ., BFD_RELOC_64, x1 + .space 16 + .reloc ., BFD_RELOC_64, x2 + .space 7 + +rodata2: + .space 24 ---- !ELF -FileHeader: - Class: ELFCLASS64 - Data: ELFDATA2LSB - Type: ET_REL - Machine: EM_X86_64 -Sections: - - Name: .rodata - Type: SHT_PROGBITS - Flags: [ SHF_ALLOC ] - AddressAlign: 0x0000000000000001 - Content: '00000000000000000000000000000000' - - Name: .rela.rodata - Type: SHT_RELA - Flags: [ SHF_INFO_LINK ] - Link: .symtab - AddressAlign: 0x0000000000000008 - EntSize: 0x0000000000000018 - Info: .rodata - Relocations: - - Offset: 0x0000000000000000 - Symbol: x - Type: R_X86_64_64 - - Offset: 0x0000000000000008 - Symbol: x - Type: R_X86_64_64 -Symbols: - - Name: x - Binding: STB_GLOBAL +rodata3: + .space 8 + .reloc ., BFD_RELOC_64, x3 + .space 8 diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test index 593735a6a1026..33438cbfa126e 100644 --- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test +++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test @@ -1,53 +1,73 @@ ## Show that --disassemble + --reloc prints relocations inline and does not dump ## the relocation sections. -# RUN: yaml2obj %s --docnum=1 -o %t1.o -# RUN: llvm-objdump %t1.o -d -r | FileCheck %s --implicit-check-not="RELOCATION RECORDS" +# RUN: rm -rf %t && split-file %s %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 1.s -o 1.o +# RUN: llvm-objdump 1.o -d -r | FileCheck %s --implicit-check-not="RELOCATION RECORDS" +# RUN: llvm-objdump 1.o -r --disassemble-symbols=x2,x4 | FileCheck %s --check-prefix=CHECK2 -# CHECK: 0: e8 00 00 00 00 callq 0x5 <.text+0x5> -# CHECK-NEXT: 0000000000000001: R_X86_64_PC32 foo-0x4 -# CHECK-NEXT: 0000000000000002: R_X86_64_NONE bar+0x8 -# CHECK-NEXT: 5: e8 00 00 00 00 callq 0xa <.text+0xa> -# CHECK-NEXT: 0000000000000006: R_X86_64_PLT32 foo+0x1 +#--- 1.s +# CHECK: 0000000000000000 : +# CHECK-NEXT: 0: e8 00 00 00 00 callq 0x5 +# CHECK-NEXT: 0000000000000001: R_X86_64_PC32 foo-0x4 +# CHECK-NEXT: 0000000000000002: R_X86_64_NONE bar+0x8 +# CHECK-NEXT: 5: e8 00 00 00 00 callq 0xa +# CHECK-NEXT: 0000000000000006: R_X86_64_PLT32 foo+0x1 +# CHECK-EMPTY: +# CHECK-NEXT: 000000000000000a : +# CHECK-NEXT: a: 90 nop +# CHECK-NEXT: b: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x12 +# CHECK-NEXT: 000000000000000e: R_X86_64_REX_GOTPCRELX var-0x4 +# CHECK-EMPTY: +# CHECK-NEXT: 0000000000000012 : +# CHECK-NEXT: 12: e8 00 00 00 00 callq 0x17 +# CHECK-NEXT: 0000000000000013: R_X86_64_PLT32 foo-0x4 +# CHECK-EMPTY: +# CHECK-NEXT: 0000000000000017 : +# CHECK-NEXT: 17: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x1e +# CHECK-NEXT: 000000000000001a: R_X86_64_REX_GOTPCRELX var-0x4 +# CHECK-NOT: {{.}} ---- !ELF -FileHeader: - Class: ELFCLASS64 - Data: ELFDATA2LSB - Type: ET_REL - Machine: EM_X86_64 -Sections: - - Name: .text - Type: SHT_PROGBITS - Flags: [SHF_ALLOC, SHF_EXECINSTR] - Content: 'e800000000e800000000' - - Name: .rela.text - Type: SHT_RELA - Info: .text - Relocations: - - Offset: 1 - Symbol: foo - Type: R_X86_64_PC32 - Addend: -4 - - Offset: 2 - Symbol: bar - Type: R_X86_64_NONE - Addend: 8 - - Offset: 6 - Symbol: foo - Type: R_X86_64_PLT32 - Addend: 1 -Symbols: - - Name: foo - - Name: bar +## FIXME: --disassemble-symbols: remove inline relocs from skipped functions +# CHECK2: 000000000000000a : +# CHECK2-NEXT: a: 90 nop +# CHECK2-NEXT: 0000000000000001: R_X86_64_PC32 foo-0x4 +# CHECK2-NEXT: 0000000000000002: R_X86_64_NONE bar+0x8 +# CHECK2-NEXT: 0000000000000006: R_X86_64_PLT32 foo+0x1 +# CHECK2-NEXT: b: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x12 +# CHECK2-NEXT: 000000000000000e: R_X86_64_REX_GOTPCRELX var-0x4 +# CHECK2-EMPTY: +# CHECK2-NEXT: 0000000000000017 : +# CHECK2-NEXT: 17: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x1e +# CHECK2-NEXT: 0000000000000013: R_X86_64_PLT32 foo-0x4 +# CHECK2-NEXT: 000000000000001a: R_X86_64_REX_GOTPCRELX var-0x4 + +.globl x1, x2, x3, x4 +x1: + .reloc .+1, R_X86_64_PC32, foo-4 + .reloc .+2, BFD_RELOC_NONE, bar+8 + .byte 0xe8, 0, 0, 0, 0 + .reloc .+1, R_X86_64_PLT32, foo+1 + .byte 0xe8, 0, 0, 0, 0 + +x2: + nop + movq var@GOTPCREL(%rip), %rax + +x3: + call foo +x4: + movq var@GOTPCREL(%rip), %rax + +#--- 2.yaml ## Check we report an error if the relocated section identified by the ## sh_info field of a relocation section is invalid. -# RUN: yaml2obj %s --docnum=2 -o %t2.o -# RUN: not llvm-objdump %t2.o -d --reloc 2>&1 | FileCheck %s -DFILE=%t2.o --check-prefix=ERR +# RUN: yaml2obj 2.yaml -o 2.o +# RUN: not llvm-objdump 2.o -d --reloc 2>&1 | FileCheck %s --check-prefix=ERR -# ERR: error: '[[FILE]]': section (1): failed to get a relocated section: invalid section index: 255 +# ERR: error: '2.o': section (1): failed to get a relocated section: invalid section index: 255 --- !ELF FileHeader: @@ -62,9 +82,10 @@ Sections: Info: 0xFF Relocations: [] +#--- 3.s ## Check ranges of addends being displayed in a dump of relocations mixed with disassembly. -# RUN: yaml2obj --docnum=3 %s -o %t3 -# RUN: llvm-objdump -d -r %t3 | FileCheck %s --check-prefix=ADDENDS +# RUN: llvm-mc -filetype=obj -triple=x86_64 3.s -o 3.o +# RUN: llvm-objdump -d -r 3.o | FileCheck %s --check-prefix=ADDENDS # ADDENDS: Disassembly of section .text: # ADDENDS: R_X86_64_64 glob-0x8000000000000000 @@ -73,44 +94,9 @@ Sections: # ADDENDS: R_X86_64_64 glob+0x12345678 # ADDENDS: R_X86_64_64 glob{{$}} ---- !ELF -FileHeader: - Class: ELFCLASS64 - Data: ELFDATA2LSB - Type: ET_REL - Machine: EM_X86_64 -Sections: -- Name: .text - Type: SHT_PROGBITS - Size: 8 - Flags: [SHF_EXECINSTR,SHF_ALLOC] -- Name: .rela.text - Type: SHT_RELA - Info: .text - Relocations: - - Offset: 0x0 - Addend: -9223372036854775808 - Symbol: glob - Type: R_X86_64_64 - - Offset: 0x1 - Symbol: glob - Type: R_X86_64_64 - Addend: 9223372036854775807 - - Offset: 0x2 - Symbol: glob - Type: R_X86_64_64 - Addend: -1 - - Offset: 0x3 - Symbol: glob - Type: R_X86_64_64 - Addend: 0x12345678 - - Offset: 0x4 - Symbol: glob - Type: R_X86_64_64 - Addend: 0 -Symbols: - - Name: glob - Section: .text - Value: 0x0 - Size: 0 - Binding: STB_GLOBAL +.reloc ., BFD_RELOC_64, glob-0x8000000000000000 +.reloc .+1, BFD_RELOC_64, glob+0x7fffffffffffffff +.reloc .+2, BFD_RELOC_64, glob-1 +.reloc .+3, BFD_RELOC_64, glob+0x12345678 +.reloc .+4, BFD_RELOC_64, glob +.space 8 From 2aaeef1fad0c1b233f6d3ca67a6c05877dc9e998 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 16 Dec 2023 16:24:32 -0800 Subject: [PATCH 048/884] [Index] Fix recursive-cxx-member-calls.cpp b8f89b84bc26c46a5a10d01eb5414fbde3c8700a inadvertently replaced startswith/endswith with starts_with/ends_with even though the test uses a custom StringRef. This patch reverts the change. --- clang/test/Index/recursive-cxx-member-calls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Index/recursive-cxx-member-calls.cpp b/clang/test/Index/recursive-cxx-member-calls.cpp index 48fc8f14544c4..be908c506e747 100644 --- a/clang/test/Index/recursive-cxx-member-calls.cpp +++ b/clang/test/Index/recursive-cxx-member-calls.cpp @@ -99,7 +99,7 @@ using namespace clang; AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) { llvm::StringRef AttrName = Name->getName(); - if (AttrName.starts_with("__") && AttrName.ends_with("__")) + if (AttrName.startswith("__") && AttrName.endswith("__")) AttrName = AttrName.substr(2, AttrName.size() - 4); return llvm::StringSwitch < AttributeList::Kind > (AttrName) From 942b0901b09a89b09ebb75e75141e678a206cc36 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 16 Dec 2023 16:54:45 -0800 Subject: [PATCH 049/884] [llvm-objdump,test] Improve zero dumping and inline relocs tests --- .../X86/disassemble-zeroes-relocations.test | 7 ++++++ .../X86/elf-disassemble-relocs-exec.test | 25 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test index d61d7455570b1..62776f8c5a990 100644 --- a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test +++ b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test @@ -2,6 +2,7 @@ ## Check we do not skip zeroes blocks if have relocations pointed to these places. # RUN: llvm-objdump -D --reloc -j .rodata %t | FileCheck %s +# RUN: llvm-objdump -D --reloc -j .rodata --disassemble-symbols=rodata3 %t | FileCheck %s --check-prefix=CHECK2 # CHECK: 0000000000000000 : # CHECK-NEXT: 0: 00 00 addb %al, (%rax) @@ -30,6 +31,12 @@ # CHECK-NEXT: 43: 00 00 addb %al, (%rax) # CHECK-NEXT: 45: 00 00 addb %al, (%rax) +# CHECK2: Disassembly of section .rodata: +# CHECK2-EMPTY: +# CHECK2-NEXT: 0000000000000037 : +# CHECK2-NEXT: ... +# CHECK2-NOT: {{.}} + ## Check that without -reloc all zeroes would be omitted. # RUN: llvm-objdump -D %t | FileCheck %s --check-prefix=SKIP diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test index 631ee205a5c18..6c3c0bd1c9803 100644 --- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test +++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test @@ -4,6 +4,7 @@ # RUN: llvm-objdump --no-print-imm-hex -dr %t | FileCheck %s --match-full-lines --strict-whitespace # RUN: llvm-objdump --no-print-imm-hex -dr --no-leading-addr %t | FileCheck %s --check-prefix=NOADDR --match-full-lines --strict-whitespace # RUN: llvm-objdump --no-print-imm-hex -dr --no-addresses %t | FileCheck %s --check-prefix=NOADDR --match-full-lines --strict-whitespace +# RUN: llvm-objdump --no-print-imm-hex -dr --disassemble-symbols=text1 %t | FileCheck %s --check-prefix=CHECK2 --match-full-lines --strict-whitespace # RUN: yaml2obj -DBITS=32 %s -o %t.32 # RUN: llvm-objdump --no-print-imm-hex -dr --no-leading-addr %t.32 | FileCheck %s --check-prefix=NOADDR --match-full-lines --strict-whitespace @@ -15,6 +16,19 @@ # CHECK-NEXT: 400006: e8 fc fe ff ff callq 0x3fff07 # CHECK-NEXT: 0000000000400007: R_X86_64_PLT32 puts-0x4 # CHECK-NEXT: 40000b: 90 nop +# CHECK-EMPTY: +# CHECK-NEXT:000000000040000c : +# CHECK-NEXT: 40000c: bf 10 00 40 00 movl $4194320, %edi # imm = 0x400010 +# CHECK-NEXT: 000000000040000d: R_X86_64_32 .rodata +# CHECK-NOT: {{.}} + +## FIXME: --disassemble-symbols: remove inline relocs from skipped functions +# CHECK2:000000000040000c : +#CHECK2-NEXT: 40000c: bf 10 00 40 00 movl $4194320, %edi # imm = 0x400010 +#CHECK2-NEXT: 0000000000400002: R_X86_64_32 .rodata +#CHECK2-NEXT: 0000000000400007: R_X86_64_PLT32 puts-0x4 +#CHECK2-NEXT: 000000000040000d: R_X86_64_32 .rodata +#CHECK2-NOT: {{.}} # NOADDR:<.text>: # NOADDR-NEXT: 90 nop @@ -42,7 +56,7 @@ Sections: Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x400000 AddressAlign: 0x10 - Content: 90BF10004000E8FCFEFFFF90 + Content: 90BF10004000E8FCFEFFFF90BF10004000 - Name: .rodata Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] @@ -62,6 +76,10 @@ Sections: Symbol: puts Type: R_X86_64_PLT32 Addend: -4 + - Offset: 0x40000d + Symbol: .rodata + Type: R_X86_64_32 + Addend: 0 Symbols: - Name: .rodata Type: STT_SECTION @@ -70,4 +88,9 @@ Symbols: - Name: puts Type: STT_FUNC Binding: STB_GLOBAL + - Name: text1 + Type: STT_FUNC + Binding: STB_GLOBAL + Section: .text + Value: 0x40000c ... From ea979b24b0a755c9839e32dd716078ea816a0508 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Sun, 17 Dec 2023 13:19:27 +0900 Subject: [PATCH 050/884] [mlir][SparseTensor][NFC] Remove `isNestedIn` helper function (#75729) Use `Region::findAncestorBlockInRegion` instead of a custom IR traversal. --- .../SparseTensor/Transforms/SparseGPUCodegen.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp index 30ab2a1f18e3f..69fd1eb746ffe 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp @@ -1155,7 +1155,7 @@ struct ForallRewriter : public OpRewritePattern { block = arg.getOwner(); else block = val.getDefiningOp()->getBlock(); - if (!isNestedIn(block, forallOp)) + if (!forallOp.getRegion().findAncestorBlockInRegion(*block)) invariants.insert(val); } }); @@ -1208,15 +1208,6 @@ struct ForallRewriter : public OpRewritePattern { } private: - // Helper method to see if block appears in given loop. - static bool isNestedIn(Block *block, scf::ParallelOp forallOp) { - for (Operation *o = block->getParentOp(); o; o = o->getParentOp()) { - if (o == forallOp) - return true; - } - return false; - } - unsigned numThreads; }; From 5139299618cfc33eb7b4772cea5a8b60131dfc90 Mon Sep 17 00:00:00 2001 From: Carl Ritson Date: Sun, 17 Dec 2023 16:44:16 +0900 Subject: [PATCH 051/884] [AMDGPU] Track physical VGPRs used for SGPR spills (#75573) Physical VGPRs used for SGPR spills need to be tracked independent of WWM reserved registers. The WWM reserved set contains extra registers allocated during WWM pre-allocation pass. This causes SGPR spills allocated after WWM pre-allocation to overlap with WWM register usage, e.g. if frame pointer is spilt during prologue/epilog insertion. --- llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp | 3 ++- llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h | 1 + llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 48c341917ddec..e8142244b7db6 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -349,8 +349,9 @@ bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills( MBB.addLiveIn(LaneVGPR); MBB.sortUniqueLiveIns(); } + SpillPhysVGPRs.push_back(LaneVGPR); } else { - LaneVGPR = WWMReservedRegs.back(); + LaneVGPR = SpillPhysVGPRs.back(); } SGPRSpillsToPhysicalVGPRLanes[FI].push_back( diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h index 7ff50c80081d3..dc63ae44c528d 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -502,6 +502,7 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction, unsigned NumVirtualVGPRSpillLanes = 0; unsigned NumPhysicalVGPRSpillLanes = 0; SmallVector SpillVGPRs; + SmallVector SpillPhysVGPRs; using WWMSpillsMap = MapVector; // To track the registers used in instructions that can potentially modify the // inactive lanes. The WWM instructions and the writelane instructions for diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir index 35e205561a416..1473e667f894c 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir @@ -39,7 +39,7 @@ body: | ; GCN-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr5, $sgpr33, 12, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.72, addrspace 5) ; GCN-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr33, 16, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.74, addrspace 5) ; GCN-NEXT: $exec_lo = S_MOV_B32 killed $sgpr1 - ; GCN-NEXT: $vgpr4 = SI_SPILL_S32_TO_VGPR $sgpr0, 4, undef $vgpr4 + ; GCN-NEXT: $vgpr3 = SI_SPILL_S32_TO_VGPR $sgpr0, 4, undef $vgpr3 ; GCN-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 24, implicit-def dead $scc ; GCN-NEXT: renamable $vgpr5 = IMPLICIT_DEF ; GCN-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, $vgpr1 @@ -198,7 +198,7 @@ body: | ; GCN-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR $vgpr1, 1 ; GCN-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr1, 0 ; GCN-NEXT: KILL killed renamable $vgpr5 - ; GCN-NEXT: $sgpr0 = SI_RESTORE_S32_FROM_VGPR $vgpr4, 4 + ; GCN-NEXT: $sgpr0 = SI_RESTORE_S32_FROM_VGPR $vgpr3, 4 ; GCN-NEXT: $sgpr1 = S_XOR_SAVEEXEC_B32 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec ; GCN-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.69, addrspace 5) ; GCN-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.70, addrspace 5) From d08b59f3337777acda520469309514cc6d8e4547 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 17 Dec 2023 00:42:26 -0800 Subject: [PATCH 052/884] [test] Improve MC/X86/index-operations.s --- llvm/test/MC/X86/index-operations.s | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/llvm/test/MC/X86/index-operations.s b/llvm/test/MC/X86/index-operations.s index a355b7ae0760d..899cf4656549f 100644 --- a/llvm/test/MC/X86/index-operations.s +++ b/llvm/test/MC/X86/index-operations.s @@ -1,5 +1,5 @@ // RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=64 %s -// RUN: FileCheck --check-prefix=ERR64 < %t.err %s +// RUN: FileCheck --input-file=%t.err %s --check-prefix=ERR64 --implicit-check-not=error: // RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=32 %s // RUN: FileCheck --check-prefix=ERR32 < %t.err %s // RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=16 %s @@ -21,7 +21,7 @@ lodsb (%esi), %al // 16: lodsb (%esi), %al # encoding: [0x67,0xac] lodsb (%si), %al -// ERR64: invalid 16-bit base register +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register // 32: lodsb (%si), %al # encoding: [0x67,0xac] // 16: lodsb (%si), %al # encoding: [0xac] @@ -31,12 +31,12 @@ lodsl %gs:(%esi) // 16: lodsl %gs:(%esi), %eax # encoding: [0x67,0x65,0x66,0xad] lodsl (%edi), %eax -// ERR64: invalid operand +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand // ERR32: invalid operand // ERR16: invalid operand lodsl 44(%edi), %eax -// ERR64: invalid operand +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand // ERR32: invalid operand // ERR16: invalid operand @@ -56,7 +56,7 @@ stos %eax, (%edi) // 16: stosl %eax, %es:(%edi) # encoding: [0x67,0x66,0xab] stosb %al, %fs:(%edi) -// ERR64: invalid operand for instruction +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand for instruction // ERR32: invalid operand for instruction // ERR16: invalid operand for instruction @@ -86,12 +86,12 @@ scasq %es:(%edi) // ERR16: 64-bit scasl %es:(%edi), %al -// ERR64: invalid operand +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand // ERR32: invalid operand // ERR16: invalid operand scas %es:(%di), %ax -// ERR64: invalid 16-bit base register +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register // 16: scasw %es:(%di), %ax # encoding: [0xaf] // 32: scasw %es:(%di), %ax # encoding: [0x67,0x66,0xaf] @@ -106,7 +106,7 @@ cmpsw (%edi), (%esi) // 16: cmpsw %es:(%edi), (%esi) # encoding: [0x67,0xa7] cmpsb (%di), (%esi) -// ERR64: invalid 16-bit base register +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register // ERR32: mismatching source and destination // ERR16: mismatching source and destination @@ -146,7 +146,7 @@ insw %dx, (%edi) // 16: insw %dx, %es:(%edi) # encoding: [0x67,0x6d] insw %dx, (%bx) -// ERR64: invalid 16-bit base register +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register // 32: insw %dx, %es:(%di) # encoding: [0x67,0x66,0x6d] // 16: insw %dx, %es:(%di) # encoding: [0x6d] @@ -161,18 +161,20 @@ insw %dx, (%rbx) // ERR16: 64-bit movdir64b 291(%si), %ecx +// ERR64: error: invalid 16-bit base register // ERR32: invalid operand // ERR16: invalid operand movdir64b 291(%esi), %cx +// ERR64: error: invalid operand for instruction // ERR32: invalid operand // ERR16: invalid operand movdir64b (%rdx), %r15d -// ERR64: invalid operand +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand movdir64b (%edx), %r15 -// ERR64: invalid operand +// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand movdir64b (%eip), %ebx // 64: movdir64b (%eip), %ebx # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00] @@ -185,4 +187,4 @@ movdir64b 291(%esi, %eiz, 4), %ebx // 32: movdir64b 291(%esi,%eiz,4), %ebx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00] movdir64b 291(%rsi, %riz, 4), %rbx -// 64: movdir64b 291(%rsi,%riz,4), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00] \ No newline at end of file +// 64: movdir64b 291(%rsi,%riz,4), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00] From a3952b4f022ce03c778ecc3b44ffff350b512735 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 00:57:24 -0800 Subject: [PATCH 053/884] [Analysis] Remove unused forward declarations (NFC) --- llvm/include/llvm/Analysis/AliasAnalysis.h | 1 - llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h | 1 - llvm/include/llvm/Analysis/InstructionSimplify.h | 1 - 3 files changed, 3 deletions(-) diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 081783e243678..e1cfb025fb658 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -64,7 +64,6 @@ class LoopInfo; class PreservedAnalyses; class TargetLibraryInfo; class Value; -template class SmallPtrSetImpl; /// The possible results of an alias query. /// diff --git a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h index 20bcbc592afbd..e4f152c232aa6 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h +++ b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h @@ -29,7 +29,6 @@ namespace llvm { class AAResults; class Function; -class FunctionPass; class AAEvaluator : public PassInfoMixin { int64_t FunctionCount = 0; diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h index c626a6522d017..a29955a06cf4e 100644 --- a/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -45,7 +45,6 @@ class DominatorTree; class Function; class Instruction; struct LoopStandardAnalysisResults; -class MDNode; class Pass; template class SmallSetVector; class TargetLibraryInfo; From 3a1ae2f46db473cfde4baa6e1b090f5dae67e8db Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sun, 17 Dec 2023 11:42:35 +0100 Subject: [PATCH 054/884] [mlir][vector] Fix invalid `LoadOp` indices being created (#75519) Fixes https://github.com/llvm/llvm-project/issues/71326. The cause of the issue was that a new `LoadOp` was created which looked something like: ```mlir %arg4 = func.func main(%arg1 : index, %arg2 : index) { %alloca_0 = memref.alloca() : memref> %1 = vector.type_cast %alloca_0 : memref> to memref<1xvector<32xi1>> %2 = memref.load %1[%arg1, %arg2] : memref<1xvector<32xi1>> return } ``` which crashed inside the `LoadOp::verify`. Note here that `%alloca_0` is 0 dimensional, `%1` has one dimension, but `memref.load` tries to index `%1` with two indices. This is now fixed by using the fact that `unpackOneDim` always unpacks one dim https://github.com/llvm/llvm-project/blob/1bce61e6b01b38e04260be4f422bbae59c34c766/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp#L897-L903 and so the `loadOp` should just index only one dimension. --------- Co-authored-by: Benjamin Maxwell --- .../Conversion/VectorToSCF/VectorToSCF.cpp | 27 ++++++++++++------- mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 6 +++-- .../Conversion/VectorToSCF/vector-to-scf.mlir | 17 ++++++++++++ mlir/test/Dialect/MemRef/invalid.mlir | 9 +++++++ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp index 2ee314e9fedfe..2026d0cd216a9 100644 --- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp +++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp @@ -369,7 +369,7 @@ struct Strategy { /// Retrieve the indices of the current StoreOp that stores into the buffer. static void getBufferIndices(TransferReadOp xferOp, SmallVector &indices) { - auto storeOp = getStoreOp(xferOp); + memref::StoreOp storeOp = getStoreOp(xferOp); auto prevIndices = memref::StoreOpAdaptor(storeOp).getIndices(); indices.append(prevIndices.begin(), prevIndices.end()); } @@ -591,8 +591,8 @@ struct PrepareTransferReadConversion if (checkPrepareXferOp(xferOp, options).failed()) return failure(); - auto buffers = allocBuffers(rewriter, xferOp); - auto *newXfer = rewriter.clone(*xferOp.getOperation()); + BufferAllocs buffers = allocBuffers(rewriter, xferOp); + Operation *newXfer = rewriter.clone(*xferOp.getOperation()); newXfer->setAttr(kPassLabel, rewriter.getUnitAttr()); if (xferOp.getMask()) { dyn_cast(newXfer).getMaskMutable().assign( @@ -885,8 +885,7 @@ struct TransferOpConversion : public VectorToSCFPattern { // If the xferOp has a mask: Find and cast mask buffer. Value castedMaskBuffer; if (xferOp.getMask()) { - auto maskBuffer = getMaskBuffer(xferOp); - auto maskBufferType = dyn_cast(maskBuffer.getType()); + Value maskBuffer = getMaskBuffer(xferOp); if (xferOp.isBroadcastDim(0) || xferOp.getMaskType().getRank() == 1) { // Do not unpack a dimension of the mask, if: // * To-be-unpacked transfer op dimension is a broadcast. @@ -897,7 +896,8 @@ struct TransferOpConversion : public VectorToSCFPattern { } else { // It's safe to assume the mask buffer can be unpacked if the data // buffer was unpacked. - auto castedMaskType = *unpackOneDim(maskBufferType); + auto maskBufferType = dyn_cast(maskBuffer.getType()); + MemRefType castedMaskType = *unpackOneDim(maskBufferType); castedMaskBuffer = locB.create(castedMaskType, maskBuffer); } @@ -938,11 +938,18 @@ struct TransferOpConversion : public VectorToSCFPattern { b.setInsertionPoint(newXfer); // Insert load before newXfer. SmallVector loadIndices; - Strategy::getBufferIndices(xferOp, loadIndices); - // In case of broadcast: Use same indices to load from memref - // as before. - if (!xferOp.isBroadcastDim(0)) + if (auto memrefType = + castedMaskBuffer.getType().dyn_cast()) { + // If castedMaskBuffer is a memref, then one dim was + // unpacked; see above. loadIndices.push_back(iv); + } else { + Strategy::getBufferIndices(xferOp, loadIndices); + // In case of broadcast: Use same indices to load from + // memref as before. + if (!xferOp.isBroadcastDim(0)) + loadIndices.push_back(iv); + } auto mask = b.create(loc, castedMaskBuffer, loadIndices); diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp index 93327a28234ea..a332fe253ba64 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp @@ -1615,8 +1615,10 @@ GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { //===----------------------------------------------------------------------===// LogicalResult LoadOp::verify() { - if (getNumOperands() != 1 + getMemRefType().getRank()) - return emitOpError("incorrect number of indices for load"); + if (static_cast(getIndices().size()) != getMemRefType().getRank()) { + return emitOpError("incorrect number of indices for load, expected ") + << getMemRefType().getRank() << " but got " << getIndices().size(); + } return success(); } diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir index ad78f0c945b24..953fcee0c372f 100644 --- a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir +++ b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir @@ -740,6 +740,23 @@ func.func @cannot_lower_transfer_read_with_leading_scalable(%arg0: memref (d0, 0, 0, d3)> +func.func @does_not_crash_on_unpack_one_dim(%subview: memref<1x1x1x1xi32>, %mask: vector<1x1xi1>) -> vector<1x1x1x1xi32> { + %c0 = arith.constant 0 : index + %c0_i32 = arith.constant 0 : i32 + %3 = vector.transfer_read %subview[%c0, %c0, %c0, %c0], %c0_i32, %mask {permutation_map = #map1} + : memref<1x1x1x1xi32>, vector<1x1x1x1xi32> + return %3 : vector<1x1x1x1xi32> +} +// CHECK-LABEL: func.func @does_not_crash_on_unpack_one_dim +// CHECK: %[[ALLOCA_0:.*]] = memref.alloca() : memref> +// CHECK: %[[MASK:.*]] = vector.type_cast %[[ALLOCA_0]] : memref> to memref<1xvector<1xi1>> +// CHECK: memref.load %[[MASK]][%{{.*}}] : memref<1xvector<1xi1>> + +// ----- + // FULL-UNROLL-LABEL: @cannot_fully_unroll_transfer_write_of_nd_scalable_vector func.func @cannot_fully_unroll_transfer_write_of_nd_scalable_vector(%vec: vector<[4]x[4]xf32>, %memref: memref) { // FULL-UNROLL-NOT: vector.extract diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir index 55b759cbb3ce7..f9b870f77266e 100644 --- a/mlir/test/Dialect/MemRef/invalid.mlir +++ b/mlir/test/Dialect/MemRef/invalid.mlir @@ -896,6 +896,15 @@ func.func @bad_alloc_wrong_symbol_count() { // ----- +func.func @load_invalid_memref_indexes() { + %0 = memref.alloca() : memref<10xi32> + %c0 = arith.constant 0 : index + // expected-error@+1 {{incorrect number of indices for load, expected 1 but got 2}} + %1 = memref.load %0[%c0, %c0] : memref<10xi32> +} + +// ----- + func.func @test_store_zero_results() { ^bb0: %0 = memref.alloc() : memref<1024x64xf32, affine_map<(d0, d1) -> (d0, d1)>, 1> From 3eaed9e6f574f59d76389c055b047ef5c50afb8a Mon Sep 17 00:00:00 2001 From: melonedo <44501064+melonedo@users.noreply.github.com> Date: Sun, 17 Dec 2023 19:29:40 +0800 Subject: [PATCH 055/884] [RISCV] Implement intrinsics for XCVbitmanip extension in CV32E40P (#74993) Implement XCVbitmanip intrinsics for CV32E40P according to the specification. This commit is part of a patch-set to upstream the vendor specific extensions of CV32E40P that need LLVM intrinsics to implement Clang builtins. Contributors: @CharKeaney, @ChunyuLiao, @jeremybennett, @lewis-revill, @NandniJamnadas, @PaoloS02, @simonpcook, @xingmingjie. Spec: https://github.com/openhwgroup/core-v-sw/blob/05481cf0ef7aa7b09067b14ff3f71faead7ba310/specifications/corev-builtin-spec.md#listing-of-pulp-bit-manipulation-builtins-xcvbitmanip. Previously reviewed on Phabricator: https://reviews.llvm.org/D157510. Parallel GCC patch: https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635795.html. Co-authored-by: melonedo --- llvm/include/llvm/IR/IntrinsicsRISCV.td | 1 + llvm/include/llvm/IR/IntrinsicsRISCVXCV.td | 37 +++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 23 +- llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td | 48 +++- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 4 +- llvm/test/CodeGen/RISCV/xcvbitmanip.ll | 231 ++++++++++++++++++ 6 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 llvm/include/llvm/IR/IntrinsicsRISCVXCV.td create mode 100644 llvm/test/CodeGen/RISCV/xcvbitmanip.ll diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 20c6a525a86ba..fc830fca392fc 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -1879,3 +1879,4 @@ let TargetPrefix = "riscv" in { //===----------------------------------------------------------------------===// include "llvm/IR/IntrinsicsRISCVXTHead.td" include "llvm/IR/IntrinsicsRISCVXsf.td" +include "llvm/IR/IntrinsicsRISCVXCV.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXCV.td b/llvm/include/llvm/IR/IntrinsicsRISCVXCV.td new file mode 100644 index 0000000000000..f1590ad66e362 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXCV.td @@ -0,0 +1,37 @@ +//===- IntrinsicsRISCVXCV.td - CORE-V intrinsics -----------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines all of the CORE-V vendor intrinsics for RISC-V. +// +//===----------------------------------------------------------------------===// + +class ScalarCoreVBitManipGprGprIntrinsic + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable]>; + +class ScalarCoreVBitManipGprIntrinsic + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], + [IntrNoMem, IntrSpeculatable]>; + +let TargetPrefix = "riscv" in { + def int_riscv_cv_bitmanip_extract : ScalarCoreVBitManipGprGprIntrinsic; + def int_riscv_cv_bitmanip_extractu : ScalarCoreVBitManipGprGprIntrinsic; + def int_riscv_cv_bitmanip_bclr : ScalarCoreVBitManipGprGprIntrinsic; + def int_riscv_cv_bitmanip_bset : ScalarCoreVBitManipGprGprIntrinsic; + + def int_riscv_cv_bitmanip_insert + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable]>; + + def int_riscv_cv_bitmanip_clb : ScalarCoreVBitManipGprIntrinsic; + + def int_riscv_cv_bitmanip_bitrev + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable, + ImmArg>, ImmArg>]>; +} // TargetPrefix = "riscv" diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4a8ff73ec4729..782a9e1db569f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -335,6 +335,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, if (Subtarget.is64Bit()) setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom); setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom); + } else if (Subtarget.hasVendorXCVbitmanip()) { + setOperationAction({ISD::ROTL}, XLenVT, Expand); } else { setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand); if (RV64LegalI32 && Subtarget.is64Bit()) @@ -355,9 +357,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ? Promote : Expand); - // Zbkb can use rev8+brev8 to implement bitreverse. - setOperationAction(ISD::BITREVERSE, XLenVT, - Subtarget.hasStdExtZbkb() ? Custom : Expand); + + if (Subtarget.hasVendorXCVbitmanip()) { + setOperationAction(ISD::BITREVERSE, XLenVT, Legal); + } else { + // Zbkb can use rev8+brev8 to implement bitreverse. + setOperationAction(ISD::BITREVERSE, XLenVT, + Subtarget.hasStdExtZbkb() ? Custom : Expand); + } if (Subtarget.hasStdExtZbb()) { setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT, @@ -372,13 +379,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, else setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom); } - } else { + } else if (!Subtarget.hasVendorXCVbitmanip()) { setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand); if (RV64LegalI32 && Subtarget.is64Bit()) setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand); } - if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb()) { + if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || + Subtarget.hasVendorXCVbitmanip()) { // We need the custom lowering to make sure that the resulting sequence // for the 32bit case is efficient on 64bit targets. if (Subtarget.is64Bit()) { @@ -1796,11 +1804,12 @@ bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const { } bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const { - return Subtarget.hasStdExtZbb(); + return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip(); } bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { - return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb(); + return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || + Subtarget.hasVendorXCVbitmanip(); } bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial( diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td index 6622e811bbb86..924e91e15c348 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td @@ -33,7 +33,7 @@ let DecoderNamespace = "XCVbitmanip" in { class CVBitManipR funct7, string opcodestr> : RVInstR { + (ins GPR:$rs1), opcodestr, "$rd, $rs1"> { let rs2 = 0b00000; } } @@ -658,3 +658,49 @@ let Predicates = [HasVendorXCVelw, IsRV32], hasSideEffects = 0, // Event load def CV_ELW : CVLoad_ri<0b011, "cv.elw">; } + +def cv_tuimm2 : TImmLeaf(Imm);}]>; +def cv_tuimm5 : TImmLeaf(Imm);}]>; +def cv_uimm10 : ImmLeaf(Imm);}]>; + +def CV_LO5: SDNodeXFormgetTargetConstant(N->getZExtValue() & 0x1f, SDLoc(N), + N->getValueType(0)); +}]>; + +def CV_HI5: SDNodeXFormgetTargetConstant(N->getZExtValue() >> 5, SDLoc(N), + N->getValueType(0)); +}]>; + +multiclass PatCoreVBitManip { + def : PatGprGpr("CV_" # NAME # "R")>; + def : Pat<(intr GPR:$rs1, cv_uimm10:$imm), + (!cast("CV_" # NAME) + GPR:$rs1, (CV_HI5 cv_uimm10:$imm), (CV_LO5 cv_uimm10:$imm))>; +} + +let Predicates = [HasVendorXCVbitmanip, IsRV32] in { + defm EXTRACT : PatCoreVBitManip; + defm EXTRACTU : PatCoreVBitManip; + defm BCLR : PatCoreVBitManip; + defm BSET : PatCoreVBitManip; + + def : Pat<(int_riscv_cv_bitmanip_insert GPR:$rs1, GPR:$rs2, GPR:$rd), + (CV_INSERTR GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_cv_bitmanip_insert GPR:$rs1, cv_uimm10:$imm, GPR:$rd), + (CV_INSERT GPR:$rd, GPR:$rs1, (CV_HI5 cv_uimm10:$imm), + (CV_LO5 cv_uimm10:$imm))>; + + def : PatGpr; + def : PatGpr; + def : PatGpr; + def : PatGpr; + + def : PatGprGpr; + + def : Pat<(int_riscv_cv_bitmanip_bitrev GPR:$rs1, cv_tuimm5:$pts, + cv_tuimm2:$radix), + (CV_BITREV GPR:$rs1, cv_tuimm2:$radix, cv_tuimm5:$pts)>; + def : Pat<(bitreverse (XLenVT GPR:$rs)), (CV_BITREV GPR:$rs, 0, 0)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 3a2f2f39cd1c9..4614446b2150b 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -172,7 +172,9 @@ RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, TargetTransformInfo::PopcntSupportKind RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) { assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); - return ST->hasStdExtZbb() ? TTI::PSK_FastHardware : TTI::PSK_Software; + return ST->hasStdExtZbb() || ST->hasVendorXCVbitmanip() + ? TTI::PSK_FastHardware + : TTI::PSK_Software; } bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const { diff --git a/llvm/test/CodeGen/RISCV/xcvbitmanip.ll b/llvm/test/CodeGen/RISCV/xcvbitmanip.ll new file mode 100644 index 0000000000000..d25ff28475c4b --- /dev/null +++ b/llvm/test/CodeGen/RISCV/xcvbitmanip.ll @@ -0,0 +1,231 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -O0 -mtriple=riscv32 -mattr=+xcvbitmanip -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-O0 +; RUN: llc -O3 -mtriple=riscv32 -mattr=+xcvbitmanip -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-O3 + +declare i32 @llvm.riscv.cv.bitmanip.extract(i32, i32) + +define i32 @test.cv.extractr(i32 %a, i32 %b) { +; CHECK-LABEL: test.cv.extractr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.extractr a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.extract(i32 %a, i32 %b) + ret i32 %1 +} + +define i32 @test.cv.extract(i32 %a) { +; CHECK-LABEL: test.cv.extract: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.extract a0, a0, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.extract(i32 %a, i32 65) + ret i32 %1 +} + +define i32 @test.cv.extract1023(i32 %a) { +; CHECK-LABEL: test.cv.extract1023: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.extract a0, a0, 31, 31 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.extract(i32 %a, i32 1023) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.extractu(i32, i32) + +define i32 @test.cv.extractur(i32 %a, i32 %b) { +; CHECK-LABEL: test.cv.extractur: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.extractur a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.extractu(i32 %a, i32 %b) + ret i32 %1 +} + +define i32 @test.cv.extractu(i32 %a) { +; CHECK-LABEL: test.cv.extractu: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.extractu a0, a0, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.extractu(i32 %a, i32 65) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.insert(i32, i32, i32) + +define i32 @test.cv.insert(i32 %c, i32 %a) { +; CHECK-LABEL: test.cv.insert: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.insert a0, a1, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.insert(i32 %a, i32 65, i32 %c) + ret i32 %1 +} + +define i32 @test.cv.insertr(i32 %c, i32 %b, i32 %a) { +; CHECK-LABEL: test.cv.insertr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.insertr a0, a2, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.insert(i32 %a, i32 %b, i32 %c) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.bclr(i32, i32) + +define i32 @test.cv.bclrr(i32 %a, i32 %b) { +; CHECK-LABEL: test.cv.bclrr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bclrr a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.bclr(i32 %a, i32 %b) + ret i32 %1 +} + +define i32 @test.cv.bclr(i32 %a) { +; CHECK-LABEL: test.cv.bclr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bclr a0, a0, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.bclr(i32 %a, i32 65) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.bset(i32, i32) + +define i32 @test.cv.bsetr(i32 %a, i32 %b) { +; CHECK-LABEL: test.cv.bsetr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bsetr a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.bset(i32 %a, i32 %b) + ret i32 %1 +} + +define i32 @test.cv.bset(i32 %a) { +; CHECK-LABEL: test.cv.bset: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bset a0, a0, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.bset(i32 %a, i32 65) + ret i32 %1 +} + +declare i32 @llvm.cttz.i32(i32, i1) + +define i32 @test.cv.ff1(i32 %a) { +; CHECK-LABEL: test.cv.ff1: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.ff1 a0, a0 +; CHECK-NEXT: ret + %1 = call i32 @llvm.cttz.i32(i32 %a, i1 0) + ret i32 %1 +} + +declare i32 @llvm.ctlz.i32(i32, i1) + +define i32 @test.cv.fl1(i32 %a) { +; CHECK-LABEL: test.cv.fl1: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.fl1 a0, a0 +; CHECK-NEXT: ret + %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 0) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.clb(i32) + +define i32 @test.cv.clb(i32 %a) { +; CHECK-LABEL: test.cv.clb: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.clb a0, a0 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.clb(i32 %a) + ret i32 %1 +} + +declare i32 @llvm.ctpop(i32) + +define i32 @test.cv.cnt(i32 %a) { +; CHECK-LABEL: test.cv.cnt: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.cnt a0, a0 +; CHECK-NEXT: ret + %1 = call i32 @llvm.ctpop(i32 %a) + ret i32 %1 +} + +declare i32 @llvm.fshl.i32(i32, i32, i32) + +define i32 @test.llvm.fshl.imm(i32 %a) { +; CHECK-LABEL: test.llvm.fshl.imm: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 30 +; CHECK-NEXT: cv.ror a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 2) + ret i32 %1 +} + +define i32 @test.llvm.fshl.reg(i32 %a, i32 %b) { +; CHECK-O0-LABEL: test.llvm.fshl.reg: +; CHECK-O0: # %bb.0: +; CHECK-O0-NEXT: mv a2, a1 +; CHECK-O0-NEXT: li a1, 0 +; CHECK-O0-NEXT: sub a1, a1, a2 +; CHECK-O0-NEXT: cv.ror a0, a0, a1 +; CHECK-O0-NEXT: ret +; +; CHECK-O3-LABEL: test.llvm.fshl.reg: +; CHECK-O3: # %bb.0: +; CHECK-O3-NEXT: neg a1, a1 +; CHECK-O3-NEXT: cv.ror a0, a0, a1 +; CHECK-O3-NEXT: ret + %1 = call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 %b) + ret i32 %1 +} + +declare i32 @llvm.fshr.i32(i32, i32, i32) + +define i32 @test.llvm.fshr.imm(i32 %a) { +; CHECK-LABEL: test.llvm.fshr.imm: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 2 +; CHECK-NEXT: cv.ror a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 2) + ret i32 %1 +} + +define i32 @test.llvm.fshr.reg(i32 %a, i32 %b) { +; CHECK-LABEL: test.llvm.fshr.reg: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.ror a0, a0, a1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 %b) + ret i32 %1 +} + +declare i32 @llvm.riscv.cv.bitmanip.bitrev(i32, i32, i32) + +define i32 @test.cv.bitrev(i32 %a) { +; CHECK-LABEL: test.cv.bitrev: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bitrev a0, a0, 2, 1 +; CHECK-NEXT: ret + %1 = call i32 @llvm.riscv.cv.bitmanip.bitrev(i32 %a, i32 1, i32 2) + ret i32 %1 +} + +declare i32 @llvm.bitreverse(i32) + +define i32 @test.llvm.bitrev(i32 %a) { +; CHECK-LABEL: test.llvm.bitrev: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.bitrev a0, a0, 0, 0 +; CHECK-NEXT: ret + %1 = call i32 @llvm.bitreverse(i32 %a) + ret i32 %1 +} From fb877c19c048040702bb99423b0f11539192e89c Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 17 Dec 2023 11:31:12 +0000 Subject: [PATCH 056/884] [X86] combineLoad - don't bother truncating the alternative target constant data. NFC. We only iterate over the original target constant/undef width, so keep the alternative data in its original form. This should help if we try to merge constant data in the future. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 99c492087a458..13f69883ad6d5 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -49946,12 +49946,10 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, } auto MatchingBits = [](const APInt &Undefs, const APInt &UserUndefs, ArrayRef Bits, ArrayRef UserBits) { - if (!UserUndefs.isSubsetOf(Undefs)) - return false; for (unsigned I = 0, E = Undefs.getBitWidth(); I != E; ++I) { if (Undefs[I]) continue; - if (Bits[I] != UserBits[I]) + if (UserUndefs[I] || Bits[I] != UserBits[I]) return false; } return true; @@ -49970,8 +49968,6 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, if (getTargetConstantBitsFromNode(SDValue(N, 0), 8, Undefs, Bits) && getTargetConstantBitsFromNode(SDValue(User, 0), 8, UserUndefs, UserBits)) { - UserUndefs = UserUndefs.trunc(Undefs.getBitWidth()); - UserBits.truncate(Bits.size()); if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) { SDValue Extract = extractSubVector( SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits()); From 9f5afc3de95d6f2b5f85024a8cf7f021fef41db0 Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sun, 17 Dec 2023 12:34:17 +0100 Subject: [PATCH 057/884] Revert "[mlir][vector] Fix invalid `LoadOp` indices being created (#75519)" This reverts commit 3a1ae2f46db473cfde4baa6e1b090f5dae67e8db. --- .../Conversion/VectorToSCF/VectorToSCF.cpp | 27 +++++++------------ mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 6 ++--- .../Conversion/VectorToSCF/vector-to-scf.mlir | 17 ------------ mlir/test/Dialect/MemRef/invalid.mlir | 9 ------- 4 files changed, 12 insertions(+), 47 deletions(-) diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp index 2026d0cd216a9..2ee314e9fedfe 100644 --- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp +++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp @@ -369,7 +369,7 @@ struct Strategy { /// Retrieve the indices of the current StoreOp that stores into the buffer. static void getBufferIndices(TransferReadOp xferOp, SmallVector &indices) { - memref::StoreOp storeOp = getStoreOp(xferOp); + auto storeOp = getStoreOp(xferOp); auto prevIndices = memref::StoreOpAdaptor(storeOp).getIndices(); indices.append(prevIndices.begin(), prevIndices.end()); } @@ -591,8 +591,8 @@ struct PrepareTransferReadConversion if (checkPrepareXferOp(xferOp, options).failed()) return failure(); - BufferAllocs buffers = allocBuffers(rewriter, xferOp); - Operation *newXfer = rewriter.clone(*xferOp.getOperation()); + auto buffers = allocBuffers(rewriter, xferOp); + auto *newXfer = rewriter.clone(*xferOp.getOperation()); newXfer->setAttr(kPassLabel, rewriter.getUnitAttr()); if (xferOp.getMask()) { dyn_cast(newXfer).getMaskMutable().assign( @@ -885,7 +885,8 @@ struct TransferOpConversion : public VectorToSCFPattern { // If the xferOp has a mask: Find and cast mask buffer. Value castedMaskBuffer; if (xferOp.getMask()) { - Value maskBuffer = getMaskBuffer(xferOp); + auto maskBuffer = getMaskBuffer(xferOp); + auto maskBufferType = dyn_cast(maskBuffer.getType()); if (xferOp.isBroadcastDim(0) || xferOp.getMaskType().getRank() == 1) { // Do not unpack a dimension of the mask, if: // * To-be-unpacked transfer op dimension is a broadcast. @@ -896,8 +897,7 @@ struct TransferOpConversion : public VectorToSCFPattern { } else { // It's safe to assume the mask buffer can be unpacked if the data // buffer was unpacked. - auto maskBufferType = dyn_cast(maskBuffer.getType()); - MemRefType castedMaskType = *unpackOneDim(maskBufferType); + auto castedMaskType = *unpackOneDim(maskBufferType); castedMaskBuffer = locB.create(castedMaskType, maskBuffer); } @@ -938,18 +938,11 @@ struct TransferOpConversion : public VectorToSCFPattern { b.setInsertionPoint(newXfer); // Insert load before newXfer. SmallVector loadIndices; - if (auto memrefType = - castedMaskBuffer.getType().dyn_cast()) { - // If castedMaskBuffer is a memref, then one dim was - // unpacked; see above. + Strategy::getBufferIndices(xferOp, loadIndices); + // In case of broadcast: Use same indices to load from memref + // as before. + if (!xferOp.isBroadcastDim(0)) loadIndices.push_back(iv); - } else { - Strategy::getBufferIndices(xferOp, loadIndices); - // In case of broadcast: Use same indices to load from - // memref as before. - if (!xferOp.isBroadcastDim(0)) - loadIndices.push_back(iv); - } auto mask = b.create(loc, castedMaskBuffer, loadIndices); diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp index a332fe253ba64..93327a28234ea 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp @@ -1615,10 +1615,8 @@ GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { //===----------------------------------------------------------------------===// LogicalResult LoadOp::verify() { - if (static_cast(getIndices().size()) != getMemRefType().getRank()) { - return emitOpError("incorrect number of indices for load, expected ") - << getMemRefType().getRank() << " but got " << getIndices().size(); - } + if (getNumOperands() != 1 + getMemRefType().getRank()) + return emitOpError("incorrect number of indices for load"); return success(); } diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir index 953fcee0c372f..ad78f0c945b24 100644 --- a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir +++ b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir @@ -740,23 +740,6 @@ func.func @cannot_lower_transfer_read_with_leading_scalable(%arg0: memref (d0, 0, 0, d3)> -func.func @does_not_crash_on_unpack_one_dim(%subview: memref<1x1x1x1xi32>, %mask: vector<1x1xi1>) -> vector<1x1x1x1xi32> { - %c0 = arith.constant 0 : index - %c0_i32 = arith.constant 0 : i32 - %3 = vector.transfer_read %subview[%c0, %c0, %c0, %c0], %c0_i32, %mask {permutation_map = #map1} - : memref<1x1x1x1xi32>, vector<1x1x1x1xi32> - return %3 : vector<1x1x1x1xi32> -} -// CHECK-LABEL: func.func @does_not_crash_on_unpack_one_dim -// CHECK: %[[ALLOCA_0:.*]] = memref.alloca() : memref> -// CHECK: %[[MASK:.*]] = vector.type_cast %[[ALLOCA_0]] : memref> to memref<1xvector<1xi1>> -// CHECK: memref.load %[[MASK]][%{{.*}}] : memref<1xvector<1xi1>> - -// ----- - // FULL-UNROLL-LABEL: @cannot_fully_unroll_transfer_write_of_nd_scalable_vector func.func @cannot_fully_unroll_transfer_write_of_nd_scalable_vector(%vec: vector<[4]x[4]xf32>, %memref: memref) { // FULL-UNROLL-NOT: vector.extract diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir index f9b870f77266e..55b759cbb3ce7 100644 --- a/mlir/test/Dialect/MemRef/invalid.mlir +++ b/mlir/test/Dialect/MemRef/invalid.mlir @@ -896,15 +896,6 @@ func.func @bad_alloc_wrong_symbol_count() { // ----- -func.func @load_invalid_memref_indexes() { - %0 = memref.alloca() : memref<10xi32> - %c0 = arith.constant 0 : index - // expected-error@+1 {{incorrect number of indices for load, expected 1 but got 2}} - %1 = memref.load %0[%c0, %c0] : memref<10xi32> -} - -// ----- - func.func @test_store_zero_results() { ^bb0: %0 = memref.alloc() : memref<1024x64xf32, affine_map<(d0, d1) -> (d0, d1)>, 1> From b6cce87110072a2db19276e042cd40b06285abbc Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Sun, 17 Dec 2023 19:59:42 +0800 Subject: [PATCH 058/884] [RISCV] Fix -Wbraced-scalar-init in RISCVISelLowering.cpp (NFC) llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp:339:24: error: braces around scalar initializer [-Werror,-Wbraced-scalar-init] 339 | setOperationAction({ISD::ROTL}, XLenVT, Expand); | ^~~~~~~~~~~ 1 error generated. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 782a9e1db569f..03e994586d0c4 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -336,7 +336,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom); setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom); } else if (Subtarget.hasVendorXCVbitmanip()) { - setOperationAction({ISD::ROTL}, XLenVT, Expand); + setOperationAction(ISD::ROTL, XLenVT, Expand); } else { setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand); if (RV64LegalI32 && Subtarget.is64Bit()) From a418be96de7872f6058207c695ef4698cb1dbb93 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 17 Dec 2023 13:44:08 +0000 Subject: [PATCH 059/884] [X86] combineLoad - extract target constants at the minimum scalar element width. No need to extract at the byte level, and will make it easier to reconstruct constants in a future patch. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 13f69883ad6d5..b80c766c7ffa7 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -49956,6 +49956,7 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, }; // See if we are loading a constant that matches in the lower // bits of a longer constant (but from a different constant pool ptr). + EVT UserVT = User->getValueType(0); SDValue UserPtr = cast(User)->getBasePtr(); const Constant *LdC = getTargetConstantFromBasePtr(Ptr); const Constant *UserC = getTargetConstantFromBasePtr(UserPtr); @@ -49965,9 +49966,12 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, if (LdSize < UserSize || !ISD::isNormalLoad(User)) { APInt Undefs, UserUndefs; SmallVector Bits, UserBits; - if (getTargetConstantBitsFromNode(SDValue(N, 0), 8, Undefs, Bits) && - getTargetConstantBitsFromNode(SDValue(User, 0), 8, UserUndefs, - UserBits)) { + unsigned NumBits = std::min(RegVT.getScalarSizeInBits(), + UserVT.getScalarSizeInBits()); + if (getTargetConstantBitsFromNode(SDValue(N, 0), NumBits, Undefs, + Bits) && + getTargetConstantBitsFromNode(SDValue(User, 0), NumBits, + UserUndefs, UserBits)) { if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) { SDValue Extract = extractSubVector( SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits()); From 4b3078ef2d8b4ce833c2b493421486bb25802b32 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 09:09:37 -0800 Subject: [PATCH 060/884] [CodeGen] Remove unnecessary includes (NFC) --- llvm/include/llvm/CodeGen/AntiDepBreaker.h | 1 - llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h | 1 - llvm/lib/CodeGen/MachineStableHash.cpp | 1 - llvm/unittests/CodeGen/DwarfStringPoolEntryRefTest.cpp | 1 - 4 files changed, 4 deletions(-) diff --git a/llvm/include/llvm/CodeGen/AntiDepBreaker.h b/llvm/include/llvm/CodeGen/AntiDepBreaker.h index c5c2b57486137..eba642684c95c 100644 --- a/llvm/include/llvm/CodeGen/AntiDepBreaker.h +++ b/llvm/include/llvm/CodeGen/AntiDepBreaker.h @@ -19,7 +19,6 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Support/Compiler.h" -#include #include #include diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index 2faa057f46073..dc772bb459c95 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -25,7 +25,6 @@ #include "llvm/CodeGen/LexicalScopes.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/Support/Casting.h" -#include #include #include diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp index debb2b3809e3f..1cd90474898e7 100644 --- a/llvm/lib/CodeGen/MachineStableHash.cpp +++ b/llvm/lib/CodeGen/MachineStableHash.cpp @@ -14,7 +14,6 @@ #include "llvm/CodeGen/MachineStableHash.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/unittests/CodeGen/DwarfStringPoolEntryRefTest.cpp b/llvm/unittests/CodeGen/DwarfStringPoolEntryRefTest.cpp index 25db003ea81de..8e82d11732e25 100644 --- a/llvm/unittests/CodeGen/DwarfStringPoolEntryRefTest.cpp +++ b/llvm/unittests/CodeGen/DwarfStringPoolEntryRefTest.cpp @@ -12,7 +12,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include using namespace llvm; From 2570c7e284c8ad1ee6db069e22d72b836ae935f6 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 09:09:39 -0800 Subject: [PATCH 061/884] [CodeGen] Remove unused forward declarations (NFC) --- llvm/include/llvm/CodeGen/AccelTable.h | 1 - llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 2 -- llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h | 1 - llvm/lib/CodeGen/AsmPrinter/DwarfFile.h | 1 - 4 files changed, 5 deletions(-) diff --git a/llvm/include/llvm/CodeGen/AccelTable.h b/llvm/include/llvm/CodeGen/AccelTable.h index 0f35fd3514fae..af874aa5e91a3 100644 --- a/llvm/include/llvm/CodeGen/AccelTable.h +++ b/llvm/include/llvm/CodeGen/AccelTable.h @@ -103,7 +103,6 @@ namespace llvm { class AsmPrinter; -class DwarfUnit; class DwarfDebug; class DwarfTypeUnit; class MCSymbol; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index a4e9c92b48976..e7debc652a0a8 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -28,11 +28,9 @@ namespace llvm { class GISelChangeObserver; -class APFloat; class APInt; class ConstantFP; class GPtrAdd; -class GStore; class GZExtLoad; class MachineIRBuilder; class MachineInstrBuilder; diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h b/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h index c9378ace4fd1b..6f553dc85c646 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h +++ b/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h @@ -18,7 +18,6 @@ namespace llvm { class AsmPrinter; class DbgVariable; class DwarfCompileUnit; -class MachineInstr; class MCSymbol; /// Byte stream of .debug_loc entries. diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h b/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h index e10fd2b2642ac..f76858fc2f36a 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h @@ -28,7 +28,6 @@ class DbgLabel; class DINode; class DILocalScope; class DwarfCompileUnit; -class DwarfTypeUnit; class DwarfUnit; class LexicalScope; class MCSection; From 6eaf15d05e3d4490bf0b32fea553027ae3a4e996 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 09:41:20 -0800 Subject: [PATCH 062/884] [Analysis] Use llvm::erase (NFC) --- llvm/unittests/Analysis/DomTreeUpdaterTest.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp b/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp index 4a5e2d73f962c..0777bbe3887bc 100644 --- a/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp +++ b/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp @@ -375,12 +375,7 @@ TEST(DomTreeUpdater, LazyUpdateDTInheritedPreds) { std::vector BasicBlocks; BasicBlocks.push_back(BB1); BasicBlocks.push_back(BB2); - auto Eraser = [&](BasicBlock *BB) { - BasicBlocks.erase( - std::remove_if(BasicBlocks.begin(), BasicBlocks.end(), - [&](const BasicBlock *i) { return i == BB; }), - BasicBlocks.end()); - }; + auto Eraser = [&](BasicBlock *BB) { llvm::erase(BasicBlocks, BB); }; ASSERT_EQ(BasicBlocks.size(), static_cast(2)); // Remove bb2 from F. This has to happen before the call to // applyUpdates() for DTU to detect there is no longer an edge between From 6655581038f8479f0f6942b7d34cbd6556d00a0e Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 09:41:22 -0800 Subject: [PATCH 063/884] [Dialect] Use llvm::is_contained (NFC) --- mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp index 902ad8fc19c5d..a6f2f435f36d6 100644 --- a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp +++ b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp @@ -231,9 +231,7 @@ static LogicalResult fillShardingOption(Operation *op, continue; for (int32_t axis : meshAxes) { - if (std::find(shardingOption.shardingArray[i].begin(), - shardingOption.shardingArray[i].end(), - axis) != shardingOption.shardingArray[i].end()) { + if (llvm::is_contained(shardingOption.shardingArray[i], axis)) { LLVM_DEBUG(DBGS() << "sharding option conflicts because mesh axes " << axis << " duplicate"); return failure(); From 6561efe142ae2a5d434ff646319b0bfb1dd39dee Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sun, 17 Dec 2023 20:24:47 +0100 Subject: [PATCH 064/884] [mlir][python][nfc] Test `-print-ir-after-all` (#75742) The functionality to `-print-ir-after-all` was added in https://github.com/llvm/llvm-project/commit/caa159f044a05f782701a525d8b0e8f346abbd64. This PR adds a test and, with that, some documentation. --------- Co-authored-by: Maksim Levental --- mlir/test/python/pass_manager.py | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mlir/test/python/pass_manager.py b/mlir/test/python/pass_manager.py index 0face028b73ff..43af80b53166c 100644 --- a/mlir/test/python/pass_manager.py +++ b/mlir/test/python/pass_manager.py @@ -281,3 +281,36 @@ def testPostPassOpInvalidation(): # CHECK: return # CHECK: } log(module) + + +# CHECK-LABEL: TEST: testPrintIrAfterAll +@run +def testPrintIrAfterAll(): + with Context() as ctx: + module = ModuleOp.parse( + """ + module { + func.func @main() { + %0 = arith.constant 10 + return + } + } + """ + ) + pm = PassManager.parse("builtin.module(canonicalize)") + ctx.enable_multithreading(False) + pm.enable_ir_printing() + # CHECK: // -----// IR Dump Before Canonicalizer (canonicalize) ('builtin.module' operation) //----- // + # CHECK: module { + # CHECK: func.func @main() { + # CHECK: %[[C10:.*]] = arith.constant 10 : i64 + # CHECK: return + # CHECK: } + # CHECK: } + # CHECK: // -----// IR Dump After Canonicalizer (canonicalize) ('builtin.module' operation) //----- // + # CHECK: module { + # CHECK: func.func @main() { + # CHECK: return + # CHECK: } + # CHECK: } + pm.run(module) From d14ee76181fba376a04cb50afd9ab30cc406ee90 Mon Sep 17 00:00:00 2001 From: darkbuck Date: Sun, 17 Dec 2023 15:02:10 -0500 Subject: [PATCH 065/884] [GISel][TableGen] Enhance default ops support (#75689) - Instead of checking the default ops directly, this change queries DAG default operands collected during patterns reading. It does not only simplify the code but also handle few cases where integer values are converted from convertible types, such as 'bits'. - A test case is added GlobalISelEmitter.td as the regression test of default 'bits' values. --- llvm/test/TableGen/GlobalISelEmitter.td | 90 +++++++++++++++-------- llvm/utils/TableGen/GlobalISelEmitter.cpp | 30 +++----- 2 files changed, 72 insertions(+), 48 deletions(-) diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index eab2acd6cb1e7..f9d7d2dcccdbb 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -59,6 +59,7 @@ def gi_cimm9 : GICustomOperandRenderer<"renderImm">; def m1 : OperandWithDefaultOps ; def Z : OperandWithDefaultOps ; def m1Z : OperandWithDefaultOps ; +def mb : OperandWithDefaultOps ; def HasA : Predicate<"Subtarget->hasA()">; def HasB : Predicate<"Subtarget->hasB()">; @@ -297,7 +298,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // R19C-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // R19C-NEXT: GIR_EraseFromParent, /*InsnID*/0, // R19C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// R19C-NEXT: // GIR_Coverage, 19, +// R19C-NEXT: // GIR_Coverage, 20, // R19C-NEXT: GIR_Done, // R19C-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] // @@ -330,12 +331,12 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21O-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // R21O-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32, // -// R21C-NEXT: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 19 // +// R21C-NEXT: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 20 // // R21C-NOT: GIR_Done, -// R21C: // GIR_Coverage, 19, +// R21C: // GIR_Coverage, 20, // R21C-NEXT: GIR_Done, // R21C-NEXT: // Label [[PREV_NUM]]: @[[PREV]] -// R21C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 21 // +// R21C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 22 // // // R21O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // R21O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), @@ -366,7 +367,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21C-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/1, /*MergeInsnID's*/0 // R21C-NEXT: GIR_EraseFromParent, /*InsnID*/0, // R21C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// R21C-NEXT: // GIR_Coverage, 21, +// R21C-NEXT: // GIR_Coverage, 22, // R21C-NEXT: GIR_Done, // R21C-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] // @@ -390,10 +391,10 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R20O-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // R20O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // -// R20N: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 21 // +// R20N: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 22 // // R20N: // Label [[PREV_NUM]]: @[[PREV]] // -// R20C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 20 // +// R20C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 21 // // // R20N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R20N-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SUB), @@ -415,7 +416,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R20C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), // R20C-NEXT: GIR_EraseFromParent, /*InsnID*/0, // R20C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// R20C-NEXT: // GIR_Coverage, 20, +// R20C-NEXT: // GIR_Coverage, 21, // R20C-NEXT: GIR_Done, // R20C-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] // @@ -455,7 +456,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // R00O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // -// R00C: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 20 // +// R00C: GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 21 // // R00C: // Label [[PREV_NUM]]: @[[PREV]] // // R00C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 0 // @@ -517,7 +518,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; // Size: 1978 bytes +// R00O-NEXT: }; // Size: 2007 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, @@ -709,6 +710,35 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1), def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1), [(set GPR32:$dst, (xor GPR32:$src1, -5))]>; +//===- Test a simple pattern with a default bits operand. -----------------===// +// +// NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), +// NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR), +// NOOPT-NEXT: // MIs[0] DstI[dst] +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// NOOPT-NEXT: // MIs[0] src1 +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// NOOPT-NEXT: // MIs[0] Operand 2 +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// NOOPT-NEXT: GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-6) +// NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -6:{ *:[i32] }) => (XORIb:{ *:[i32] } GPR32:{ *:[i32] }:$src1) +// NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::XORIb), +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] +// NOOPT-NEXT: GIR_AddImm8, /*InsnID*/0, /*Imm*/13, +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 +// NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, +// NOOPT-NEXT: // GIR_Coverage, 6, +// NOOPT-NEXT: GIR_Done, +// NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] + +// The -6 is just to distinguish it from the other cases. +def XORIb : I<(outs GPR32:$dst), (ins mb:$src2, GPR32:$src1), + [(set GPR32:$dst, (xor GPR32:$src1, -6))]>; + //===- Test a simple pattern with constant immediate operands. ------------===// // // This must precede the 3-register variants because constant immediates have @@ -733,7 +763,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1) // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // Wm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 22, +// NOOPT-NEXT: // GIR_Coverage, 23, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -774,7 +804,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src3 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 6, +// NOOPT-NEXT: // GIR_Coverage, 7, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -812,7 +842,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src3 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 27, +// NOOPT-NEXT: // GIR_Coverage, 28, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -836,7 +866,7 @@ def MULADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3), // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 7, +// NOOPT-NEXT: // GIR_Coverage, 8, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -859,7 +889,7 @@ def MOV1 : I<(outs GPR32:$dst), (ins), [(set GPR32:$dst, 1)]>; // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 8, +// NOOPT-NEXT: // GIR_Coverage, 9, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -883,7 +913,7 @@ def MOVimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm8:$i // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 9, +// NOOPT-NEXT: // GIR_Coverage, 10, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -907,7 +937,7 @@ def MOVimm9 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm9:$i // NOOPT-NEXT: GIR_CustomRenderer, /*InsnID*/0, /*OldInsnID*/0, /*Renderer*/GIMT_Encode2(GICR_renderImm), // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 10, +// NOOPT-NEXT: // GIR_Coverage, 11, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -930,7 +960,7 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIR_CopyFConstantAsFPImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 17, +// NOOPT-NEXT: // GIR_Coverage, 18, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -950,7 +980,7 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src1)<><> => (LOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 11, +// NOOPT-NEXT: // GIR_Coverage, 12, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -973,7 +1003,7 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<><> => (LOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 23, +// NOOPT-NEXT: // GIR_Coverage, 24, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -996,7 +1026,7 @@ def : Pat<(load GPR32:$src), // NOOPT-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src1)<><><> => (SEXTLOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SEXTLOAD), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 12, +// NOOPT-NEXT: // GIR_Coverage, 13, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1020,7 +1050,7 @@ def SEXTLOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 13, +// NOOPT-NEXT: // GIR_Coverage, 14, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1046,7 +1076,7 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 14, +// NOOPT-NEXT: // GIR_Coverage, 15, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1073,7 +1103,7 @@ def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32 // NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) => (InsnWithSpeciallyNamedDef:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InsnWithSpeciallyNamedDef), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 24, +// NOOPT-NEXT: // GIR_Coverage, 25, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1096,7 +1126,7 @@ def : Pat<(add i32:$samename, i32:$othername), // NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 25, +// NOOPT-NEXT: // GIR_Coverage, 26, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1125,7 +1155,7 @@ def : Pat<(add i32:$src1, i32:$src2), // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 15, +// NOOPT-NEXT: // GIR_Coverage, 16, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1148,7 +1178,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1), // NOOPT-NEXT: // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] }) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), // NOOPT-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(MyTarget::GPR32RegClassID), -// NOOPT-NEXT: // GIR_Coverage, 26, +// NOOPT-NEXT: // GIR_Coverage, 27, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1171,7 +1201,7 @@ def : Pat<(i32 (bitconvert FPR32:$src1)), // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 16, +// NOOPT-NEXT: // GIR_Coverage, 17, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1190,7 +1220,7 @@ def MOVfpimmz : I<(outs FPR32:$dst), (ins f32imm:$imm), [(set FPR32:$dst, fpimmz // NOOPT-NEXT: // (br (bb:{ *:[Other] }):$target) => (BR (bb:{ *:[Other] }):$target) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::BR), // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 18, +// NOOPT-NEXT: // GIR_Coverage, 19, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1198,5 +1228,5 @@ def BR : I<(outs), (ins unknown:$target), [(br bb:$target)]>; // NOOPT-NEXT: GIM_Reject, -// NOOPT-NEXT: }; // Size: 1680 bytes +// NOOPT-NEXT: }; // Size: 1738 bytes // NOOPT-NEXT: return MatchTable0; diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index f1b2ff68e3431..c204b9819dc21 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -408,7 +408,7 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter { const TreePatternNode *DstChild, const TreePatternNode *Src); Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, - DagInit *DefaultOps) const; + const DAGDefaultOperand &DefaultOp) const; Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder, const std::vector &ImplicitDefs) const; @@ -1681,11 +1681,11 @@ Expected GlobalISelEmitter::importExplicitUseRenderers( // overridden, or which we aren't letting it override; emit the 'default // ops' operands. - const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[InstOpNo]; - DagInit *DefaultOps = DstIOperand.Rec->getValueAsDag("DefaultOps"); - if (auto Error = importDefaultOperandRenderers(InsertPt, M, DstMIBuilder, - DefaultOps)) + Record *OperandNode = DstI->Operands[InstOpNo].Rec; + if (auto Error = importDefaultOperandRenderers( + InsertPt, M, DstMIBuilder, CGP.getDefaultOperand(OperandNode))) return std::move(Error); + ++NumDefaultOps; continue; } @@ -1710,22 +1710,16 @@ Expected GlobalISelEmitter::importExplicitUseRenderers( Error GlobalISelEmitter::importDefaultOperandRenderers( action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, - DagInit *DefaultOps) const { - for (const auto *DefaultOp : DefaultOps->getArgs()) { - std::optional OpTyOrNone; + const DAGDefaultOperand &DefaultOp) const { + for (const auto &Op : DefaultOp.DefaultOps) { + const auto *N = Op.get(); + if (!N->isLeaf()) + return failedImport("Could not add default op"); - // Look through ValueType operators. - if (const DagInit *DefaultDagOp = dyn_cast(DefaultOp)) { - if (const DefInit *DefaultDagOperator = - dyn_cast(DefaultDagOp->getOperator())) { - if (DefaultDagOperator->getDef()->isSubClassOf("ValueType")) { - OpTyOrNone = MVTToLLT(getValueType(DefaultDagOperator->getDef())); - DefaultOp = DefaultDagOp->getArg(0); - } - } - } + const auto *DefaultOp = N->getLeafValue(); if (const DefInit *DefaultDefOp = dyn_cast(DefaultOp)) { + std::optional OpTyOrNone = MVTToLLT(N->getSimpleType(0)); auto Def = DefaultDefOp->getDef(); if (Def->getName() == "undef_tied_input") { unsigned TempRegID = M.allocateTempRegID(); From fc520f8b29416a3b0738e6c8c3a6d4eee67e42a6 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Sun, 17 Dec 2023 17:34:46 +0100 Subject: [PATCH 066/884] [InstCombine] Precommit tests for PR75745 (NFC) --- .../test/Transforms/InstCombine/insert-const-shuf.ll | 1 + llvm/test/Transforms/InstCombine/vec_shuffle.ll | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll index 68dcc45e4b6c3..d2fa651b39449 100644 --- a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll +++ b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll @@ -92,6 +92,7 @@ define <3 x float> @twoShufUses(<3 x float> %x) { ; The inserted scalar constant index is out-of-bounds for the shuffle vector constant. +; FIXME: This is a miscompilation define <5 x i8> @longerMask(<3 x i8> %x) { ; CHECK-LABEL: @longerMask( ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> , <5 x i32> diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index 0081da2c0aad7..e1174007b0fe0 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -2332,3 +2332,15 @@ define <2 x float> @uitofp_shuf_narrow(<4 x i32> %x, <4 x i32> %y) { %r = shufflevector <4 x float> %nx, <4 x float> %ny, <2 x i32> ret <2 x float> %r } + +; FIXME: This is a miscompilation +define <4 x i16> @blend_elements_from_load(ptr align 8 %_0) { +; CHECK-LABEL: @blend_elements_from_load( +; CHECK-NEXT: [[LOAD:%.*]] = load <3 x i16>, ptr [[_0:%.*]], align 8 +; CHECK-NEXT: [[RV:%.*]] = shufflevector <3 x i16> , <3 x i16> [[LOAD]], <4 x i32> +; CHECK-NEXT: ret <4 x i16> [[RV]] +; + %load = load <3 x i16>, ptr %_0, align 8 + %rv = shufflevector <3 x i16> , <3 x i16> %load, <4 x i32> + ret <4 x i16> %rv +} From 151ddf07a6f7a6c1440c587f2df52b127f29f99c Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Sun, 17 Dec 2023 17:43:38 +0100 Subject: [PATCH 067/884] [InstCombine] Stop propagating `undef` when element is demanded Do not poison `undef` demanded elements in `SimplifyDemandedVectorElts`. A miscompilation issue has been addressed with refined checking. Proofs: https://alive2.llvm.org/ce/z/WA5oD5. --- .../lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp | 2 +- llvm/test/Transforms/InstCombine/insert-const-shuf.ll | 3 +-- llvm/test/Transforms/InstCombine/vec_shuffle.ll | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 846116a929b15..2490f5b9b97eb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1378,7 +1378,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!Elt) return nullptr; Elts.push_back(Elt); - if (isa(Elt)) // Already undef or poison. + if (isa(Elt)) // Already poison. UndefElts.setBit(i); } diff --git a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll index d2fa651b39449..1a6528d885568 100644 --- a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll +++ b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll @@ -92,10 +92,9 @@ define <3 x float> @twoShufUses(<3 x float> %x) { ; The inserted scalar constant index is out-of-bounds for the shuffle vector constant. -; FIXME: This is a miscompilation define <5 x i8> @longerMask(<3 x i8> %x) { ; CHECK-LABEL: @longerMask( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> , <5 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> , <5 x i32> ; CHECK-NEXT: [[INS:%.*]] = insertelement <5 x i8> [[SHUF]], i8 42, i64 4 ; CHECK-NEXT: ret <5 x i8> [[INS]] ; diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index e1174007b0fe0..978d90d7df94e 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -2333,11 +2333,10 @@ define <2 x float> @uitofp_shuf_narrow(<4 x i32> %x, <4 x i32> %y) { ret <2 x float> %r } -; FIXME: This is a miscompilation define <4 x i16> @blend_elements_from_load(ptr align 8 %_0) { ; CHECK-LABEL: @blend_elements_from_load( ; CHECK-NEXT: [[LOAD:%.*]] = load <3 x i16>, ptr [[_0:%.*]], align 8 -; CHECK-NEXT: [[RV:%.*]] = shufflevector <3 x i16> , <3 x i16> [[LOAD]], <4 x i32> +; CHECK-NEXT: [[RV:%.*]] = shufflevector <3 x i16> , <3 x i16> [[LOAD]], <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[RV]] ; %load = load <3 x i16>, ptr %_0, align 8 From c014454f43bf523fee2bf695c075882b1cefd21c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 17 Dec 2023 21:08:25 +0000 Subject: [PATCH 068/884] [ConstraintElim] Add extra tests with AND and OR of conditions. Add additional tests where one of the operands of the AND/OR implies the other. --- .../and-implied-by-operands.ll | 84 +++++ .../or-implied-by-operands.ll | 310 ++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 llvm/test/Transforms/ConstraintElimination/or-implied-by-operands.ll diff --git a/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll b/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll index 22f20f739b9e6..dc3b0f17c7960 100644 --- a/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll +++ b/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll @@ -26,6 +26,31 @@ else: ret i1 1 } +define i1 @test_first_and_condition_implied_by_second_ops(i8 %x) { +; CHECK-LABEL: @test_first_and_condition_implied_by_second_ops( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[AND:%.*]] = and i1 [[T_1]], [[C_1]] +; CHECK-NEXT: br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ugt i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %and = and i1 %t.1, %c.1 + br i1 %and, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + define i1 @test_second_and_condition_implied_by_first_select_form(i8 %x) { ; CHECK-LABEL: @test_second_and_condition_implied_by_first_select_form( ; CHECK-NEXT: entry: @@ -51,6 +76,31 @@ else: ret i1 1 } +define i1 @test_first_and_condition_implied_by_second_select_form(i8 %x) { +; CHECK-LABEL: @test_first_and_condition_implied_by_second_select_form( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[AND:%.*]] = select i1 [[T_1]], i1 [[C_1]], i1 false +; CHECK-NEXT: br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ugt i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %and = select i1 %t.1, i1 %c.1, i1 false + br i1 %and, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + define i1 @test_same_cond_for_and(i8 %x) { ; CHECK-LABEL: @test_same_cond_for_and( ; CHECK-NEXT: entry: @@ -394,3 +444,37 @@ then: else: ret i1 %t.1 } + +define i1 @and_select_first_implies_second_may_be_poison(ptr noundef %A, ptr noundef %B) { +; CHECK-LABEL: @and_select_first_implies_second_may_be_poison( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ne ptr [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds ptr, ptr [[B]], i64 -1 +; CHECK-NEXT: [[C_2:%.*]] = icmp ugt ptr [[GEP]], [[A]] +; CHECK-NEXT: [[AND:%.*]] = select i1 [[C_2]], i1 true, i1 false +; CHECK-NEXT: ret i1 [[AND]] +; +entry: + %c.1 = icmp ne ptr %A, %B + %gep = getelementptr inbounds ptr, ptr %B, i64 -1 + %c.2 = icmp ugt ptr %gep, %A + %and = select i1 %c.2, i1 %c.1, i1 false + ret i1 %and +} + +define i1 @and_select_second_implies_first_may_be_poison(ptr noundef %A, ptr noundef %B) { +; CHECK-LABEL: @and_select_second_implies_first_may_be_poison( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ne ptr [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds ptr, ptr [[B]], i64 -1 +; CHECK-NEXT: [[C_2:%.*]] = icmp ugt ptr [[GEP]], [[A]] +; CHECK-NEXT: [[AND:%.*]] = select i1 [[C_1]], i1 [[C_2]], i1 false +; CHECK-NEXT: ret i1 [[AND]] +; +entry: + %c.1 = icmp ne ptr %A, %B + %gep = getelementptr inbounds ptr, ptr %B, i64 -1 + %c.2 = icmp ugt ptr %gep, %A + %and = select i1 %c.1, i1 %c.2, i1 false + ret i1 %and +} diff --git a/llvm/test/Transforms/ConstraintElimination/or-implied-by-operands.ll b/llvm/test/Transforms/ConstraintElimination/or-implied-by-operands.ll new file mode 100644 index 0000000000000..61e6e250f6dd9 --- /dev/null +++ b/llvm/test/Transforms/ConstraintElimination/or-implied-by-operands.ll @@ -0,0 +1,310 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -passes=constraint-elimination -S %s | FileCheck %s + +define i1 @test_second_or_condition_implied_by_first(i8 %x) { +; CHECK-LABEL: @test_second_or_condition_implied_by_first( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ule i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_1]], [[T_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ule i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %or = or i1 %c.1, %t.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_first_or_condition_implied_by_second_ops(i8 %x) { +; CHECK-LABEL: @test_first_or_condition_implied_by_second_ops( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ule i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[T_1]], [[C_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ule i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %or = or i1 %t.1, %c.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_second_or_condition_implied_by_first_select_form(i8 %x) { +; CHECK-LABEL: @test_second_or_condition_implied_by_first_select_form( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ule i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = select i1 [[C_1]], i1 false, i1 [[T_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ule i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %or = select i1 %c.1, i1 false, i1 %t.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_first_or_condition_implied_by_second_select_form(i8 %x) { +; CHECK-LABEL: @test_first_or_condition_implied_by_second_select_form( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ule i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = select i1 [[T_1]], i1 false, i1 [[C_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ule i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %or = select i1 %t.1, i1 false, i1 %c.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_same_cond_for_or(i8 %x) { +; CHECK-LABEL: @test_same_cond_for_or( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_1]], [[C_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ugt i8 %x, 10 + %or = or i1 %c.1, %c.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_same_cond_for_or_select_form(i8 %x) { +; CHECK-LABEL: @test_same_cond_for_or_select_form( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[OR:%.*]] = select i1 [[C_1]], i1 false, i1 [[C_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ugt i8 %x, 10 + %or = select i1 %c.1, i1 false, i1 %c.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_second_or_condition_not_implied_by_first(i8 %x) { +; CHECK-LABEL: @test_second_or_condition_not_implied_by_first( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_2]], [[C_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 false +; CHECK: else: +; CHECK-NEXT: ret i1 true +; +entry: + %c.1 = icmp ugt i8 %x, 10 + %c.2 = icmp ugt i8 %x, 5 + %or = or i1 %c.2, %c.1 + br i1 %or, label %then, label %else + +then: + ret i1 0 + +else: + ret i1 1 +} + +define i1 @test_remove_variables(i1 %c, ptr %A, i64 %B, ptr %C) { +; CHECK-LABEL: @test_remove_variables( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C:%.*]], label [[THEN_1:%.*]], label [[EXIT:%.*]] +; CHECK: then.1: +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[C:%.*]], align 8 +; CHECK-NEXT: [[C_1:%.*]] = icmp ult ptr [[TMP0]], [[A:%.*]] +; CHECK-NEXT: br i1 [[C_1]], label [[THEN_2:%.*]], label [[ELSE_2:%.*]] +; CHECK: then.2: +; CHECK-NEXT: [[C_2:%.*]] = icmp ne ptr [[A]], null +; CHECK-NEXT: [[C_3:%.*]] = icmp sgt i64 [[B:%.*]], 0 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_2]], [[C_3]] +; CHECK-NEXT: ret i1 [[OR]] +; CHECK: else.2: +; CHECK-NEXT: ret i1 false +; CHECK: exit: +; CHECK-NEXT: ret i1 true +; +entry: + br i1 %c, label %then.1, label %exit + +then.1: + %0 = load ptr, ptr %C, align 8 + %c.1 = icmp ult ptr %0, %A + br i1 %c.1, label %then.2, label %else.2 + +then.2: + %c.2 = icmp ne ptr %A, null + %c.3 = icmp sgt i64 %B, 0 + %or = or i1 %c.2, %c.3 + ret i1 %or + +else.2: + ret i1 0 + +exit: + %t = icmp eq ptr null, null + ret i1 %t +} + +define i1 @test_or_op_0_simplified(i32 %v) { +; CHECK-LABEL: @test_or_op_0_simplified( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i32 [[V:%.*]], 0 +; CHECK-NEXT: [[OR:%.*]] = or i1 false, [[C_1]] +; CHECK-NEXT: ret i1 [[OR]] +; +entry: + %c.1 = icmp sgt i32 %v, 0 + %t.1 = icmp sgt i32 0, 0 + %or = or i1 %t.1, %c.1 + ret i1 %or +} + +define i1 @test_or_op_1_simplified(i32 %v) { +; CHECK-LABEL: @test_or_op_1_simplified( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i32 [[V:%.*]], 0 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_1]], false +; CHECK-NEXT: ret i1 [[OR]] +; +entry: + %c.1 = icmp sgt i32 %v, 0 + %t.1 = icmp sgt i32 0, 0 + %or = or i1 %c.1, %t.1 + ret i1 %or +} + +define i1 @test_or_used_in_false_branch(i8 %x) { +; CHECK-LABEL: @test_or_used_in_false_branch( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ule i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ule i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_1]], [[T_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 [[T_1]] +; CHECK: else: +; CHECK-NEXT: ret i1 false +; + +entry: + %c.1 = icmp ule i8 %x, 10 + %t.1 = icmp ule i8 %x, 5 + %or = or i1 %c.1, %t.1 + br i1 %or, label %then, label %else + +then: + ret i1 %t.1 + +else: + ret i1 %t.1 +} + +define i1 @test_or_used_in_false_branch2(i8 %x) { +; CHECK-LABEL: @test_or_used_in_false_branch2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i8 [[X:%.*]], 10 +; CHECK-NEXT: [[T_1:%.*]] = icmp ugt i8 [[X]], 5 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[C_1]], [[T_1]] +; CHECK-NEXT: br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i1 [[T_1]] +; CHECK: else: +; CHECK-NEXT: ret i1 false +; + +entry: + %c.1 = icmp ugt i8 %x, 10 + %t.1 = icmp ugt i8 %x, 5 + %or = or i1 %c.1, %t.1 + br i1 %or, label %then, label %else + +then: + ret i1 %t.1 + +else: + ret i1 %t.1 +} + +define i1 @select_or_set_operand(ptr noundef %a, ptr noundef %b) { +; CHECK-LABEL: @select_or_set_operand( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP_EQ:%.*]] = icmp eq ptr [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[INCDEC_PTR12_I:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1 +; CHECK-NEXT: [[CMP_EQ_1:%.*]] = icmp eq ptr [[INCDEC_PTR12_I]], [[B]] +; CHECK-NEXT: [[OR:%.*]] = select i1 [[CMP_EQ]], i1 true, i1 [[CMP_EQ_1]] +; CHECK-NEXT: ret i1 [[OR]] +; +entry: + %cmp.eq = icmp eq ptr %a, %b + %incdec.ptr12.i = getelementptr inbounds i32, ptr %a, i64 1 + %cmp.eq.1 = icmp eq ptr %incdec.ptr12.i, %b + %or = select i1 %cmp.eq, i1 true, i1 %cmp.eq.1 + ret i1 %or +} From aad5c2f887d3cd1c69f798186e6502f0ed6e3dde Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Sun, 17 Dec 2023 22:43:34 +0100 Subject: [PATCH 069/884] [cmake] Honor CMAKE_VERBOSE_MAKEFILE when building external projects (#75749) When the top-level CMake invocation has `CMAKE_VERBOSE_MAKEFILE=ON`, indicating the user wants to have verbose builds (i.e. all executed commands explicitly echoed), some of the subprojects and runtimes (such as compiler-rt, libcxx, etc) do not build in verbose mode. For example, with Ninja: ``` [ 99% 6252/6308] cd /build/runtimes/builtins-bins && /usr/local/bin/cmake --build . [ 0% 6/308] Building C object CMakeFiles/clang_rt.builtins-i386.dir/absvti2.c.o [ 0% 7/308] Building C object CMakeFiles/clang_rt.builtins-i386.dir/absvdi2.c.o [ 0% 8/308] Building C object CMakeFiles/clang_rt.builtins-i386.dir/absvsi2.c.o ... ``` This is because `llvm_ExternalProject_Add()` and `add_custom_libcxx()` use CMake's `ExternalProject_Add()` function to configure such subproject builds, and do not pass through the `CMAKE_VERBOSE_MAKEFILE` setting. Similar to what is done in `clang/CMakeLists.txt`, add `-DCMAKE_VERBOSE_MAKEFILE=ON` to the `ExternalProject_Add()` invocations in `llvm_ExternalProject_Add()` and `add_custom_libcxx()`, whenever the top-level CMake invocation had `CMAKE_VERBOSE_MAKEFILE` turned on. --- compiler-rt/cmake/Modules/AddCompilerRT.cmake | 5 +++++ llvm/cmake/modules/LLVMExternalProjectUtils.cmake | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake index 4d9b68a3cc25b..7aca0abc637d4 100644 --- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake +++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake @@ -670,6 +670,10 @@ macro(add_custom_libcxx name prefix) get_property(CXX_FLAGS CACHE CMAKE_CXX_FLAGS PROPERTY VALUE) set(LIBCXX_CXX_FLAGS "${LIBCXX_CXX_FLAGS} ${CXX_FLAGS}") + if(CMAKE_VERBOSE_MAKEFILE) + set(verbose -DCMAKE_VERBOSE_MAKEFILE=ON) + endif() + ExternalProject_Add(${name} DEPENDS ${name}-clobber ${LIBCXX_DEPS} PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} @@ -677,6 +681,7 @@ macro(add_custom_libcxx name prefix) BINARY_DIR ${prefix} CMAKE_ARGS ${CMAKE_PASSTHROUGH_VARIABLES} ${compiler_args} + ${verbose} -DCMAKE_C_FLAGS=${LIBCXX_C_FLAGS} -DCMAKE_CXX_FLAGS=${LIBCXX_CXX_FLAGS} -DCMAKE_BUILD_TYPE=Release diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake index 4b5b600307ec9..2089f979acd00 100644 --- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake +++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake @@ -319,6 +319,10 @@ function(llvm_ExternalProject_Add name source_dir) list(APPEND compiler_args -DCMAKE_ASM_COMPILER_TARGET=${ARG_TARGET_TRIPLE}) endif() + if(CMAKE_VERBOSE_MAKEFILE) + set(verbose -DCMAKE_VERBOSE_MAKEFILE=ON) + endif() + ExternalProject_Add(${name} DEPENDS ${ARG_DEPENDS} llvm-config ${name}-clobber @@ -330,6 +334,7 @@ function(llvm_ExternalProject_Add name source_dir) CMAKE_ARGS ${${nameCanon}_CMAKE_ARGS} --no-warn-unused-cli ${compiler_args} + ${verbose} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} ${sysroot_arg} -DLLVM_BINARY_DIR=${PROJECT_BINARY_DIR} From 68c976bf64f50fe9c16a335378a964c166851962 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Fri, 15 Dec 2023 13:01:55 -0800 Subject: [PATCH 070/884] [X86] Fix referencing local tagged globals We should treat the medium code model like the small code model. Classifying non-local references already properly handled this. --- llvm/lib/Target/X86/X86Subtarget.cpp | 9 ++-- llvm/test/CodeGen/X86/tagged-globals-pic.ll | 46 ++++++++++++++++++- .../test/CodeGen/X86/tagged-globals-static.ll | 26 ++++++++++- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp index d63f1ca1695b2..07f535685e8f9 100644 --- a/llvm/lib/Target/X86/X86Subtarget.cpp +++ b/llvm/lib/Target/X86/X86Subtarget.cpp @@ -69,11 +69,11 @@ X86Subtarget::classifyGlobalReference(const GlobalValue *GV) const { unsigned char X86Subtarget::classifyLocalReference(const GlobalValue *GV) const { + CodeModel::Model CM = TM.getCodeModel(); // Tagged globals have non-zero upper bits, which makes direct references - // require a 64-bit immediate. On the small code model this causes relocation - // errors, so we go through the GOT instead. - if (AllowTaggedGlobals && TM.getCodeModel() == CodeModel::Small && GV && - !isa(GV)) + // require a 64-bit immediate. With the small/medium code models this causes + // relocation errors, so we go through the GOT instead. + if (AllowTaggedGlobals && CM != CodeModel::Large && GV && !isa(GV)) return X86II::MO_GOTPCREL_NORELAX; // If we're not PIC, it's not very interesting. @@ -83,7 +83,6 @@ X86Subtarget::classifyLocalReference(const GlobalValue *GV) const { if (is64Bit()) { // 64-bit ELF PIC local references may use GOTOFF relocations. if (isTargetELF()) { - CodeModel::Model CM = TM.getCodeModel(); assert(CM != CodeModel::Tiny && "Tiny codesize model not supported on X86"); // In the large code model, all text is far from any global data, so we diff --git a/llvm/test/CodeGen/X86/tagged-globals-pic.ll b/llvm/test/CodeGen/X86/tagged-globals-pic.ll index 4f85b5ed99695..156487ee163a2 100644 --- a/llvm/test/CodeGen/X86/tagged-globals-pic.ll +++ b/llvm/test/CodeGen/X86/tagged-globals-pic.ll @@ -1,5 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc --relocation-model=pic < %s | FileCheck %s +; RUN: llc --relocation-model=pic -code-model=small < %s | FileCheck %s +; RUN: llc --relocation-model=pic -code-model=medium < %s | FileCheck %s +; RUN: llc --relocation-model=pic -code-model=large < %s | FileCheck %s --check-prefix=LARGE target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -12,6 +14,16 @@ define ptr @global_addr() #0 { ; CHECK: # %bb.0: ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_addr: +; LARGE: # %bb.0: +; LARGE-NEXT: .L0$pb: +; LARGE-NEXT: leaq .L0$pb(%rip), %rax +; LARGE-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx +; LARGE-NEXT: addq %rax, %rcx +; LARGE-NEXT: movabsq $global@GOT, %rax +; LARGE-NEXT: movq (%rcx,%rax), %rax +; LARGE-NEXT: retq ret ptr @global } @@ -21,6 +33,17 @@ define i32 @global_load() #0 { ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: movl (%rax), %eax ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_load: +; LARGE: # %bb.0: +; LARGE-NEXT: .L1$pb: +; LARGE-NEXT: leaq .L1$pb(%rip), %rax +; LARGE-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx +; LARGE-NEXT: addq %rax, %rcx +; LARGE-NEXT: movabsq $global@GOT, %rax +; LARGE-NEXT: movq (%rcx,%rax), %rax +; LARGE-NEXT: movl (%rax), %eax +; LARGE-NEXT: retq %load = load i32, ptr @global ret i32 %load } @@ -31,6 +54,17 @@ define void @global_store() #0 { ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: movl $0, (%rax) ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_store: +; LARGE: # %bb.0: +; LARGE-NEXT: .L2$pb: +; LARGE-NEXT: leaq .L2$pb(%rip), %rax +; LARGE-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx +; LARGE-NEXT: addq %rax, %rcx +; LARGE-NEXT: movabsq $global@GOT, %rax +; LARGE-NEXT: movq (%rcx,%rax), %rax +; LARGE-NEXT: movl $0, (%rax) +; LARGE-NEXT: retq store i32 0, ptr @global ret void } @@ -40,6 +74,16 @@ define ptr @func_addr() #0 { ; CHECK: # %bb.0: ; CHECK-NEXT: movq func@GOTPCREL(%rip), %rax ; CHECK-NEXT: retq +; +; LARGE-LABEL: func_addr: +; LARGE: # %bb.0: +; LARGE-NEXT: .L3$pb: +; LARGE-NEXT: leaq .L3$pb(%rip), %rax +; LARGE-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx +; LARGE-NEXT: addq %rax, %rcx +; LARGE-NEXT: movabsq $func@GOT, %rax +; LARGE-NEXT: movq (%rcx,%rax), %rax +; LARGE-NEXT: retq ret ptr @func } diff --git a/llvm/test/CodeGen/X86/tagged-globals-static.ll b/llvm/test/CodeGen/X86/tagged-globals-static.ll index bddbaa5592da5..0eb21267b06e0 100644 --- a/llvm/test/CodeGen/X86/tagged-globals-static.ll +++ b/llvm/test/CodeGen/X86/tagged-globals-static.ll @@ -1,5 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc --relocation-model=static < %s | FileCheck %s +; RUN: llc --relocation-model=static -code-model=small < %s | FileCheck %s +; RUN: llc --relocation-model=static -code-model=medium < %s | FileCheck %s +; RUN: llc --relocation-model=static -code-model=large < %s | FileCheck %s --check-prefix=LARGE target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -12,6 +14,11 @@ define ptr @global_addr() #0 { ; CHECK: # %bb.0: ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_addr: +; LARGE: # %bb.0: +; LARGE-NEXT: movabsq $global, %rax +; LARGE-NEXT: retq ret ptr @global } @@ -21,6 +28,12 @@ define i32 @global_load() #0 { ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: movl (%rax), %eax ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_load: +; LARGE: # %bb.0: +; LARGE-NEXT: movabsq $global, %rax +; LARGE-NEXT: movl (%rax), %eax +; LARGE-NEXT: retq %load = load i32, ptr @global ret i32 %load } @@ -31,6 +44,12 @@ define void @global_store() #0 { ; CHECK-NEXT: movq global@GOTPCREL_NORELAX(%rip), %rax ; CHECK-NEXT: movl $0, (%rax) ; CHECK-NEXT: retq +; +; LARGE-LABEL: global_store: +; LARGE: # %bb.0: +; LARGE-NEXT: movabsq $global, %rax +; LARGE-NEXT: movl $0, (%rax) +; LARGE-NEXT: retq store i32 0, ptr @global ret void } @@ -40,6 +59,11 @@ define ptr @func_addr() #0 { ; CHECK: # %bb.0: ; CHECK-NEXT: movl $func, %eax ; CHECK-NEXT: retq +; +; LARGE-LABEL: func_addr: +; LARGE: # %bb.0: +; LARGE-NEXT: movabsq $func, %rax +; LARGE-NEXT: retq ret ptr @func } From 401f0396c3070567ce1ad0b12be7e48713ec0c65 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sun, 17 Dec 2023 15:07:11 -0800 Subject: [PATCH 071/884] [clang-format] Fix a bug in `IndentExternBlock: NoIndent` (#75731) Fixes #36620. Fixes #75719. --- clang/lib/Format/Format.cpp | 5 ----- clang/unittests/Format/FormatTest.cpp | 7 +++++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 668e959a9416b..28271181e07d0 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -1315,7 +1315,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterUnion = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.SplitEmptyFunction = true; Expanded.BraceWrapping.SplitEmptyRecord = false; break; @@ -1335,7 +1334,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterUnion = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.BeforeCatch = true; Expanded.BraceWrapping.BeforeElse = true; Expanded.BraceWrapping.BeforeLambdaBody = true; @@ -1350,7 +1348,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { Expanded.BraceWrapping.AfterObjCDeclaration = true; Expanded.BraceWrapping.AfterStruct = true; Expanded.BraceWrapping.AfterExternBlock = true; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Expanded.BraceWrapping.BeforeCatch = true; Expanded.BraceWrapping.BeforeElse = true; Expanded.BraceWrapping.BeforeLambdaBody = true; @@ -1375,7 +1372,6 @@ static void expandPresetsBraceWrapping(FormatStyle &Expanded) { /*SplitEmptyFunction=*/true, /*SplitEmptyRecord=*/true, /*SplitEmptyNamespace=*/true}; - Expanded.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; break; case FormatStyle::BS_WebKit: Expanded.BraceWrapping.AfterFunction = true; @@ -1909,7 +1905,6 @@ FormatStyle getMicrosoftStyle(FormatStyle::LanguageKind Language) { Style.BraceWrapping.AfterObjCDeclaration = true; Style.BraceWrapping.AfterStruct = true; Style.BraceWrapping.AfterExternBlock = true; - Style.IndentExternBlock = FormatStyle::IEBS_AfterExternBlock; Style.BraceWrapping.BeforeCatch = true; Style.BraceWrapping.BeforeElse = true; Style.BraceWrapping.BeforeWhile = false; diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 24b2fd599dc39..0e08723aa9e94 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -4571,6 +4571,13 @@ TEST_F(FormatTest, IndentExternBlockStyle) { "}", Style); + Style.BreakBeforeBraces = FormatStyle::BS_Allman; + verifyFormat("extern \"C\"\n" + "{\n" + "int i;\n" + "}", + Style); + Style.BreakBeforeBraces = FormatStyle::BS_Custom; Style.BraceWrapping.AfterExternBlock = true; Style.IndentExternBlock = FormatStyle::IEBS_Indent; From f1ab90ab632d137fc3c7deaa237751db31bcb22e Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 15:36:42 -0800 Subject: [PATCH 072/884] [IR] Use llvm::find (NFC) --- llvm/lib/IR/Metadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index 7bc25e30b8932..515893d079b8c 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -1566,7 +1566,7 @@ void Instruction::updateDIAssignIDMapping(DIAssignID *ID) { "Expect existing attachment to be mapped"); auto &InstVec = InstrsIt->second; - auto *InstIt = std::find(InstVec.begin(), InstVec.end(), this); + auto *InstIt = llvm::find(InstVec, this); assert(InstIt != InstVec.end() && "Expect instruction to be mapped to attachment"); // The vector contains a ptr to this. If this is the only element in the From 211f5d00e26b62edc80bc86655a73c28e57b6964 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 15:36:44 -0800 Subject: [PATCH 073/884] [llvm] Fix typos in documentation --- llvm/docs/AliasAnalysis.rst | 2 +- llvm/docs/ConvergentOperations.rst | 2 +- llvm/docs/JITLink.rst | 2 +- llvm/docs/LangRef.rst | 6 +++--- llvm/docs/NVPTXUsage.rst | 2 +- llvm/docs/TableGen/ProgRef.rst | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/docs/AliasAnalysis.rst b/llvm/docs/AliasAnalysis.rst index 046dd24d7332e..7afe0e277bd4f 100644 --- a/llvm/docs/AliasAnalysis.rst +++ b/llvm/docs/AliasAnalysis.rst @@ -207,7 +207,7 @@ Writing a new ``AliasAnalysis`` Implementation Writing a new alias analysis implementation for LLVM is quite straight-forward. There are already several implementations that you can use for examples, and the -following information should help fill in any details. For a examples, take a +following information should help fill in any details. For examples, take a look at the `various alias analysis implementations`_ included with LLVM. Different Pass styles diff --git a/llvm/docs/ConvergentOperations.rst b/llvm/docs/ConvergentOperations.rst index 5dd3ac2f3d98b..332675f3edefd 100644 --- a/llvm/docs/ConvergentOperations.rst +++ b/llvm/docs/ConvergentOperations.rst @@ -607,7 +607,7 @@ those in the caller. only if both threads entered the function by executing converged dynamic instances of the call-site. -This intrinsic can occur at most once in a function, and only in the the entry +This intrinsic can occur at most once in a function, and only in the entry block of the function. If this intrinsic occurs in a basic block, then it must precede any other convergent operation in the same basic block. diff --git a/llvm/docs/JITLink.rst b/llvm/docs/JITLink.rst index 72607a8c085ad..b0a0dc77880df 100644 --- a/llvm/docs/JITLink.rst +++ b/llvm/docs/JITLink.rst @@ -466,7 +466,7 @@ finally transferring linked memory to the executing process. Calls the ``JITLinkContext``'s ``JITLinkMemoryManager`` to allocate both working and target memory for the graph. As part of this process the - ``JITLinkMemoryManager`` will update the the addresses of all nodes + ``JITLinkMemoryManager`` will update the addresses of all nodes defined in the graph to their assigned target address. Note: This step only updates the addresses of nodes defined in this graph. diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 8f0c45f674ead..7f4a316a21ace 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -1515,7 +1515,7 @@ Currently, only the following parameter attributes are defined: over-alignment specification through language attributes). ``allocalign`` - The function parameter marked with this attribute is is the alignment in bytes of the + The function parameter marked with this attribute is the alignment in bytes of the newly allocated block returned by this function. The returned value must either have the specified alignment or be the null pointer. The return value MAY be more aligned than the requested alignment, but not less aligned. Invalid (e.g. non-power-of-2) @@ -22798,7 +22798,7 @@ Semantics: The '``llvm.vp.fcmp``' compares its first two operands according to the condition code given as the third operand. The operands are compared element by -element on each enabled lane, where the the semantics of the comparison are +element on each enabled lane, where the semantics of the comparison are defined :ref:`according to the condition code `. Masked-off lanes are ``poison``. @@ -22856,7 +22856,7 @@ Semantics: The '``llvm.vp.icmp``' compares its first two operands according to the condition code given as the third operand. The operands are compared element by -element on each enabled lane, where the the semantics of the comparison are +element on each enabled lane, where the semantics of the comparison are defined :ref:`according to the condition code `. Masked-off lanes are ``poison``. diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst index 5c28a3f3eee90..22acc6c9cb37f 100644 --- a/llvm/docs/NVPTXUsage.rst +++ b/llvm/docs/NVPTXUsage.rst @@ -329,7 +329,7 @@ optimization pipeline before dead-code elimination. The NVPTX TargetMachine knows how to schedule ``NVVMReflect`` at the beginning of your pass manager; just use the following code when setting up your pass manager and the PassBuilder will use ``registerPassBuilderCallbacks`` to let -NVPTXTargetMachine::registerPassBuilderCallbacks add the the pass to the +NVPTXTargetMachine::registerPassBuilderCallbacks add the pass to the pass manager: .. code-block:: c++ diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst index e5420a05dad78..59ddef975c487 100644 --- a/llvm/docs/TableGen/ProgRef.rst +++ b/llvm/docs/TableGen/ProgRef.rst @@ -661,7 +661,7 @@ The argument values can be specified in two forms: argument with name ``a`` and ``a1`` will be assigned to the argument with name ``b``. -Required arguments can alse be specified as named argument. +Required arguments can also be specified as named argument. Note that the argument can only be specified once regardless of the way (named or positional) to specify and positional arguments should be put before named From 364d7e775fcad5ef20a5c5788586f79c467b47db Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 15:51:48 -0800 Subject: [PATCH 074/884] [lldb] Use StringRef::starts_with (NFC) This patch replaces uses of StringRef::startswith with StringRef::starts_with for consistency with std::{string,string_view}::starts_with in C++20. I'm planning to deprecate and eventually remove StringRef::{starts,ends}with. --- lldb/bindings/python/python-typemaps.swig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/bindings/python/python-typemaps.swig b/lldb/bindings/python/python-typemaps.swig index 7660e0282c8fc..8d4b740e5f35c 100644 --- a/lldb/bindings/python/python-typemaps.swig +++ b/lldb/bindings/python/python-typemaps.swig @@ -110,7 +110,7 @@ AND call SWIG_fail at the same time, because it will result in a double free. SWIG_fail; } - if (llvm::StringRef(type_name.get()).startswith("SB")) { + if (llvm::StringRef(type_name.get()).starts_with("SB")) { std::string error_msg = "Input type is invalid: " + type_name.get(); PyErr_SetString(PyExc_TypeError, error_msg.c_str()); SWIG_fail; From 5ac12951b4e9bbfcc5791282d0961ec2b65575e9 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 17 Dec 2023 15:52:50 -0800 Subject: [PATCH 075/884] [ADT] Deprecate StringRef::{starts,ends}with (#75491) This patch deprecates StringRef::{starts,ends}with. Note that I've replaced all known uses of StringRef::{starts,ends}with with StringRef::{starts,ends}_with for consistency with std::{string,string_view}::{starts,ends}_with in C++20. --- llvm/include/llvm/ADT/StringRef.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 4e69d5b633546..d892333de391c 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -258,7 +258,9 @@ namespace llvm { return Length >= Prefix.Length && compareMemory(Data, Prefix.Data, Prefix.Length) == 0; } - [[nodiscard]] bool startswith(StringRef Prefix) const { + [[nodiscard]] LLVM_DEPRECATED( + "Use starts_with instead", + "starts_with") bool startswith(StringRef Prefix) const { return starts_with(Prefix); } @@ -271,7 +273,9 @@ namespace llvm { compareMemory(end() - Suffix.Length, Suffix.Data, Suffix.Length) == 0; } - [[nodiscard]] bool endswith(StringRef Suffix) const { + [[nodiscard]] LLVM_DEPRECATED( + "Use ends_with instead", + "ends_with") bool endswith(StringRef Suffix) const { return ends_with(Suffix); } From dbe9a602561d5eecfc1652aab7e127754cb963c0 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sun, 17 Dec 2023 17:39:49 -0800 Subject: [PATCH 076/884] [RISCV] Correct the VLOperand for masked vssrl/vssra intrinsics. Though I can't prove it matters for anything. The only use of VLOperand I know of is for handling i64 splat operands to .vx intrinsics on RV32. Shifts are special and always use XLen for .vx so they are always legal. --- llvm/include/llvm/IR/IntrinsicsRISCV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index fc830fca392fc..a391bc53cdb0e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -702,7 +702,7 @@ let TargetPrefix = "riscv" in { LLVMMatchType<2>, LLVMMatchType<2>], [ImmArg>,ImmArg>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let VLOperand = 6; + let VLOperand = 5; } // For Saturating binary operations. // The destination vector type is NOT the same as first source vector. From dd45be028d2788cc401400e208ab0fa64d929b0a Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Sun, 17 Dec 2023 20:59:46 -0500 Subject: [PATCH 077/884] [mlir][gpu] Trim trailing whitespace in dialect docs. NFC. --- mlir/include/mlir/Dialect/GPU/IR/GPUBase.td | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUBase.td b/mlir/include/mlir/Dialect/GPU/IR/GPUBase.td index ccb9580adbd1f..7b9d46fda12f5 100644 --- a/mlir/include/mlir/Dialect/GPU/IR/GPUBase.td +++ b/mlir/include/mlir/Dialect/GPU/IR/GPUBase.td @@ -52,14 +52,14 @@ def GPU_Dialect : Dialect { /// Returns the numeric value used to identify the private memory address /// space. static AddressSpace getPrivateAddressSpace() { return AddressSpace::Private; } - - /// Return true if the given MemRefType has an address space that matches + + /// Return true if the given MemRefType has an address space that matches /// with the gpu::AddressSpaceAttr attribute with value 'workgroup`. static bool hasWorkgroupMemoryAddressSpace(MemRefType type); - /// Return true if the given Attribute is an gpu::AddressSpaceAttr + /// Return true if the given Attribute is an gpu::AddressSpaceAttr /// attribute with value 'workgroup`. - static bool isWorkgroupMemoryAddressSpace(Attribute memorySpace); + static bool isWorkgroupMemoryAddressSpace(Attribute memorySpace); }]; let dependentDialects = ["arith::ArithDialect"]; From 31429e7a89590f88034920edd3e997aeabff8124 Mon Sep 17 00:00:00 2001 From: Akira Hatanaka Date: Sun, 17 Dec 2023 18:22:44 -0800 Subject: [PATCH 078/884] [CodeGen] Emit a more accurate alignment for non-temporal loads/stores (#75675) Call EmitPointerWithAlignment to compute the alignment based on the underlying lvalue's alignment when it's available. --- clang/lib/CodeGen/CGBuiltin.cpp | 8 ++++---- clang/test/CodeGen/Nontemporal.cpp | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 3327866d2b962..c96f86a823a46 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -232,19 +232,19 @@ static Value *MakeBinaryAtomicValue( static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { Value *Val = CGF.EmitScalarExpr(E->getArg(0)); - Value *Address = CGF.EmitScalarExpr(E->getArg(1)); + Address Addr = CGF.EmitPointerWithAlignment(E->getArg(1)); Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); - LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getArg(0)->getType()); + LValue LV = CGF.MakeAddrLValue(Addr, E->getArg(0)->getType()); LV.setNontemporal(true); CGF.EmitStoreOfScalar(Val, LV, false); return nullptr; } static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { - Value *Address = CGF.EmitScalarExpr(E->getArg(0)); + Address Addr = CGF.EmitPointerWithAlignment(E->getArg(0)); - LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); + LValue LV = CGF.MakeAddrLValue(Addr, E->getType()); LV.setNontemporal(true); return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); } diff --git a/clang/test/CodeGen/Nontemporal.cpp b/clang/test/CodeGen/Nontemporal.cpp index e14ca18717928..5052cb225d411 100644 --- a/clang/test/CodeGen/Nontemporal.cpp +++ b/clang/test/CodeGen/Nontemporal.cpp @@ -46,3 +46,17 @@ void test_all_sizes(void) // CHECK-LABEL: test_all_sizes vf2 = __builtin_nontemporal_load(&vf1); // CHECK: load <4 x float>{{.*}}align 16, !nontemporal vc2 = __builtin_nontemporal_load(&vc1); // CHECK: load <8 x i8>{{.*}}align 8, !nontemporal } + +struct S { char c[16]; }; +S x; + +typedef int v4si __attribute__ ((vector_size(16))); + +// CHECK-LABEL: define void @_Z14test_alignmentv() +// CHECK: load <4 x i32>, ptr @x, align 1, !nontemporal +// CHECK: store <4 x i32> %1, ptr @x, align 1, !nontemporal + +void test_alignment() { + auto t = __builtin_nontemporal_load((v4si*)x.c); + __builtin_nontemporal_store(t, (v4si*)x.c); +} From 2c668fddadd885384381107be42f936f08ec0c4d Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Sun, 17 Dec 2023 21:34:25 -0500 Subject: [PATCH 079/884] [mlir][gpu] Trim trailing whitespace in GPUOps.td. NFC. --- mlir/include/mlir/Dialect/GPU/IR/GPUOps.td | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td index 7cad1cd89fd63..2e1a5f5cc78ae 100644 --- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td +++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td @@ -68,7 +68,7 @@ def GPU_ClusterDimOp : GPU_IndexOp<"cluster_dim"> { def GPU_ClusterIdOp : GPU_IndexOp<"cluster_id"> { let description = [{ - Returns the cluster id, i.e. the index of the current cluster within the + Returns the cluster id, i.e. the index of the current cluster within the grid along the x, y, or z `dimension`. Example: @@ -462,23 +462,23 @@ def GPU_GPUFuncOp : GPU_Op<"func", [ def GPU_DynamicSharedMemoryOp : GPU_Op<"dynamic_shared_memory", [Pure]> { let summary = "Get the memref for dynamic shared memory"; - + let description = [{ - This operation provides a memref pointer to the start of dynamic shared + This operation provides a memref pointer to the start of dynamic shared memory, often referred to as workgroup memory. It's important to note that - this dynamic shared memory needs to be allocated at kernel launch. One can - conveniently utilize `the dynamic_shared_memory_size` parameter of + this dynamic shared memory needs to be allocated at kernel launch. One can + conveniently utilize `the dynamic_shared_memory_size` parameter of `gpu.launch` for this purpose. - - Examples: - ```mlir + + Examples: + ```mlir %0 = gpu.dynamic.shared.memory : memref> - %1 = memref.view %0[%c8192][] : memref> + %1 = memref.view %0[%c8192][] : memref> to memref<32x64xf32, #gpu.address_space> - %2 = memref.view %0[%c16384][] : memref> + %2 = memref.view %0[%c16384][] : memref> to memref<32x64xf32, #gpu.address_space> ``` - }]; + }]; let arguments = (ins); let results = (outs Arg>:$resultMemref); let assemblyFormat = [{ attr-dict `:` type($resultMemref) }]; @@ -493,11 +493,11 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [ "blockSizeY", "blockSizeZ"]>]>, Arguments<(ins Variadic:$asyncDependencies, SymbolRefAttr:$kernel, - LaunchIndx:$gridSizeX, - LaunchIndx:$gridSizeY, + LaunchIndx:$gridSizeX, + LaunchIndx:$gridSizeY, LaunchIndx:$gridSizeZ, - LaunchIndx:$blockSizeX, - LaunchIndx:$blockSizeY, + LaunchIndx:$blockSizeX, + LaunchIndx:$blockSizeY, LaunchIndx:$blockSizeZ, Optional:$clusterSizeX, Optional:$clusterSizeY, @@ -539,10 +539,10 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [ The remaining operands if present are passed as arguments to the kernel function. - The `gpu.launch_func` also supports kernel launching with clusters if - supported by the target architecture. The cluster size can be set by - `clusterSizeX`, `clusterSizeY`, and `clusterSizeZ` arguments. When these - arguments are present, the Op launches a kernel that clusters the given + The `gpu.launch_func` also supports kernel launching with clusters if + supported by the target architecture. The cluster size can be set by + `clusterSizeX`, `clusterSizeY`, and `clusterSizeZ` arguments. When these + arguments are present, the Op launches a kernel that clusters the given thread blocks. This feature is exclusive to certain architectures. Example: @@ -593,7 +593,7 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [ async // (Optional) Don't block host, return token. [%t0] // (Optional) Execute only after %t0 has completed. @kernels::@kernel_1 // Kernel function. - clusters in (%cst, %cst, %cst) // (Optional) Cluster size only for support architectures. + clusters in (%cst, %cst, %cst) // (Optional) Cluster size only for support architectures. blocks in (%cst, %cst, %cst) // Grid size. threads in (%cst, %cst, %cst) // Block size. dynamic_shared_memory_size %s // (Optional) Amount of dynamic shared @@ -659,7 +659,7 @@ def GPU_LaunchFuncOp :GPU_Op<"launch_func", [ let assemblyFormat = [{ custom(type($asyncToken), $asyncDependencies) (`<` $asyncObject^ `:` type($asyncObject) `>`)? - $kernel + $kernel ( `clusters` `in` ` ` `(` $clusterSizeX^ `,` $clusterSizeY `,` $clusterSizeZ `)` )? `blocks` `in` ` ` `(` $gridSizeX `,` $gridSizeY `,` $gridSizeZ `)` `threads` `in` ` ` `(` $blockSizeX `,` $blockSizeY `,` $blockSizeZ `)` From 5c1f44193dd6a7d3453fc002130f5cbc7cb351c2 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sun, 17 Dec 2023 19:21:36 -0800 Subject: [PATCH 080/884] [RISCV] Simplify PrintExtension. NFC (#75427) Instead of using a format string that needs to be parsed, we can use left_justify to print each string with padding. --- llvm/lib/Support/RISCVISAInfo.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index bbbaf26a7bd49..4a800ceb0c810 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -215,11 +215,12 @@ static void verifyTables() { #endif } -static void PrintExtension(const std::string Name, const std::string Version, - const std::string Description) { - outs() << " " - << format(Description.empty() ? "%-20s%s\n" : "%-20s%-10s%s\n", - Name.c_str(), Version.c_str(), Description.c_str()); +static void PrintExtension(StringRef Name, StringRef Version, + StringRef Description) { + outs().indent(4); + unsigned VersionWidth = Description.empty() ? 0 : 10; + outs() << left_justify(Name, 20) << left_justify(Version, VersionWidth) + << Description << "\n"; } void llvm::riscvExtensionsHelp(StringMap DescMap) { @@ -233,7 +234,7 @@ void llvm::riscvExtensionsHelp(StringMap DescMap) { for (const auto &E : ExtMap) { std::string Version = std::to_string(E.second.MajorVersion) + "." + std::to_string(E.second.MinorVersion); - PrintExtension(E.first, Version, DescMap[E.first].str()); + PrintExtension(E.first, Version, DescMap[E.first]); } outs() << "\nExperimental extensions\n"; @@ -243,7 +244,7 @@ void llvm::riscvExtensionsHelp(StringMap DescMap) { for (const auto &E : ExtMap) { std::string Version = std::to_string(E.second.MajorVersion) + "." + std::to_string(E.second.MinorVersion); - PrintExtension(E.first, Version, DescMap["experimental-" + E.first].str()); + PrintExtension(E.first, Version, DescMap["experimental-" + E.first]); } outs() << "\nUse -march to specify the target's extension.\n" From b83b28779ee56236aaf8827398f889334abbd28d Mon Sep 17 00:00:00 2001 From: Yeting Kuo <46629943+yetingk@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:46:22 +0800 Subject: [PATCH 081/884] [RISCV] Make Zhinx and Zvfh imply Zhinxmin and Zvfhmin respectively (#75735) Zhinxmin is a subset of Zhinx and Zvfhmin is also a subset of Zvfh. --- llvm/lib/Support/RISCVISAInfo.cpp | 4 ++-- llvm/test/CodeGen/RISCV/attributes.ll | 4 ++++ llvm/test/MC/RISCV/attribute-arch.s | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 4a800ceb0c810..54363e988b702 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -1014,7 +1014,7 @@ static const char *ImpliedExtsZfbfmin[] = {"f"}; static const char *ImpliedExtsZfh[] = {"zfhmin"}; static const char *ImpliedExtsZfhmin[] = {"f"}; static const char *ImpliedExtsZfinx[] = {"zicsr"}; -static const char *ImpliedExtsZhinx[] = {"zfinx"}; +static const char *ImpliedExtsZhinx[] = {"zhinxmin"}; static const char *ImpliedExtsZhinxmin[] = {"zfinx"}; static const char *ImpliedExtsZicntr[] = {"zicsr"}; static const char *ImpliedExtsZihpm[] = {"zicsr"}; @@ -1030,7 +1030,7 @@ static const char *ImpliedExtsZve64f[] = {"zve64x", "zve32f"}; static const char *ImpliedExtsZve64x[] = {"zve32x", "zvl64b"}; static const char *ImpliedExtsZvfbfmin[] = {"zve32f", "zfbfmin"}; static const char *ImpliedExtsZvfbfwma[] = {"zvfbfmin"}; -static const char *ImpliedExtsZvfh[] = {"zve32f", "zfhmin"}; +static const char *ImpliedExtsZvfh[] = {"zvfhmin", "zfhmin"}; static const char *ImpliedExtsZvfhmin[] = {"zve32f"}; static const char *ImpliedExtsZvkn[] = {"zvkb", "zvkned", "zvknhb", "zvkt"}; static const char *ImpliedExtsZvknc[] = {"zvbc", "zvkn"}; diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index b3d4dc8bb638a..25f6e4a56d932 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -82,6 +82,7 @@ ; RUN: llc -mtriple=riscv32 -mattr=+zve64x -mattr=+experimental-zvksg %s -o - | FileCheck --check-prefix=RV32ZVKSG %s ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvksh %s -o - | FileCheck --check-prefix=RV32ZVKSH %s ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvkt %s -o - | FileCheck --check-prefix=RV32ZVKT %s +; RUN: llc -mtriple=riscv32 -mattr=+zvfh %s -o - | FileCheck --check-prefix=RV32ZVFH %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicond %s -o - | FileCheck --check-prefix=RV32ZICOND %s ; RUN: llc -mtriple=riscv32 -mattr=+smaia %s -o - | FileCheck --check-prefixes=CHECK,RV32SMAIA %s ; RUN: llc -mtriple=riscv32 -mattr=+ssaia %s -o - | FileCheck --check-prefixes=CHECK,RV32SSAIA %s @@ -172,6 +173,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvksg %s -o - | FileCheck --check-prefix=RV64ZVKSG %s ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvksh %s -o - | FileCheck --check-prefix=RV64ZVKSH %s ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvkt %s -o - | FileCheck --check-prefix=RV64ZVKT %s +; RUN: llc -mtriple=riscv64 -mattr=+zvfh %s -o - | FileCheck --check-prefix=RV64ZVFH %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicond %s -o - | FileCheck --check-prefix=RV64ZICOND %s ; RUN: llc -mtriple=riscv64 -mattr=+smaia %s -o - | FileCheck --check-prefixes=CHECK,RV64SMAIA %s ; RUN: llc -mtriple=riscv64 -mattr=+ssaia %s -o - | FileCheck --check-prefixes=CHECK,RV64SSAIA %s @@ -264,6 +266,7 @@ ; RV32ZVKSG: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zve64x1p0_zvkb1p0_zvkg1p0_zvks1p0_zvksed1p0_zvksg1p0_zvksh1p0_zvkt1p0_zvl32b1p0_zvl64b1p0" ; RV32ZVKSH: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvksh1p0_zvl32b1p0" ; RV32ZVKT: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvkt1p0_zvl32b1p0" +; RV32ZVFH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfhmin1p0_zve32f1p0_zve32x1p0_zvfh1p0_zvfhmin1p0_zvl32b1p0" ; RV32ZICOND: .attribute 5, "rv32i2p1_zicond1p0" ; RV32SMAIA: .attribute 5, "rv32i2p1_smaia1p0" ; RV32SSAIA: .attribute 5, "rv32i2p1_ssaia1p0" @@ -353,6 +356,7 @@ ; RV64ZVKSG: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvkb1p0_zvkg1p0_zvks1p0_zvksed1p0_zvksg1p0_zvksh1p0_zvkt1p0_zvl32b1p0" ; RV64ZVKSH: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvksh1p0_zvl32b1p0" ; RV64ZVKT: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvkt1p0_zvl32b1p0" +; RV64ZVFH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfhmin1p0_zve32f1p0_zve32x1p0_zvfh1p0_zvfhmin1p0_zvl32b1p0" ; RV64ZICOND: .attribute 5, "rv64i2p1_zicond1p0" ; RV64SMAIA: .attribute 5, "rv64i2p1_smaia1p0" ; RV64SSAIA: .attribute 5, "rv64i2p1_ssaia1p0" diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s index 25f84f3cc1232..0fedef007a39c 100644 --- a/llvm/test/MC/RISCV/attribute-arch.s +++ b/llvm/test/MC/RISCV/attribute-arch.s @@ -178,7 +178,7 @@ # CHECK: attribute 5, "rv32i2p1_zicsr2p0_zfinx1p0_zhinxmin1p0" .attribute arch, "rv32izfinx_zhinx1p0" -# CHECK: attribute 5, "rv32i2p1_zicsr2p0_zfinx1p0_zhinx1p0" +# CHECK: attribute 5, "rv32i2p1_zicsr2p0_zfinx1p0_zhinx1p0_zhinxmin1p0" .attribute arch, "rv32i_zbkb1p0" # CHECK: attribute 5, "rv32i2p1_zbkb1p0" From 111a2290650743b27f70f9b24618411e54493b59 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Mon, 18 Dec 2023 15:52:14 +0800 Subject: [PATCH 082/884] [RISCV] Implement multi-lib reuse rule for RISC-V bare-metal toolchain (#73765) Extend the multi-lib re-use selection mechanism for RISC-V. This funciton will try to re-use multi-lib if they are compatible. Definition of compatible: - ABI must be the same. - multi-lib is a subset of current arch, e.g. multi-lib=march=rv32im is a subset of march=rv32imc. - march that contains atomic extension can't reuse multi-lib that doesn't has atomic, vice versa. e.g. multi-lib=march=rv32im and march=rv32ima are not compatible, because software and hardware atomic operation can't work together correctly. --- clang/lib/Driver/ToolChains/Gnu.cpp | 127 +++++++++++++++++- .../riscv-toolchain-gcc-multilib-reuse.c | 81 +++++++++++ 2 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index 835215a83c403..02228e19dcf2e 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -30,6 +30,7 @@ #include "llvm/Option/ArgList.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Path.h" +#include "llvm/Support/RISCVISAInfo.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/TargetParser.h" #include @@ -1715,6 +1716,129 @@ static void findCSKYMultilibs(const Driver &D, const llvm::Triple &TargetTriple, Result.Multilibs = CSKYMultilibs; } +/// Extend the multi-lib re-use selection mechanism for RISC-V. +/// This function will try to re-use multi-lib if they are compatible. +/// Definition of compatible: +/// - ABI must be the same. +/// - multi-lib is a subset of current arch, e.g. multi-lib=march=rv32im +/// is a subset of march=rv32imc. +/// - march that contains atomic extension can't reuse multi-lib that +/// doesn't have atomic, vice versa. e.g. multi-lib=march=rv32im and +/// march=rv32ima are not compatible, because software and hardware +/// atomic operation can't work together correctly. +static bool +selectRISCVMultilib(const MultilibSet &RISCVMultilibSet, StringRef Arch, + const Multilib::flags_list &Flags, + llvm::SmallVectorImpl &SelectedMultilibs) { + // Try to find the perfect matching multi-lib first. + if (RISCVMultilibSet.select(Flags, SelectedMultilibs)) + return true; + + Multilib::flags_list NewFlags; + std::vector NewMultilibs; + + llvm::Expected> ParseResult = + llvm::RISCVISAInfo::parseArchString( + Arch, /*EnableExperimentalExtension=*/true, + /*ExperimentalExtensionVersionCheck=*/false); + if (!ParseResult) { + // Ignore any error here, we assume it will be handled in another place. + consumeError(ParseResult.takeError()); + return false; + } + + auto &ISAInfo = *ParseResult; + + addMultilibFlag(ISAInfo->getXLen() == 32, "-m32", NewFlags); + addMultilibFlag(ISAInfo->getXLen() == 64, "-m64", NewFlags); + + // Collect all flags except march=* + for (StringRef Flag : Flags) { + if (Flag.startswith("!march=") || Flag.startswith("-march=")) + continue; + + NewFlags.push_back(Flag.str()); + } + + llvm::StringSet<> AllArchExts; + // Reconstruct multi-lib list, and break march option into separated + // extension. e.g. march=rv32im -> +i +m + for (const auto &M : RISCVMultilibSet) { + bool Skip = false; + + MultilibBuilder NewMultilib = + MultilibBuilder(M.gccSuffix(), M.osSuffix(), M.includeSuffix()); + for (StringRef Flag : M.flags()) { + // Add back all flags except -march. + if (!Flag.consume_front("-march=")) { + NewMultilib.flag(Flag); + continue; + } + + // Break down -march into individual extension. + llvm::Expected> MLConfigParseResult = + llvm::RISCVISAInfo::parseArchString( + Flag, /*EnableExperimentalExtension=*/true, + /*ExperimentalExtensionVersionCheck=*/false); + if (!MLConfigParseResult) { + // Ignore any error here, we assume it will handled in another place. + llvm::consumeError(MLConfigParseResult.takeError()); + + // We might get a parsing error if rv32e in the list, we could just skip + // that and process the rest of multi-lib configs. + Skip = true; + continue; + } + auto &MLConfigISAInfo = *MLConfigParseResult; + + const llvm::RISCVISAInfo::OrderedExtensionMap &MLConfigArchExts = + MLConfigISAInfo->getExtensions(); + for (auto MLConfigArchExt : MLConfigArchExts) { + auto ExtName = MLConfigArchExt.first; + NewMultilib.flag(Twine("-", ExtName).str()); + + if (AllArchExts.insert(ExtName).second) { + addMultilibFlag(ISAInfo->hasExtension(ExtName), + Twine("-", ExtName).str(), NewFlags); + } + } + + // Check the XLEN explicitly. + if (MLConfigISAInfo->getXLen() == 32) { + NewMultilib.flag("-m32"); + NewMultilib.flag("!m64"); + } else { + NewMultilib.flag("!m32"); + NewMultilib.flag("-m64"); + } + + // Atomic extension must be explicitly checked, soft and hard atomic + // operation never co-work correctly. + if (!MLConfigISAInfo->hasExtension("a")) + NewMultilib.flag("!a"); + } + + if (Skip) + continue; + + NewMultilibs.emplace_back(NewMultilib); + } + + // Build an internal used only multi-lib list, used for checking any + // compatible multi-lib. + MultilibSet NewRISCVMultilibs = + MultilibSetBuilder().Either(NewMultilibs).makeMultilibSet(); + + if (NewRISCVMultilibs.select(NewFlags, SelectedMultilibs)) + for (const Multilib &NewSelectedM : SelectedMultilibs) + for (const auto &M : RISCVMultilibSet) + // Look up the corresponding multi-lib entry in original multi-lib set. + if (M.gccSuffix() == NewSelectedM.gccSuffix()) + return true; + + return false; +} + static void findRISCVBareMetalMultilibs(const Driver &D, const llvm::Triple &TargetTriple, StringRef Path, const ArgList &Args, @@ -1766,7 +1890,8 @@ static void findRISCVBareMetalMultilibs(const Driver &D, } } - if (RISCVMultilibs.select(Flags, Result.SelectedMultilibs)) + if (selectRISCVMultilib(RISCVMultilibs, MArch, Flags, + Result.SelectedMultilibs)) Result.Multilibs = RISCVMultilibs; } diff --git a/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c b/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c new file mode 100644 index 0000000000000..1f8a5a8821edf --- /dev/null +++ b/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c @@ -0,0 +1,81 @@ +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32imc -mabi=ilp32 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMC-ILP32 %s +// GCC-MULTI-LIB-REUSE-RV32IMC-ILP32: rv32im/ilp32 +// GCC-MULTI-LIB-REUSE-RV32IMC-ILP32-NOT: {{^.+$}} + +// Check rv32imac won't reuse rv32im or rv32ic +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32imac -mabi=ilp32 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32 %s +// GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32: rv32imac/ilp32 +// GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32--NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32iac -mabi=ilp32 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IAC-ILP32 %s +// GCC-MULTI-LIB-REUSE-RV32IAC-ILP32: rv32iac/ilp32 +// GCC-MULTI-LIB-REUSE-RV32IAC-ILP32-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32imafdc -mabi=ilp32f \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F %s +// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F: rv32imafc/ilp32f +// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32imafdc -mabi=ilp32d \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D %s +// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D: . +// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv64imafc -mabi=lp64 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64 %s +// GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64: rv64imac/lp64 +// GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32imafc_zfh -mabi=ilp32 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32 %s +// GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32: rv32imac/ilp32 +// GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv32i_zvkb -mabi=ilp32 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32 %s +// GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32: rv32i/ilp32 +// GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32-NOT: {{^.+$}} + +// RUN: %clang %s \ +// RUN: -target riscv64-unknown-elf \ +// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ +// RUN: --print-multi-directory \ +// RUN: -march=rv64imfc -mabi=lp64 \ +// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV64IMFC-LP64 %s +// GCC-MULTI-LIB-REUSE-RV64IMFC-LP64: . +// GCC-MULTI-LIB-REUSE-RV64IMFC-LP64-NOT: {{^.+$}} From 9bb47f7f8bcc17d90763d201f383d28489b9b071 Mon Sep 17 00:00:00 2001 From: David Green Date: Mon, 18 Dec 2023 07:59:51 +0000 Subject: [PATCH 083/884] [Flang] Add Maxloc to fir simplify intrinsics pass (#75463) This takes the code from D144103 and extends it to maxloc, to allow the simplifyMinMaxlocReduction method to work with both min and max intrinsics by switching condition and limit/initial value. --- .../Transforms/SimplifyIntrinsics.cpp | 83 +++--- flang/test/Transforms/simplifyintrinsics.fir | 246 ++++++++++++++++++ 2 files changed, 293 insertions(+), 36 deletions(-) diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp index 8ecf7fb44f15d..c89ee6d5e2039 100644 --- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp +++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp @@ -99,8 +99,8 @@ class SimplifyIntrinsicsPass void simplifyLogicalDim1Reduction(fir::CallOp call, const fir::KindMapping &kindMap, GenReductionBodyTy genBodyFunc); - void simplifyMinlocReduction(fir::CallOp call, - const fir::KindMapping &kindMap); + void simplifyMinMaxlocReduction(fir::CallOp call, + const fir::KindMapping &kindMap, bool isMax); void simplifyReductionBody(fir::CallOp call, const fir::KindMapping &kindMap, GenReductionBodyTy genBodyFunc, fir::FirOpBuilder &builder, @@ -353,16 +353,15 @@ genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, // Return the reduction value from the function. builder.create(loc, results[resultIndex]); } -using MinlocBodyOpGeneratorTy = llvm::function_ref &)>; -static void -genMinlocReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, - InitValGeneratorTy initVal, - MinlocBodyOpGeneratorTy genBody, unsigned rank, - mlir::Type elementType, mlir::Location loc, bool hasMask, - mlir::Type maskElemType, mlir::Value resultArr) { +static void genMinMaxlocReductionLoop( + fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, + InitValGeneratorTy initVal, MinMaxlocBodyOpGeneratorTy genBody, + unsigned rank, mlir::Type elementType, mlir::Location loc, bool hasMask, + mlir::Type maskElemType, mlir::Value resultArr) { mlir::IndexType idxTy = builder.getIndexType(); @@ -751,21 +750,24 @@ static mlir::FunctionType genRuntimeMinlocType(fir::FirOpBuilder &builder, {boxRefType, boxType, boxType}, {}); } -static void genRuntimeMinlocBody(fir::FirOpBuilder &builder, - mlir::func::FuncOp &funcOp, unsigned rank, - int maskRank, mlir::Type elementType, - mlir::Type maskElemType, - mlir::Type resultElemTy) { - auto init = [](fir::FirOpBuilder builder, mlir::Location loc, - mlir::Type elementType) { +static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder, + mlir::func::FuncOp &funcOp, bool isMax, + unsigned rank, int maskRank, + mlir::Type elementType, + mlir::Type maskElemType, + mlir::Type resultElemTy) { + auto init = [isMax](fir::FirOpBuilder builder, mlir::Location loc, + mlir::Type elementType) { if (auto ty = elementType.dyn_cast()) { const llvm::fltSemantics &sem = ty.getFloatSemantics(); return builder.createRealConstant( - loc, elementType, llvm::APFloat::getLargest(sem, /*Negative=*/false)); + loc, elementType, llvm::APFloat::getLargest(sem, /*Negative=*/isMax)); } unsigned bits = elementType.getIntOrFloatBitWidth(); - int64_t maxInt = llvm::APInt::getSignedMaxValue(bits).getSExtValue(); - return builder.createIntegerConstant(loc, elementType, maxInt); + int64_t initValue = (isMax ? llvm::APInt::getSignedMinValue(bits) + : llvm::APInt::getSignedMaxValue(bits)) + .getSExtValue(); + return builder.createIntegerConstant(loc, elementType, initValue); }; mlir::Location loc = mlir::UnknownLoc::get(builder.getContext()); @@ -797,18 +799,24 @@ static void genRuntimeMinlocBody(fir::FirOpBuilder &builder, } auto genBodyOp = - [&rank, &resultArr]( - fir::FirOpBuilder builder, mlir::Location loc, mlir::Type elementType, - mlir::Value elem1, mlir::Value elem2, - llvm::SmallVector indices) + [&rank, &resultArr, + isMax](fir::FirOpBuilder builder, mlir::Location loc, + mlir::Type elementType, mlir::Value elem1, mlir::Value elem2, + llvm::SmallVector indices) -> mlir::Value { mlir::Value cmp; if (elementType.isa()) { cmp = builder.create( - loc, mlir::arith::CmpFPredicate::OLT, elem1, elem2); + loc, + isMax ? mlir::arith::CmpFPredicate::OGT + : mlir::arith::CmpFPredicate::OLT, + elem1, elem2); } else if (elementType.isa()) { cmp = builder.create( - loc, mlir::arith::CmpIPredicate::slt, elem1, elem2); + loc, + isMax ? mlir::arith::CmpIPredicate::sgt + : mlir::arith::CmpIPredicate::slt, + elem1, elem2); } else { llvm_unreachable("unsupported type"); } @@ -875,9 +883,8 @@ static void genRuntimeMinlocBody(fir::FirOpBuilder &builder, // bit of a hack - maskRank is set to -1 for absent mask arg, so don't // generate high level mask or element by element mask. bool hasMask = maskRank > 0; - - genMinlocReductionLoop(builder, funcOp, init, genBodyOp, rank, elementType, - loc, hasMask, maskElemType, resultArr); + genMinMaxlocReductionLoop(builder, funcOp, init, genBodyOp, rank, elementType, + loc, hasMask, maskElemType, resultArr); } /// Generate function type for the simplified version of RTNAME(DotProduct) @@ -1150,8 +1157,8 @@ void SimplifyIntrinsicsPass::simplifyLogicalDim1Reduction( intElementType); } -void SimplifyIntrinsicsPass::simplifyMinlocReduction( - fir::CallOp call, const fir::KindMapping &kindMap) { +void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( + fir::CallOp call, const fir::KindMapping &kindMap, bool isMax) { mlir::Operation::operand_range args = call.getArgs(); @@ -1217,11 +1224,11 @@ void SimplifyIntrinsicsPass::simplifyMinlocReduction( auto typeGenerator = [rank](fir::FirOpBuilder &builder) { return genRuntimeMinlocType(builder, rank); }; - auto bodyGenerator = [rank, maskRank, inputType, logicalElemType, - outType](fir::FirOpBuilder &builder, - mlir::func::FuncOp &funcOp) { - genRuntimeMinlocBody(builder, funcOp, rank, maskRank, inputType, - logicalElemType, outType); + auto bodyGenerator = [rank, maskRank, inputType, logicalElemType, outType, + isMax](fir::FirOpBuilder &builder, + mlir::func::FuncOp &funcOp) { + genRuntimeMinMaxlocBody(builder, funcOp, isMax, rank, maskRank, inputType, + logicalElemType, outType); }; mlir::func::FuncOp newFunc = @@ -1367,7 +1374,11 @@ void SimplifyIntrinsicsPass::runOnOperation() { return; } if (funcName.starts_with(RTNAME_STRING(Minloc))) { - simplifyMinlocReduction(call, kindMap); + simplifyMinMaxlocReduction(call, kindMap, false); + return; + } + if (funcName.starts_with(RTNAME_STRING(Maxloc))) { + simplifyMinMaxlocReduction(call, kindMap, true); return; } } diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir index 39483a9cc18fe..0bd6ac7c436ff 100644 --- a/flang/test/Transforms/simplifyintrinsics.fir +++ b/flang/test/Transforms/simplifyintrinsics.fir @@ -2348,3 +2348,249 @@ func.func @_QPtestminloc_doesntwork1d_unknownmask(%arg0: !fir.ref : (!fir.ref>, !fir.box, !fir.box) -> () // CHECK: fir.call @_FortranAMinlocInteger4({{.*}}) fastmath : (!fir.ref>, !fir.box, i32, !fir.ref, i32, !fir.box, i1) -> none // CHECK-NOT: fir.call @_FortranAMinlocInteger4x1_i32_contract_simplified({{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () + +// ----- +// Check Maxloc simplifies similarly to minloc +func.func @_QPtestmaxloc_works1d(%arg0: !fir.ref> {fir.bindc_name = "a"}, %arg1: !fir.ref>> {fir.bindc_name = "b"}) -> !fir.array<1xi32> { + %0 = fir.alloca !fir.box>> + %c10 = arith.constant 10 : index + %c10_0 = arith.constant 10 : index + %c1 = arith.constant 1 : index + %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testmaxloc_works1d", uniq_name = "_QFtestmaxloc_works1dEtestmaxloc_works1d"} + %2 = fir.shape %c1 : (index) -> !fir.shape<1> + %3 = fir.array_load %1(%2) : (!fir.ref>, !fir.shape<1>) -> !fir.array<1xi32> + %4 = fir.shape %c10 : (index) -> !fir.shape<1> + %5 = fir.embox %arg0(%4) : (!fir.ref>, !fir.shape<1>) -> !fir.box> + %6 = fir.shape %c10_0 : (index) -> !fir.shape<1> + %7 = fir.embox %arg1(%6) : (!fir.ref>>, !fir.shape<1>) -> !fir.box>> + %c4 = arith.constant 4 : index + %false = arith.constant false + %8 = fir.zero_bits !fir.heap> + %c0 = arith.constant 0 : index + %9 = fir.shape %c0 : (index) -> !fir.shape<1> + %10 = fir.embox %8(%9) : (!fir.heap>, !fir.shape<1>) -> !fir.box>> + fir.store %10 to %0 : !fir.ref>>> + %11 = fir.address_of(@_QQclXea5bcf7f706678e1796661f8916f3379) : !fir.ref> + %c5_i32 = arith.constant 5 : i32 + %12 = fir.convert %0 : (!fir.ref>>>) -> !fir.ref> + %13 = fir.convert %5 : (!fir.box>) -> !fir.box + %14 = fir.convert %c4 : (index) -> i32 + %15 = fir.convert %11 : (!fir.ref>) -> !fir.ref + %16 = fir.convert %7 : (!fir.box>>) -> !fir.box + %17 = fir.call @_FortranAMaxlocInteger4(%12, %13, %14, %15, %c5_i32, %16, %false) fastmath : (!fir.ref>, !fir.box, i32, !fir.ref, i32, !fir.box, i1) -> none + %18 = fir.load %0 : !fir.ref>>> + %c0_1 = arith.constant 0 : index + %19:3 = fir.box_dims %18, %c0_1 : (!fir.box>>, index) -> (index, index, index) + %20 = fir.box_addr %18 : (!fir.box>>) -> !fir.heap> + %21 = fir.shape_shift %19#0, %19#1 : (index, index) -> !fir.shapeshift<1> + %22 = fir.array_load %20(%21) : (!fir.heap>, !fir.shapeshift<1>) -> !fir.array + %c1_2 = arith.constant 1 : index + %c0_3 = arith.constant 0 : index + %23 = arith.subi %c1, %c1_2 : index + %24 = fir.do_loop %arg2 = %c0_3 to %23 step %c1_2 unordered iter_args(%arg3 = %3) -> (!fir.array<1xi32>) { + %26 = fir.array_fetch %22, %arg2 : (!fir.array, index) -> i32 + %27 = fir.array_update %arg3, %26, %arg2 : (!fir.array<1xi32>, i32, index) -> !fir.array<1xi32> + fir.result %27 : !fir.array<1xi32> + } + fir.array_merge_store %3, %24 to %1 : !fir.array<1xi32>, !fir.array<1xi32>, !fir.ref> + fir.freemem %20 : !fir.heap> + %25 = fir.load %1 : !fir.ref> + return %25 : !fir.array<1xi32> +} + +// CHECK-LABEL: func.func @_QPtestmaxloc_works1d( +// CHECK-SAME: %[[INARR:.*]]: !fir.ref> {fir.bindc_name = "a"}, +// CHECK-SAME: %[[MASK:.*]]: !fir.ref>> {fir.bindc_name = "b"}) -> !fir.array<1xi32> { +// CHECK: %[[OUTARR:.*]] = fir.alloca !fir.box>> +// CHECK: %[[SIZE10_0:.*]] = arith.constant 10 : index +// CHECK: %[[SIZE10_1:.*]] = arith.constant 10 : index +// CHECK: %[[INARR_SHAPE:.*]] = fir.shape %[[SIZE10_0]] : (index) -> !fir.shape<1> +// CHECK: %[[BOX_INARR:.*]] = fir.embox %[[INARR]](%[[INARR_SHAPE]]) : (!fir.ref>, !fir.shape<1>) -> !fir.box> +// CHECK: %[[MASK_SHAPE:.*]] = fir.shape %[[SIZE10_1]] : (index) -> !fir.shape<1> +// CHECK: %[[BOX_MASK:.*]] = fir.embox %[[MASK]](%[[MASK_SHAPE]]) : (!fir.ref>>, !fir.shape<1>) -> !fir.box>> +// CHECK: %[[REF_BOX_OUTARR_NONE:.*]] = fir.convert %[[OUTARR]] : (!fir.ref>>>) -> !fir.ref> +// CHECK: %[[BOX_INARR_NONE:.*]] = fir.convert %[[BOX_INARR]] : (!fir.box>) -> !fir.box +// CHECK: %[[BOX_MASK_NONE:.*]] = fir.convert %[[BOX_MASK]] : (!fir.box>>) -> !fir.box +// CHECK: fir.call @_FortranAMaxlocInteger4x1_Logical4x1_i32_contract_simplified(%[[REF_BOX_OUTARR_NONE]], %[[BOX_INARR_NONE]], %[[BOX_MASK_NONE]]) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () + +// CHECK-LABEL: func.func private @_FortranAMaxlocInteger4x1_Logical4x1_i32_contract_simplified( +// CHECK-SAME: %[[REF_BOX_OUTARR_NONE:.*]]: !fir.ref>, +// CHECK-SAME: %[[BOX_INARR_NONE:.*]]: !fir.box, +// CHECK-SAME: %[[BOX_MASK_NONE:.*]]: !fir.box) attributes {llvm.linkage = #llvm.linkage} { +// CHECK: %[[FLAG_ALLOC:.*]] = fir.alloca i32 +// CHECK: %[[INIT_OUT_IDX:.*]] = arith.constant 0 : i32 +// CHECK: %[[OUTARR_SIZE:.*]] = arith.constant 1 : index +// CHECK: %[[OUTARR:.*]] = fir.allocmem !fir.array<1xi32> +// CHECK: %[[OUTARR_SHAPE:.*]] = fir.shape %[[OUTARR_SIZE]] : (index) -> !fir.shape<1> +// CHECK: %[[BOX_OUTARR:.*]] = fir.embox %[[OUTARR]](%[[OUTARR_SHAPE]]) : (!fir.heap>, !fir.shape<1>) -> !fir.box>> +// CHECK: %[[OUTARR_IDX0:.*]] = arith.constant 0 : index +// CHECK: %[[OUTARR_ITEM0:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX0]] : (!fir.box>>, index) -> !fir.ref +// CHECK: fir.store %[[INIT_OUT_IDX]] to %[[OUTARR_ITEM0]] : !fir.ref +// CHECK: %[[CINDEX_0:.*]] = arith.constant 0 : index +// CHECK: %[[BOX_INARR:.*]] = fir.convert %[[BOX_INARR_NONE]] : (!fir.box) -> !fir.box> +// CHECK: %[[FLAG_SET:.*]] = arith.constant 1 : i32 +// CHECK: %[[FLAG_EMPTY:.*]] = arith.constant 0 : i32 +// CHECK: fir.store %[[FLAG_EMPTY]] to %[[FLAG_ALLOC]] : !fir.ref +// CHECK: %[[BOX_MASK:.*]] = fir.convert %[[BOX_MASK_NONE]] : (!fir.box) -> !fir.box>> +// CHECK: %[[MAX:.*]] = arith.constant -2147483648 : i32 +// CHECK: %[[CINDEX_1:.*]] = arith.constant 1 : index +// CHECK: %[[DIM_INDEX0:.*]] = arith.constant 0 : index +// CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[BOX_INARR]], %[[DIM_INDEX0]] : (!fir.box>, index) -> (index, index, index) +// CHECK: %[[EXTENT:.*]] = arith.subi %[[DIMS]]#1, %[[CINDEX_1]] : index +// CHECK: %[[DOLOOP:.*]] = fir.do_loop %[[ITER:.*]] = %[[CINDEX_0]] to %[[EXTENT]] step %[[CINDEX_1]] iter_args(%[[MIN:.*]] = %[[MAX]]) -> (i32) { +// CHECK: %[[MASK_ITEM:.*]] = fir.coordinate_of %[[BOX_MASK]], %[[ITER]] : (!fir.box>>, index) -> !fir.ref> +// CHECK: %[[MASK_ITEMVAL:.*]] = fir.load %[[MASK_ITEM]] : !fir.ref> +// CHECK: %[[MASK_IF_ITEM:.*]] = fir.convert %[[MASK_ITEMVAL]] : (!fir.logical<4>) -> i1 +// CHECK: %[[IF_MASK:.*]] = fir.if %[[MASK_IF_ITEM]] -> (i32) { +// CHECK: fir.store %[[FLAG_SET]] to %[[FLAG_ALLOC]] : !fir.ref +// CHECK: %[[INARR_ITEM:.*]] = fir.coordinate_of %[[BOX_INARR]], %[[ITER]] : (!fir.box>, index) -> !fir.ref +// CHECK: %[[INARR_ITEMVAL:.*]] = fir.load %[[INARR_ITEM]] : !fir.ref +// CHECK: %[[NEW_MIN:.*]] = arith.cmpi sgt, %[[INARR_ITEMVAL]], %[[MIN]] : i32 +// CHECK: %[[IF_NEW_MIN:.*]] = fir.if %[[NEW_MIN]] -> (i32) { +// CHECK: %[[ONE:.*]] = arith.constant 1 : i32 +// CHECK: %[[OUTARR_IDX:.*]] = arith.constant 0 : index +// CHECK: %[[OUTARR_ITEM:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX]] : (!fir.box>>, index) -> !fir.ref +// CHECK: %[[ITER_I32:.*]] = fir.convert %[[ITER]] : (index) -> i32 +// CHECK: %[[FORTRAN_IDX:.*]] = arith.addi %[[ITER_I32]], %[[ONE]] : i32 +// CHECK: fir.store %[[FORTRAN_IDX]] to %[[OUTARR_ITEM]] : !fir.ref +// CHECK: fir.result %[[INARR_ITEMVAL]] : i32 +// CHECK: } else { +// CHECK: fir.result %[[MIN]] : i32 +// CHECK: } +// CHECK: fir.result %[[IF_NEW_MIN:.*]] : i32 +// CHECK: } else { +// CHECK: fir.result %[[MIN]] : i32 +// CHECK: } +// CHECK: fir.result %[[IF_MASK:.*]] : i32 +// CHECK: } +// CHECK: %[[FLAG_VAL:.*]] = fir.load %[[FLAG_ALLOC]] : !fir.ref +// CHECK: %[[FLAG_WAS_SET:.*]] = arith.cmpi eq, %[[FLAG_VAL]], %[[FLAG_SET]] : i32 +// CHECK: fir.if %[[FLAG_WAS_SET]] { +// CHECK: %[[TEST_MAX:.*]] = arith.constant -2147483648 : i32 +// CHECK: %[[INIT_NOT_CHANGED:.*]] = arith.cmpi eq, %[[TEST_MAX]], %[[DO_LOOP:.*]] : i32 +// CHECK: fir.if %[[INIT_NOT_CHANGED]] { +// CHECK: %[[FLAG_OUTARR_IDX:.*]] = arith.constant 0 : index +// CHECK: %[[FLAG_OUTARR_ITEM:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[FLAG_OUTARR_IDX]] : (!fir.box>>, index) -> !fir.ref +// CHECK: fir.store %[[FLAG_SET]] to %[[FLAG_OUTARR_ITEM]] : !fir.ref +// CHECK: } +// CHECK: } +// CHECK: %[[REF_BOX_OUTARR:.*]] = fir.convert %[[REF_BOX_OUTARR_NONE]] : (!fir.ref>) -> !fir.ref>>> +// CHECK: fir.store %[[BOX_OUTARR]] to %[[REF_BOX_OUTARR]] : !fir.ref>>> +// CHECK: return +// CHECK: } + +// ----- +// Check Maxloc simplifies correctly for 1D case with scalar mask and f64 input + +func.func @_QPtestmaxloc_works1d_scalarmask_f64(%arg0: !fir.ref> {fir.bindc_name = "a"}, %arg1: !fir.ref> {fir.bindc_name = "b"}) -> !fir.array<1xi32> { + %0 = fir.alloca !fir.box>> + %c10 = arith.constant 10 : index + %c1 = arith.constant 1 : index + %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testmaxloc_works1d_scalarmask_f64", uniq_name = "_QFtestmaxloc_works1d_scalarmask_f64Etestminloc_works1d_scalarmask_f64"} + %2 = fir.shape %c1 : (index) -> !fir.shape<1> + %3 = fir.array_load %1(%2) : (!fir.ref>, !fir.shape<1>) -> !fir.array<1xi32> + %4 = fir.shape %c10 : (index) -> !fir.shape<1> + %5 = fir.embox %arg0(%4) : (!fir.ref>, !fir.shape<1>) -> !fir.box> + %6 = fir.embox %arg1 : (!fir.ref>) -> !fir.box> + %c4 = arith.constant 4 : index + %false = arith.constant false + %7 = fir.zero_bits !fir.heap> + %c0 = arith.constant 0 : index + %8 = fir.shape %c0 : (index) -> !fir.shape<1> + %9 = fir.embox %7(%8) : (!fir.heap>, !fir.shape<1>) -> !fir.box>> + fir.store %9 to %0 : !fir.ref>>> + %10 = fir.address_of(@_QQclX66951c28c5b8bab5cdb25c1ac762b978) : !fir.ref> + %c6_i32 = arith.constant 6 : i32 + %11 = fir.convert %0 : (!fir.ref>>>) -> !fir.ref> + %12 = fir.convert %5 : (!fir.box>) -> !fir.box + %13 = fir.convert %c4 : (index) -> i32 + %14 = fir.convert %10 : (!fir.ref>) -> !fir.ref + %15 = fir.convert %6 : (!fir.box>) -> !fir.box + %16 = fir.call @_FortranAMaxlocReal8(%11, %12, %13, %14, %c6_i32, %15, %false) fastmath : (!fir.ref>, !fir.box, i32, !fir.ref, i32, !fir.box, i1) -> none + %17 = fir.load %0 : !fir.ref>>> + %c0_0 = arith.constant 0 : index + %18:3 = fir.box_dims %17, %c0_0 : (!fir.box>>, index) -> (index, index, index) + %19 = fir.box_addr %17 : (!fir.box>>) -> !fir.heap> + %20 = fir.shape_shift %18#0, %18#1 : (index, index) -> !fir.shapeshift<1> + %21 = fir.array_load %19(%20) : (!fir.heap>, !fir.shapeshift<1>) -> !fir.array + %c1_1 = arith.constant 1 : index + %c0_2 = arith.constant 0 : index + %22 = arith.subi %c1, %c1_1 : index + %23 = fir.do_loop %arg2 = %c0_2 to %22 step %c1_1 unordered iter_args(%arg3 = %3) -> (!fir.array<1xi32>) { + %25 = fir.array_fetch %21, %arg2 : (!fir.array, index) -> i32 + %26 = fir.array_update %arg3, %25, %arg2 : (!fir.array<1xi32>, i32, index) -> !fir.array<1xi32> + fir.result %26 : !fir.array<1xi32> + } + fir.array_merge_store %3, %23 to %1 : !fir.array<1xi32>, !fir.array<1xi32>, !fir.ref> + fir.freemem %19 : !fir.heap> + %24 = fir.load %1 : !fir.ref> + return %24 : !fir.array<1xi32> +} + +// CHECK-LABEL: func.func @_QPtestmaxloc_works1d_scalarmask_f64( +// CHECK-SAME: %[[INARR:.*]]: !fir.ref> {fir.bindc_name = "a"}, +// CHECK-SAME: %[[MASK:.*]]: !fir.ref> {fir.bindc_name = "b"}) -> !fir.array<1xi32> { +// CHECK: fir.call @_FortranAMaxlocReal8x1_Logical4x0_i32_contract_simplified({{.*}}, {{.*}}, {{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () + +// CHECK-LABEL: func.func private @_FortranAMaxlocReal8x1_Logical4x0_i32_contract_simplified( +// CHECK-SAME: %[[REF_BOX_OUTARR_NONE:.*]]: !fir.ref>, +// CHECK-SAME: %[[BOX_INARR_NONE:.*]]: !fir.box, +// CHECK-SAME: %[[BOX_MASK_NONE:.*]]: !fir.box) attributes {llvm.linkage = #llvm.linkage} { +// CHECK: %[[FLAG_ALLOC:.*]] = fir.alloca i32 +// CHECK: %[[INIT_OUT_IDX:.*]] = arith.constant 0 : i32 +// CHECK: %[[OUTARR_SIZE:.*]] = arith.constant 1 : index +// CHECK: %[[OUTARR:.*]] = fir.allocmem !fir.array<1xi32> +// CHECK: %[[OUTARR_SHAPE:.*]] = fir.shape %[[OUTARR_SIZE]] : (index) -> !fir.shape<1> +// CHECK: %[[BOX_OUTARR:.*]] = fir.embox %[[OUTARR]](%[[OUTARR_SHAPE]]) : (!fir.heap>, !fir.shape<1>) -> !fir.box>> +// CHECK: %[[OUTARR_IDX0:.*]] = arith.constant 0 : index +// CHECK: %[[OUTARR_ITEM0:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX0]] : (!fir.box>>, index) -> !fir.ref +// CHECK: fir.store %[[INIT_OUT_IDX]] to %[[OUTARR_ITEM0]] : !fir.ref +// CHECK: %[[BOX_MASK:.*]] = fir.convert %[[BOX_MASK_NONE]] : (!fir.box) -> !fir.box> +// CHECK: %[[MASK_IDX0:.*]] = arith.constant 0 : index +// CHECK: %[[MASK_ITEM:.*]] = fir.coordinate_of %[[BOX_MASK]], %[[MASK_IDX0]] : (!fir.box>, index) -> !fir.ref +// CHECK: %[[MASK:.*]] = fir.load %[[MASK_ITEM]] : !fir.ref +// CHECK: %[[INIT_RES:.*]] = fir.if %[[MASK]] -> (f64) { +// CHECK: %[[C_INDEX0:.*]] = arith.constant 0 : index +// CHECK: %[[BOX_INARR:.*]] = fir.convert %[[BOX_INARR_NONE]] : (!fir.box) -> !fir.box> +// CHECK: %[[FLAG_SET:.*]] = arith.constant 1 : i32 +// CHECK: %[[FLAG_EMPTY:.*]] = arith.constant 0 : i32 +// CHECK: fir.store %[[FLAG_EMPTY]] to %[[FLAG_ALLOC]] : !fir.ref +// CHECK: %[[MAX:.*]] = arith.constant -1.7976931348623157E+308 : f64 +// CHECK: %[[C_INDEX1:.*]] = arith.constant 1 : index +// CHECK: %[[DIM_INDEX:.*]] = arith.constant 0 : index +// CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[BOX_INARR]], %[[DIM_INDEX]] : (!fir.box>, index) -> (index, index, index) +// CHECK: %[[EXTENT:.*]] = arith.subi %[[DIMS]]#1, %[[C_INDEX1]] : index +// CHECK: %[[DOLOOP:.*]] = fir.do_loop %[[ITER:.*]] = %[[C_INDEX0]] to %[[EXTENT]] step %[[C_INDEX1]] iter_args(%[[MIN:.*]] = %[[MAX]]) -> (f64) { +// CHECK: %[[INARR_ITEM:.*]] = fir.coordinate_of %[[BOX_INARR]], %[[ITER]] : (!fir.box>, index) -> !fir.ref +// CHECK: %[[INARR_ITEMVAL:.*]] = fir.load %[[INARR_ITEM]] : !fir.ref +// CHECK: %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[INARR_ITEMVAL]], %[[MIN]] fastmath<{{.*}}> : f64 +// CHECK: %[[IF_NEW_MIN:.*]] = fir.if %[[NEW_MIN]] -> (f64) { +// CHECK: %[[ONE:.*]] = arith.constant 1 : i32 +// CHECK: %[[OUTARR_IDX:.*]] = arith.constant 0 : index +// CHECK: %[[OUTARR_ITEM:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX]] : (!fir.box>>, index) -> !fir.ref +// CHECK: %[[ITER_I32:.*]] = fir.convert %[[ITER]] : (index) -> i32 +// CHECK: %[[FORTRAN_IDX:.*]] = arith.addi %[[ITER_I32]], %[[ONE]] : i32 +// CHECK: fir.store %[[FORTRAN_IDX]] to %[[OUTARR_ITEM]] : !fir.ref +// CHECK: fir.result %[[INARR_ITEMVAL]] : f64 +// CHECK: } else { +// CHECK: fir.result %[[MIN]] : f64 +// CHECK: } +// CHECK: fir.result %[[IF_NEW_MIN:.*]] : f64 +// CHECK: } +// CHECK: } +// CHECK: %[[FLAG_CHECK:.*]] = arith.constant 1 : i32 +// CHECK: %[[FLAG_VAL:.*]] = fir.load %[[FLAG_ALLOC]] : !fir.ref +// CHECK: %[[FLAG_WAS_SET:.*]] = arith.cmpi eq, %[[FLAG_VAL]], %[[FLAG_CHECK]] : i32 +// CHECK: fir.if %[[FLAG_WAS_SET]] { +// CHECK: %[[TEST_MAX:.*]] = arith.constant -1.7976931348623157E+308 : f64 +// CHECK: %[[INIT_NOT_CHANGED:.*]] = arith.cmpf oeq, %[[TEST_MAX]], %[[INIT_RES:.*]] fastmath<{{.*}}> : f64 +// CHECK: fir.if %[[INIT_NOT_CHANGED]] { +// CHECK: %[[FLAG_OUTARR_IDX:.*]] = arith.constant 0 : index +// CHECK: %[[FLAG_OUTARR_ITEM:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[FLAG_OUTARR_IDX]] : (!fir.box>>, index) -> !fir.ref +// CHECK: fir.store %[[FLAG_CHECK]] to %[[FLAG_OUTARR_ITEM]] : !fir.ref +// CHECK: } +// CHECK: } +// CHECK: %[[REF_BOX_OUTARR:.*]] = fir.convert %[[VAL_0]] : (!fir.ref>) -> !fir.ref>>> +// CHECK: fir.store %[[BOX_OUTARR]] to %[[REF_BOX_OUTARR]] : !fir.ref>>> +// CHECK: return +// CHECK: } From ca1034341cfec226c09ff0e473c6ecbcc2a1194c Mon Sep 17 00:00:00 2001 From: martinboehme Date: Mon, 18 Dec 2023 09:10:03 +0100 Subject: [PATCH 084/884] [clang][dataflow] Fix an issue with `Environment::getResultObjectLocation()`. (#75483) So far, if there was a chain of record type prvalues, `getResultObjectLocation()` would assign a different result object location to each one. This makes no sense, of course, as all of these prvalues end up initializing the same result object. This patch fixes this by propagating storage locations up through the entire chain of prvalues. The new implementation also has the desirable effect of making it possible to make `getResultObjectLocation()` const, which seems appropriate given that, logically, it is just an accessor. --- .../FlowSensitive/DataflowEnvironment.h | 33 ++------ .../FlowSensitive/DataflowEnvironment.cpp | 79 ++++++++++++++----- clang/lib/Analysis/FlowSensitive/Transfer.cpp | 9 ++- .../Analysis/FlowSensitive/TransferTest.cpp | 49 ++++++++++++ 4 files changed, 123 insertions(+), 47 deletions(-) diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index d7e39ab2616fd..5943af50b6ad8 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -325,7 +325,8 @@ class Environment { /// /// Requirements: /// `E` must be a prvalue of record type. - RecordStorageLocation &getResultObjectLocation(const Expr &RecordPRValue); + RecordStorageLocation & + getResultObjectLocation(const Expr &RecordPRValue) const; /// Returns the return value of the current function. This can be null if: /// - The function has a void return type @@ -434,24 +435,14 @@ class Environment { /// Assigns `Val` as the value of the prvalue `E` in the environment. /// - /// If `E` is not yet associated with a storage location, associates it with - /// a newly created storage location. In any case, associates the storage - /// location of `E` with `Val`. - /// - /// Once the migration to strict handling of value categories is complete - /// (see https://discourse.llvm.org/t/70086), this function will be renamed to - /// `setValue()`. At this point, prvalue expressions will be associated - /// directly with `Value`s, and the legacy behavior of associating prvalue - /// expressions with storage locations (as described above) will be - /// eliminated. - /// /// Requirements: /// - /// `E` must be a prvalue - /// If `Val` is a `RecordValue`, its `RecordStorageLocation` must be the - /// same as that of any `RecordValue` that has already been associated with - /// `E`. This is to guarantee that the result object initialized by a prvalue - /// `RecordValue` has a durable storage location. + /// - `E` must be a prvalue + /// - If `Val` is a `RecordValue`, its `RecordStorageLocation` must be + /// `getResultObjectLocation(E)`. An exception to this is if `E` is an + /// expression that originally creates a `RecordValue` (such as a + /// `CXXConstructExpr` or `CallExpr`), as these establish the location of + /// the result object in the first place. void setValue(const Expr &E, Value &Val); /// Returns the value assigned to `Loc` in the environment or null if `Loc` @@ -608,14 +599,6 @@ class Environment { // The copy-constructor is for use in fork() only. Environment(const Environment &) = default; - /// Internal version of `setStorageLocation()` that doesn't check if the - /// expression is a prvalue. - void setStorageLocationInternal(const Expr &E, StorageLocation &Loc); - - /// Internal version of `getStorageLocation()` that doesn't check if the - /// expression is a prvalue. - StorageLocation *getStorageLocationInternal(const Expr &E) const; - /// Creates a value appropriate for `Type`, if `Type` is supported, otherwise /// return null. /// diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index b98037b736452..93919cd0243d0 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -726,27 +726,70 @@ void Environment::setStorageLocation(const Expr &E, StorageLocation &Loc) { // so allow these as an exception. assert(E.isGLValue() || E.getType()->isSpecificBuiltinType(BuiltinType::BuiltinFn)); - setStorageLocationInternal(E, Loc); + const Expr &CanonE = ignoreCFGOmittedNodes(E); + assert(!ExprToLoc.contains(&CanonE)); + ExprToLoc[&CanonE] = &Loc; } StorageLocation *Environment::getStorageLocation(const Expr &E) const { // See comment in `setStorageLocation()`. assert(E.isGLValue() || E.getType()->isSpecificBuiltinType(BuiltinType::BuiltinFn)); - return getStorageLocationInternal(E); + auto It = ExprToLoc.find(&ignoreCFGOmittedNodes(E)); + return It == ExprToLoc.end() ? nullptr : &*It->second; +} + +// Returns whether a prvalue of record type is the one that originally +// constructs the object (i.e. it doesn't propagate it from one of its +// children). +static bool isOriginalRecordConstructor(const Expr &RecordPRValue) { + if (auto *Init = dyn_cast(&RecordPRValue)) + return !Init->isSemanticForm() || !Init->isTransparent(); + return isa(RecordPRValue) || isa(RecordPRValue) || + isa(RecordPRValue) || + // The framework currently does not propagate the objects created in + // the two branches of a `ConditionalOperator` because there is no way + // to reconcile their storage locations, which are different. We + // therefore claim that the `ConditionalOperator` is the expression + // that originally constructs the object. + // Ultimately, this will be fixed by propagating locations down from + // the result object, rather than up from the original constructor as + // we do now (see also the FIXME in the documentation for + // `getResultObjectLocation()`). + isa(RecordPRValue); } RecordStorageLocation & -Environment::getResultObjectLocation(const Expr &RecordPRValue) { +Environment::getResultObjectLocation(const Expr &RecordPRValue) const { assert(RecordPRValue.getType()->isRecordType()); assert(RecordPRValue.isPRValue()); - if (StorageLocation *ExistingLoc = getStorageLocationInternal(RecordPRValue)) - return *cast(ExistingLoc); - auto &Loc = cast( - DACtx->getStableStorageLocation(RecordPRValue)); - setStorageLocationInternal(RecordPRValue, Loc); - return Loc; + // Returns a storage location that we can use if assertions fail. + auto FallbackForAssertFailure = + [this, &RecordPRValue]() -> RecordStorageLocation & { + return cast( + DACtx->getStableStorageLocation(RecordPRValue)); + }; + + if (isOriginalRecordConstructor(RecordPRValue)) { + auto *Val = cast_or_null(getValue(RecordPRValue)); + // The builtin transfer function should have created a `RecordValue` for all + // original record constructors. + assert(Val); + if (!Val) + return FallbackForAssertFailure(); + return Val->getLoc(); + } + + // Expression nodes that propagate a record prvalue should have exactly one + // child. + llvm::SmallVector children(RecordPRValue.child_begin(), + RecordPRValue.child_end()); + assert(children.size() == 1); + if (children.empty()) + return FallbackForAssertFailure(); + + return getResultObjectLocation(*cast(children[0])); } PointerValue &Environment::getOrCreateNullPointerValue(QualType PointeeType) { @@ -760,6 +803,11 @@ void Environment::setValue(const StorageLocation &Loc, Value &Val) { } void Environment::setValue(const Expr &E, Value &Val) { + if (auto *RecordVal = dyn_cast(&Val)) { + assert(isOriginalRecordConstructor(E) || + &RecordVal->getLoc() == &getResultObjectLocation(E)); + } + assert(E.isPRValue()); ExprToVal[&E] = &Val; } @@ -799,18 +847,6 @@ Value *Environment::createValue(QualType Type) { return Val; } -void Environment::setStorageLocationInternal(const Expr &E, - StorageLocation &Loc) { - const Expr &CanonE = ignoreCFGOmittedNodes(E); - assert(!ExprToLoc.contains(&CanonE)); - ExprToLoc[&CanonE] = &Loc; -} - -StorageLocation *Environment::getStorageLocationInternal(const Expr &E) const { - auto It = ExprToLoc.find(&ignoreCFGOmittedNodes(E)); - return It == ExprToLoc.end() ? nullptr : &*It->second; -} - Value *Environment::createValueUnlessSelfReferential( QualType Type, llvm::DenseSet &Visited, int Depth, int &CreatedValuesCount) { @@ -1044,6 +1080,7 @@ RecordValue &refreshRecordValue(const Expr &Expr, Environment &Env) { if (auto *ExistingVal = cast_or_null(Env.getValue(Expr))) { auto &NewVal = Env.create(ExistingVal->getLoc()); Env.setValue(Expr, NewVal); + Env.setValue(NewVal.getLoc(), NewVal); return NewVal; } diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp index bbf5f12359bc7..346469660662e 100644 --- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp +++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp @@ -489,7 +489,6 @@ class TransferVisitor : public ConstStmtVisitor { if (S->getType()->isRecordType()) { auto &InitialVal = *cast(Env.createValue(S->getType())); Env.setValue(*S, InitialVal); - copyRecord(InitialVal.getLoc(), Env.getResultObjectLocation(*S), Env); } transferInlineCall(S, ConstructorDecl); @@ -582,6 +581,14 @@ class TransferVisitor : public ConstStmtVisitor { Env.setValue(*S, *ArgVal); } else if (const FunctionDecl *F = S->getDirectCallee()) { transferInlineCall(S, F); + + // If this call produces a prvalue of record type, make sure that we have + // a `RecordValue` for it. This is required so that + // `Environment::getResultObjectLocation()` is able to return a location + // for this `CallExpr`. + if (S->getType()->isRecordType() && S->isPRValue()) + if (Env.getValue(*S) == nullptr) + refreshRecordValue(*S, Env); } } diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index 8da55953a3298..056c4f3383d83 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -2635,6 +2635,55 @@ TEST(TransferTest, BindTemporary) { }); } +TEST(TransferTest, ResultObjectLocation) { + std::string Code = R"( + struct A { + virtual ~A() = default; + }; + + void target() { + A(); + (void)0; // [[p]] + } + )"; + using ast_matchers::cxxBindTemporaryExpr; + using ast_matchers::cxxTemporaryObjectExpr; + using ast_matchers::exprWithCleanups; + using ast_matchers::has; + using ast_matchers::match; + using ast_matchers::selectFirst; + using ast_matchers::traverse; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + + // The expresssion `A()` in the code above produces the following + // structure, consisting of three prvalues of record type. + // `Env.getResultObjectLocation()` should return the same location for + // all of these. + auto MatchResult = match( + traverse(TK_AsIs, + exprWithCleanups( + has(cxxBindTemporaryExpr( + has(cxxTemporaryObjectExpr().bind("toe"))) + .bind("bte"))) + .bind("ewc")), + ASTCtx); + auto *TOE = selectFirst("toe", MatchResult); + ASSERT_NE(TOE, nullptr); + auto *EWC = selectFirst("ewc", MatchResult); + ASSERT_NE(EWC, nullptr); + auto *BTE = selectFirst("bte", MatchResult); + ASSERT_NE(BTE, nullptr); + + RecordStorageLocation &Loc = Env.getResultObjectLocation(*TOE); + EXPECT_EQ(&Loc, &Env.getResultObjectLocation(*EWC)); + EXPECT_EQ(&Loc, &Env.getResultObjectLocation(*BTE)); + }); +} + TEST(TransferTest, StaticCast) { std::string Code = R"( void target(int Foo) { From c7cdf3cd5d748901e1370f1bfe803685ca658fb6 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Mon, 18 Dec 2023 16:10:01 +0800 Subject: [PATCH 085/884] [clang] Use 'starts_with' instead of 'startswith' in Gnu.cpp (NFC) llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp:1757:14: error: 'startswith' is deprecated: Use starts_with instead [-Werror,-Wdeprecated-declarations] 1757 | if (Flag.startswith("!march=") || Flag.startswith("-march=")) | ^~~~~~~~~~ | starts_with --- clang/lib/Driver/ToolChains/Gnu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index 02228e19dcf2e..7f463ddd17d3d 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -1754,7 +1754,7 @@ selectRISCVMultilib(const MultilibSet &RISCVMultilibSet, StringRef Arch, // Collect all flags except march=* for (StringRef Flag : Flags) { - if (Flag.startswith("!march=") || Flag.startswith("-march=")) + if (Flag.starts_with("!march=") || Flag.starts_with("-march=")) continue; NewFlags.push_back(Flag.str()); From 945c645acb9670b7b866a4abd94bcc9b0ae5d87d Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Mon, 18 Dec 2023 09:32:34 +0000 Subject: [PATCH 086/884] [AArch64][SME] Warn when using a streaming builtin from a non-streaming function (#75487) This PR adds a warning that's emitted when a non-streaming or non-streaming-compatible builtin is called in an unsuitable function. Uses work by Kerry McLaughlin. This is a re-upload of #74064 and fixes a compile time increase. --- clang/include/clang/Basic/CMakeLists.txt | 6 + clang/include/clang/Basic/arm_sve.td | 1164 ++++++++--------- clang/include/clang/Sema/Sema.h | 1 + clang/lib/Sema/SemaChecking.cpp | 51 +- .../aarch64-sme-intrinsics/acle_sme_add-i32.c | 16 +- .../aarch64-sme-intrinsics/acle_sme_add-i64.c | 16 +- .../acle_sme_mopa-za32.c | 14 +- .../acle_sme_mopa-za64.c | 10 +- .../acle_sme_mops-za32.c | 14 +- .../acle_sme_mops-za64.c | 10 +- .../aarch64-sme-intrinsics/acle_sme_read.c | 192 +-- .../aarch64-sme-intrinsics/acle_sme_write.c | 192 +-- .../Sema/aarch64-incompat-sm-builtin-calls.c | 77 ++ .../aarch64-sme-intrinsics/acle_sme_imm.cpp | 14 +- .../aarch64-sme-intrinsics/acle_sme_target.c | 9 +- clang/utils/TableGen/NeonEmitter.cpp | 28 + clang/utils/TableGen/SveEmitter.cpp | 55 + clang/utils/TableGen/TableGen.cpp | 12 + clang/utils/TableGen/TableGenBackends.h | 2 + 19 files changed, 1054 insertions(+), 829 deletions(-) diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt index 085e316fcc671..73fd521aeeec3 100644 --- a/clang/include/clang/Basic/CMakeLists.txt +++ b/clang/include/clang/Basic/CMakeLists.txt @@ -88,6 +88,9 @@ clang_tablegen(arm_sve_typeflags.inc -gen-arm-sve-typeflags clang_tablegen(arm_sve_sema_rangechecks.inc -gen-arm-sve-sema-rangechecks SOURCE arm_sve.td TARGET ClangARMSveSemaRangeChecks) +clang_tablegen(arm_sve_streaming_attrs.inc -gen-arm-sve-streaming-attrs + SOURCE arm_sve.td + TARGET ClangARMSveStreamingAttrs) clang_tablegen(arm_sme_builtins.inc -gen-arm-sme-builtins SOURCE arm_sme.td TARGET ClangARMSmeBuiltins) @@ -97,6 +100,9 @@ clang_tablegen(arm_sme_builtin_cg.inc -gen-arm-sme-builtin-codegen clang_tablegen(arm_sme_sema_rangechecks.inc -gen-arm-sme-sema-rangechecks SOURCE arm_sme.td TARGET ClangARMSmeSemaRangeChecks) +clang_tablegen(arm_sme_streaming_attrs.inc -gen-arm-sme-streaming-attrs + SOURCE arm_sme.td + TARGET ClangARMSmeStreamingAttrs) clang_tablegen(arm_cde_builtins.inc -gen-arm-cde-builtin-def SOURCE arm_cde.td TARGET ClangARMCdeBuiltinsDef) diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 55fd35c3b6c2d..98434c5c53e28 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -19,27 +19,27 @@ include "arm_sve_sme_incl.td" // Loads // Load one vector (scalar base) -def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; -def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">; -def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; +def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; let TargetGuard = "sve,bf16" in { - def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; - def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; + def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; + def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; } // Load one vector (scalar base, VL displacement) -def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">; -def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">; -def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">; -def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">; -def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ld1">; +def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_ld1">; +def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_ld1">; +def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; +def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_ld1">; // Load one vector (vector base) def SVLD1_GATHER_BASES_U : MInst<"svld1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; @@ -243,27 +243,27 @@ let TargetGuard = "sve,bf16" in { } // Load one vector, unextended load, non-temporal (scalar base) -def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; +def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; // Load one vector, unextended load, non-temporal (scalar base, VL displacement) -def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; +def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; let TargetGuard = "sve,bf16" in { - def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; - def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">; + def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; + def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_ldnt1">; } // Load one quadword and replicate (scalar base) -def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq">; +def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq">; + def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq", [IsStreamingCompatible]>; } multiclass StructLoad { - def : SInst; + def : SInst; let TargetGuard = "sve,bf16" in { - def: SInst; + def: SInst; } } @@ -286,16 +286,16 @@ let TargetGuard = "sve,f64mm,bf16" in { } let TargetGuard = "sve,bf16" in { - def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; - def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; - def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; - def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>; - def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>; - def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>; - def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>; - def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>; - def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; - def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, IsStreamingCompatible]>; + def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_3>]>; + def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; } let TargetGuard = "sve2p1" in { @@ -334,26 +334,26 @@ let TargetGuard = "sve2p1" in { // Stores // Store one vector (scalar base) -def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; -def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; -def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; +def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; // Store one vector (scalar base, VL displacement) -def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; -def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">; -def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">; -def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; -def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; +def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_st1">; +def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_st1">; +def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; +def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_st1">; let TargetGuard = "sve,bf16" in { - def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; - def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">; + def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; + def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_st1">; } // Store one vector (vector base) @@ -426,9 +426,9 @@ def SVST1H_SCATTER_INDEX_S : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "v def SVST1W_SCATTER_INDEX_S : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; multiclass StructStore { - def : SInst; + def : SInst; let TargetGuard = "sve,bf16" in { - def: SInst; + def: SInst; } } // Store N vectors into N-element structure (scalar base) @@ -442,14 +442,14 @@ defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">; defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">; // Store one vector, with no truncation, non-temporal (scalar base) -def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; +def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; // Store one vector, with no truncation, non-temporal (scalar base, VL displacement) -def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; +def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; let TargetGuard = "sve,bf16" in { - def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; - def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; + def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; + def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_stnt1">; } let TargetGuard = "sve2p1" in { @@ -488,16 +488,16 @@ let TargetGuard = "sve2p1" in { // Prefetches // Prefetch (Scalar base) -def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; -def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; -def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; -def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; +def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch, IsStreamingCompatible], MemEltTyInt64, "aarch64_sve_prf">; // Prefetch (Scalar base, VL displacement) -def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; -def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; -def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; -def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; +def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch, IsStreamingCompatible], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch, IsStreamingCompatible], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch, IsStreamingCompatible], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch, IsStreamingCompatible], MemEltTyInt64, "aarch64_sve_prf">; // Prefetch (Vector bases) def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; @@ -552,9 +552,9 @@ def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss", "iUif", MergeNone>; def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss", "lUld", MergeNone>; multiclass svdup_base { - def NAME : SInst; + def NAME : SInst; let TargetGuard = "sve,bf16" in { - def _BF16: SInst; + def _BF16: SInst; } } @@ -563,14 +563,14 @@ defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1, "aarch64_sve_du defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs", MergeAnyExp, "aarch64_sve_dup">; defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs", MergeZeroExp, "aarch64_sve_dup">; -def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index">; +def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index", [IsStreamingCompatible]>; // Integer arithmetic -multiclass SInstZPZ flags=[]> { - def _M : SInst; - def _X : SInst; - def _Z : SInst; +multiclass SInstZPZ { + def _M : SInst; + def _X : SInst; + def _Z : SInst; } defm SVABS : SInstZPZ<"svabs", "csil", "aarch64_sve_abs">; @@ -579,13 +579,13 @@ defm SVNEG : SInstZPZ<"svneg", "csil", "aarch64_sve_neg">; //------------------------------------------------------------------------------ multiclass SInstZPZZ flags=[]> { - def _M : SInst; - def _X : SInst; - def _Z : SInst; + def _M : SInst; + def _X : SInst; + def _Z : SInst; - def _N_M : SInst; - def _N_X : SInst; - def _N_Z : SInst; + def _N_M : SInst; + def _N_X : SInst; + def _N_Z : SInst; } defm SVABD_S : SInstZPZZ<"svabd", "csil", "aarch64_sve_sabd", "aarch64_sve_sabd_u">; @@ -617,26 +617,26 @@ multiclass SInstZPZZZ; } -defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp]>; -defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u">; -defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u">; -defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp]>; +defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp, IsStreamingCompatible]>; +defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u", [IsStreamingCompatible]>; +defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u", [IsStreamingCompatible]>; +defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp, IsStreamingCompatible]>; //------------------------------------------------------------------------------ -def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot">; -def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot">; -def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x">; -def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">; -def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x">; -def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">; +def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot", [IsStreamingCompatible]>; +def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot", [IsStreamingCompatible]>; +def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x", [IsStreamingCompatible]>; +def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [IsStreamingCompatible]>; +def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x", [IsStreamingCompatible]>; +def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [IsStreamingCompatible]>; -def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot">; -def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot">; -def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x">; -def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">; -def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x">; -def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">; +def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot", [IsStreamingCompatible]>; +def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot", [IsStreamingCompatible]>; +def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x", [IsStreamingCompatible]>; +def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [IsStreamingCompatible]>; +def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x", [IsStreamingCompatible]>; +def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [IsStreamingCompatible]>; def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_sdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]", "ddqqi", "UiUl", MergeNone, "aarch64_sve_udot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; @@ -656,107 +656,107 @@ defm SVNOT : SInstZPZ<"svnot", "csilUcUsUiUl", "aarch64_sve_not">; // Shifts multiclass SInst_SHIFT { - def _M : SInst; - def _X : SInst; - def _Z : SInst; + def _M : SInst; + def _X : SInst; + def _Z : SInst; - def _N_M : SInst; - def _N_X : SInst; - def _N_Z : SInst; + def _N_M : SInst; + def _N_X : SInst; + def _N_Z : SInst; - def _WIDE_M : SInst; - def _WIDE_X : SInst; - def _WIDE_Z : SInst; + def _WIDE_M : SInst; + def _WIDE_X : SInst; + def _WIDE_Z : SInst; - def _WIDE_N_M : SInst; - def _WIDE_N_X : SInst; - def _WIDE_N_Z : SInst; + def _WIDE_N_M : SInst; + def _WIDE_N_X : SInst; + def _WIDE_N_Z : SInst; } defm SVASR : SInst_SHIFT<"svasr", "aarch64_sve_asr", "csil", "csi">; defm SVLSL : SInst_SHIFT<"svlsl", "aarch64_sve_lsl", "csilUcUsUiUl", "csiUcUsUi">; defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">; -def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr">; +def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr">; + def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // Integer reductions -def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv">; -def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv">; -def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv">; -def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv">; -def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv">; -def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv">; -def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv">; -def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv">; -def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv">; +def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv", [IsStreamingCompatible]>; +def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv", [IsStreamingCompatible]>; +def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv", [IsStreamingCompatible]>; +def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv", [IsStreamingCompatible]>; +def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv", [IsStreamingCompatible]>; +def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv", [IsStreamingCompatible]>; +def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv", [IsStreamingCompatible]>; +def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv", [IsStreamingCompatible]>; +def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Integer comparisons -def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">; -def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">; -def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge">; -def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt">; -def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>; -def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>; -def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">; -def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">; -def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>; -def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>; - -def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">; -def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">; -def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge">; -def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt">; -def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>; -def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>; -def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">; -def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">; -def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>; -def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>; - -def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide">; -def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide">; -def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide">; -def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide">; -def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide">; -def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide">; -def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">; -def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">; -def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">; -def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">; - -def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide">; -def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide">; -def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide">; -def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide">; -def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide">; -def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide">; -def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">; -def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">; -def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">; -def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">; +def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [IsStreamingCompatible]>; +def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [IsStreamingCompatible]>; +def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [IsStreamingCompatible]>; +def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [IsStreamingCompatible]>; +def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [IsStreamingCompatible]>; +def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [IsStreamingCompatible]>; +def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, IsStreamingCompatible]>; + +def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [IsStreamingCompatible]>; +def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [IsStreamingCompatible]>; +def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [IsStreamingCompatible]>; +def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [IsStreamingCompatible]>; +def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [IsStreamingCompatible]>; +def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [IsStreamingCompatible]>; +def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, IsStreamingCompatible]>; + +def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [IsStreamingCompatible]>; +def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide", [IsStreamingCompatible]>; +def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide", [IsStreamingCompatible]>; +def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [IsStreamingCompatible]>; +def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide", [IsStreamingCompatible]>; +def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide", [IsStreamingCompatible]>; +def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [IsStreamingCompatible]>; +def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [IsStreamingCompatible]>; +def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [IsStreamingCompatible]>; +def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [IsStreamingCompatible]>; + +def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [IsStreamingCompatible]>; +def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide", [IsStreamingCompatible]>; +def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide", [IsStreamingCompatible]>; +def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [IsStreamingCompatible]>; +def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide", [IsStreamingCompatible]>; +def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide", [IsStreamingCompatible]>; +def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [IsStreamingCompatible]>; +def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [IsStreamingCompatible]>; +def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [IsStreamingCompatible]>; +def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // While comparisons -def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>; -def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>; -def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>; -def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>; -def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>; -def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>; -def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>; -def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>; +def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Counting bit @@ -767,12 +767,12 @@ multiclass SInstCLS def _Z : SInst; } -defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls">; -defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz">; -defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt">; +defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls", [IsStreamingCompatible]>; +defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz", [IsStreamingCompatible]>; +defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt">; + defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -827,13 +827,13 @@ def SVTMAD : SInst<"svtmad[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_ftma def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftsmul_x">; def SVTSSEL : SInst<"svtssel[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftssel_x">; -def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale">; -def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale">; -def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale">; +def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale", [IsStreamingCompatible]>; -def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale">; -def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale">; -def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale">; +def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale", [IsStreamingCompatible]>; +def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale", [IsStreamingCompatible]>; defm SVMAD_F : SInstZPZZZ<"svmad", "hfd", "aarch64_sve_fmad", "aarch64_sve_fmla_u", [ReverseMergeAnyAccOp]>; defm SVMLA_F : SInstZPZZZ<"svmla", "hfd", "aarch64_sve_fmla", "aarch64_sve_fmla_u">; @@ -844,42 +844,42 @@ defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fn defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u">; defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [ReverseMergeAnyAccOp]>; -def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>; -def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>; +def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; +def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x">; -def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x">; -def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x">; -def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x">; +def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x", [IsStreamingCompatible]>; +def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x", [IsStreamingCompatible]>; +def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x", [IsStreamingCompatible]>; +def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point reductions -def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda">; -def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv">; -def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv">; -def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv">; -def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv">; -def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv">; +def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda", [IsStreamingCompatible]>; +def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv", [IsStreamingCompatible]>; +def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv", [IsStreamingCompatible]>; +def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv", [IsStreamingCompatible]>; +def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv", [IsStreamingCompatible]>; +def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point comparisons -def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge">; -def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt">; -def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare]>; -def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>; -def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo">; +def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [IsStreamingCompatible]>; +def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [IsStreamingCompatible]>; +def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, IsStreamingCompatible]>; +def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo", [IsStreamingCompatible]>; def SVACGE_N : SInst<"svacge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge">; def SVACGT_N : SInst<"svacgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt">; @@ -887,19 +887,19 @@ def SVACLE_N : SInst<"svacle[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_ def SVACLT_N : SInst<"svaclt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>; def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo">; -def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq">; -def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne">; -def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge">; -def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt">; -def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>; -def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>; +def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq", [IsStreamingCompatible]>; +def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne", [IsStreamingCompatible]>; +def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [IsStreamingCompatible]>; +def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [IsStreamingCompatible]>; +def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, IsStreamingCompatible]>; -def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq">; -def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne">; -def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge">; -def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt">; -def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>; -def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>; +def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq", [IsStreamingCompatible]>; +def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne", [IsStreamingCompatible]>; +def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [IsStreamingCompatible]>; +def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [IsStreamingCompatible]>; +def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, IsStreamingCompatible]>; +def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Floating-point conversions @@ -907,16 +907,16 @@ def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sv multiclass SInstCvtMXZ< string name, string m_types, string xz_types, string types, string intrinsic, list flags = [IsOverloadNone]> { - def _M : SInst; - def _X : SInst; - def _Z : SInst; + def _M : SInst; + def _X : SInst; + def _Z : SInst; } multiclass SInstCvtMX flags = [IsOverloadNone]> { - def _M : SInst; - def _X : SInst; + def _M : SInst; + def _X : SInst; } // svcvt_s##_f16 @@ -930,7 +930,7 @@ defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aar let TargetGuard = "sve,bf16" in { defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvt_bf16f32">; - def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone]>; + def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone, IsStreamingCompatible]>; } // svcvt_s##_f64 @@ -994,11 +994,11 @@ defm SVCVTLT_F64 : SInstCvtMX<"svcvtlt_f64[_f32]", "ddPh", "dPh", "d", "aarc defm SVCVTX_F32 : SInstCvtMXZ<"svcvtx_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">; -def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone]>; -def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone]>; +def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, IsStreamingCompatible]>; +def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, IsStreamingCompatible]>; // SVCVTNT_X : Implemented as macro by SveEmitter.cpp -def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone]>; +def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, IsStreamingCompatible]>; // SVCVTXNT_X_F32 : Implemented as macro by SveEmitter.cpp } @@ -1007,9 +1007,9 @@ def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch6 // Permutations and selection multiclass SVEPerm { - def : SInst; + def : SInst; let TargetGuard = "sve,bf16" in { - def: SInst; + def: SInst; } } @@ -1033,81 +1033,81 @@ def SVDUPQ_LANE : SInst<"svdupq_lane[_{d}]", "ddn", "csilUcUsUiUlhfd", MergeNo let TargetGuard = "sve,bf16" in { def SVDUPQ_LANE_BF16 : SInst<"svdupq_lane[_{d}]", "ddn", "b", MergeNone, "aarch64_sve_dupq_lane">; } -def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>; +def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [IsStreamingCompatible], [ImmCheck<2, ImmCheckExtract, 1>]>; defm SVLASTA : SVEPerm<"svlasta[_{d}]", "sPd", "aarch64_sve_lasta">; defm SVLASTB : SVEPerm<"svlastb[_{d}]", "sPd", "aarch64_sve_lastb">; -def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev">; -def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel">; -def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice">; -def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">; +def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice", [IsStreamingCompatible]>; +def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { - def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl">; + def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>; } -def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1">; -def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2">; -def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi">; -def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi">; -def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo">; -def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo">; -def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1">; -def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2">; -def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1">; -def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">; +def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi", [IsStreamingCompatible]>; +def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi", [IsStreamingCompatible]>; +def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo", [IsStreamingCompatible]>; +def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo", [IsStreamingCompatible]>; +def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { -def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>; -def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev">; -def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel">; -def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice">; -def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1">; -def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2">; -def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1">; -def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2">; -def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1">; -def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2">; -} - -def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev">; -def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone]>; -def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone]>; -def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone]>; -def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel">; -def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1">; -def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone]>; -def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone]>; -def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone]>; -def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2">; -def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone]>; -def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone]>; -def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone]>; -def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi">; -def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo">; -def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1">; -def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone]>; -def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone]>; -def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone]>; -def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2">; -def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone]>; -def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone]>; -def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone]>; -def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1">; -def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone]>; -def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone]>; -def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone]>; -def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2">; -def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone]>; -def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone]>; -def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone]>; +def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [IsStreamingCompatible], [ImmCheck<2, ImmCheckExtract, 1>]>; +def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice", [IsStreamingCompatible]>; +def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; +} + +def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev", [IsStreamingCompatible]>; +def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel", [IsStreamingCompatible]>; +def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1", [IsStreamingCompatible]>; +def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2", [IsStreamingCompatible]>; +def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi", [IsStreamingCompatible]>; +def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo", [IsStreamingCompatible]>; +def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1", [IsStreamingCompatible]>; +def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2", [IsStreamingCompatible]>; +def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1", [IsStreamingCompatible]>; +def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2", [IsStreamingCompatible]>; +def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone, IsStreamingCompatible]>; +def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone, IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Predicate creation -def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone]>; +def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; -def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue">; -def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL]>; +def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsStreamingCompatible]>; +def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL, IsStreamingCompatible]>; def SVDUPQ_B8 : SInst<"svdupq[_n]_{d}", "Pssssssssssssssss", "Pc", MergeNone>; def SVDUPQ_B16 : SInst<"svdupq[_n]_{d}", "Pssssssss", "Ps", MergeNone>; @@ -1119,33 +1119,33 @@ def SVDUP_N_B : SInst<"svdup[_n]_{d}", "Ps", "PcPsPiPl", MergeNone>; //////////////////////////////////////////////////////////////////////////////// // Predicate operations -def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z">; -def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z">; -def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z">; -def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion -def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z">; -def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z">; -def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion -def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z">; -def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z">; - -def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka">; -def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z">; -def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb">; -def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z">; -def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z">; -def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z">; -def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z">; - -def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst">; -def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext">; +def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z", [IsStreamingCompatible]>; +def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z", [IsStreamingCompatible]>; +def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z", [IsStreamingCompatible]>; +def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone, "", [IsStreamingCompatible]>; // Uses custom expansion +def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z", [IsStreamingCompatible]>; +def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z", [IsStreamingCompatible]>; +def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone, "", [IsStreamingCompatible]>; // Uses custom expansion +def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z", [IsStreamingCompatible]>; +def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z", [IsStreamingCompatible]>; + +def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka", [IsStreamingCompatible]>; +def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z", [IsStreamingCompatible]>; +def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb", [IsStreamingCompatible]>; +def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z", [IsStreamingCompatible]>; +def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z", [IsStreamingCompatible]>; +def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z", [IsStreamingCompatible]>; +def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z", [IsStreamingCompatible]>; + +def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst", [IsStreamingCompatible]>; +def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // Testing predicates -def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any">; -def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first">; -def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last">; +def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any", [IsStreamingCompatible]>; +def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first", [IsStreamingCompatible]>; +def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last", [IsStreamingCompatible]>; //////////////////////////////////////////////////////////////////////////////// // FFR manipulation @@ -1158,21 +1158,21 @@ def SVWRFFR : SInst<"svwrffr", "vP", "Pc", MergeNone, "", [IsOverloadNone]>; //////////////////////////////////////////////////////////////////////////////// // Counting elements -def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone]>; -def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone]>; -def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone]>; -def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone]>; +def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone, IsStreamingCompatible]>; +def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone, IsStreamingCompatible]>; -def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone]>; -def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone]>; +def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; +def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone, IsStreamingCompatible]>; -def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp">; -def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone>; +def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp", [IsStreamingCompatible]>; +def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone, "", [IsStreamingCompatible]>; let TargetGuard = "sve,bf16" in { -def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone>; +def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone, "", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1189,20 +1189,20 @@ def UnsignedWord : sat_type<"U", "Ui">; def UnsignedDoubleWord : sat_type<"U", "Ul">; multiclass SInst_SAT1 { - def _N32 : SInst]>; - def _N64 : SInst]>; - def _N32_ALL : SInst]>; - def _N64_ALL : SInst]>; + def _N32 : SInst]>; + def _N64 : SInst]>; + def _N32_ALL : SInst]>; + def _N64_ALL : SInst]>; } multiclass SInst_SAT2 { - def "" : SInst]>; - def _ALL : SInst]>; + def "" : SInst]>; + def _ALL : SInst]>; - def _N32 : SInst]>; - def _N64 : SInst]>; - def _N32_ALL : SInst]>; - def _N64_ALL : SInst]>; + def _N32 : SInst]>; + def _N64 : SInst]>; + def _N32_ALL : SInst]>; + def _N64_ALL : SInst]>; } defm SVQDECB_S : SInst_SAT1<"svqdecb", "aarch64_sve_sqdecb", SignedByte>; @@ -1223,32 +1223,32 @@ defm SVQINCW_U : SInst_SAT2<"svqincw", "aarch64_sve_uqincw", UnsignedWord>; defm SVQINCD_S : SInst_SAT2<"svqincd", "aarch64_sve_sqincd", SignedDoubleWord>; defm SVQINCD_U : SInst_SAT2<"svqincd", "aarch64_sve_uqincd", UnsignedDoubleWord>; -def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp">; -def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp">; -def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp">; -def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp">; +def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp", [IsStreamingCompatible]>; +def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp", [IsStreamingCompatible]>; +def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp", [IsStreamingCompatible]>; +def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp", [IsStreamingCompatible]>; -def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32">; -def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64">; -def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32">; -def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64">; -def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32">; -def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64">; -def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32">; -def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64">; +def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32", [IsStreamingCompatible]>; +def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64", [IsStreamingCompatible]>; +def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32", [IsStreamingCompatible]>; +def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64", [IsStreamingCompatible]>; +def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32", [IsStreamingCompatible]>; +def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64", [IsStreamingCompatible]>; +def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32", [IsStreamingCompatible]>; +def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64", [IsStreamingCompatible]>; let TargetGuard = "sve,i8mm" in { def SVMLLA_S32 : SInst<"svmmla[_s32]", "ddqq","i", MergeNone, "aarch64_sve_smmla">; def SVMLLA_U32 : SInst<"svmmla[_u32]", "ddqq","Ui", MergeNone, "aarch64_sve_ummla">; def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i", MergeNone, "aarch64_sve_usmmla">; -def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot">; -def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot">; -def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>; -def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>; +def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot", [IsStreamingCompatible]>; +def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot", [IsStreamingCompatible]>; +def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, IsStreamingCompatible]>; +def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, IsStreamingCompatible]>; -def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; -def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; +def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; +def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; } let TargetGuard = "sve,f32mm" in { @@ -1257,12 +1257,12 @@ def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla let TargetGuard = "sve,f64mm" in { def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">; -def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q">; -def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q">; -def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q">; -def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q">; -def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q">; -def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">; +def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q", [IsStreamingCompatible]>; +def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q", [IsStreamingCompatible]>; +def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q", [IsStreamingCompatible]>; +def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q", [IsStreamingCompatible]>; +def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q", [IsStreamingCompatible]>; +def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q", [IsStreamingCompatible]>; } let TargetGuard = "sve,bf16,f64mm" in { @@ -1276,20 +1276,20 @@ def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarc //////////////////////////////////////////////////////////////////////////////// // Vector creation -def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; -def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; +def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>; def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>; let TargetGuard = "sve,bf16" in { -def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef]>; -def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef]>; +def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; +def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>; def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd", "b", MergeNone, "", [IsTupleCreate]>; def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsTupleCreate]>; @@ -1331,14 +1331,14 @@ let TargetGuard = "sve2p1" in { //////////////////////////////////////////////////////////////////////////////// // SVE2 WhileGE/GT let TargetGuard = "sve2" in { -def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>; -def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>; -def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>; -def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>; -def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>; -def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>; -def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>; -def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>; +def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>; +def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1380,49 +1380,49 @@ multiclass SInstZPZxZ; -defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl">; -defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl">; -defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl">; -defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl">; -defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl">; -defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd">; -defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd">; - -def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba">; -def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">; -def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh">; -def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh">; -def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah">; -def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh">; - -def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba">; -def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">; -def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh">; -def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh">; -def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah">; -def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh">; - -def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; - -def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; -def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; -def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl", [IsStreamingCompatible]>; +defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl", [IsStreamingCompatible]>; +defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl", [IsStreamingCompatible]>; +defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl", [IsStreamingCompatible]>; +defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl", [IsStreamingCompatible]>; +defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl", [IsStreamingCompatible]>; +defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd", [IsStreamingCompatible]>; +defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd", [IsStreamingCompatible]>; + +def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba", [IsStreamingCompatible]>; +def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [IsStreamingCompatible]>; +def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh", [IsStreamingCompatible]>; +def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh", [IsStreamingCompatible]>; +def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah", [IsStreamingCompatible]>; +def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [IsStreamingCompatible]>; + +def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba", [IsStreamingCompatible]>; +def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [IsStreamingCompatible]>; +def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh", [IsStreamingCompatible]>; +def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh", [IsStreamingCompatible]>; +def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah", [IsStreamingCompatible]>; +def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [IsStreamingCompatible]>; + +def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; + +def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; +def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1434,29 +1434,29 @@ multiclass SInstPairwise; -defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp">; -defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp">; -defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp">; -defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp">; -defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp">; -defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp">; -defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp">; -defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp">; -defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp">; +defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp", [IsStreamingCompatible]>; +defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp", [IsStreamingCompatible]>; +defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp", [IsStreamingCompatible]>; +defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp", [IsStreamingCompatible]>; +defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp", [IsStreamingCompatible]>; +defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp", [IsStreamingCompatible]>; +defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp", [IsStreamingCompatible]>; +defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp", [IsStreamingCompatible]>; +defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp", [IsStreamingCompatible]>; +defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Widening pairwise arithmetic let TargetGuard = "sve2" in { -def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp">; -def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp">; -def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp">; +def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp", [IsStreamingCompatible]>; +def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp", [IsStreamingCompatible]>; +def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp", [IsStreamingCompatible]>; -def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp">; -def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp">; -def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp">; +def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp", [IsStreamingCompatible]>; +def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp", [IsStreamingCompatible]>; +def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1464,56 +1464,56 @@ def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_s // let TargetGuard = "sve2" in { -def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">; -def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">; -def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">; -def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">; -def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">; -def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">; - -def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">; -def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">; -def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">; -def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">; -def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">; -def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">; -def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>; +def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [IsStreamingCompatible]>; +def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [IsStreamingCompatible]>; +def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [IsStreamingCompatible]>; +def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [IsStreamingCompatible]>; +def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [IsStreamingCompatible]>; +def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [IsStreamingCompatible]>; + +def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [IsStreamingCompatible]>; +def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [IsStreamingCompatible]>; +def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [IsStreamingCompatible]>; +def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [IsStreamingCompatible]>; +def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [IsStreamingCompatible]>; +def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [IsStreamingCompatible]>; +def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRight, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Large integer arithmetic let TargetGuard = "sve2" in { -def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb">; -def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt">; -def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb">; -def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt">; +def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb", [IsStreamingCompatible]>; +def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt", [IsStreamingCompatible]>; +def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb", [IsStreamingCompatible]>; +def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt", [IsStreamingCompatible]>; -def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb">; -def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt">; -def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb">; -def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt">; +def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb", [IsStreamingCompatible]>; +def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt", [IsStreamingCompatible]>; +def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb", [IsStreamingCompatible]>; +def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Multiplication by indexed elements let TargetGuard = "sve2" in { -def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Uniform complex integer arithmetic let TargetGuard = "sve2" in { -def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>; -def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>; -def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [IsStreamingCompatible], [ImmCheck<2, ImmCheckComplexRot90_270>]>; +def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [IsStreamingCompatible], [ImmCheck<2, ImmCheckComplexRot90_270>]>; +def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; -def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, +def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, ImmCheck<4, ImmCheckComplexRotAll90>]>; } @@ -1521,18 +1521,18 @@ def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", // SVE2 - Widening DSP operations multiclass SInstWideDSPAcc { - def : SInst; - def _N : SInst; + def : SInst; + def _N : SInst; } multiclass SInstWideDSPLong { - def : SInst; - def _N : SInst; + def : SInst; + def _N : SInst; } multiclass SInstWideDSPWide { - def : SInst; - def _N : SInst; + def : SInst; + def _N : SInst; } let TargetGuard = "sve2" in { @@ -1581,87 +1581,87 @@ defm SVSUBWB_U : SInstWideDSPWide<"svsubwb", "UsUiUl", "aarch64_sve_usubwb">; defm SVSUBWT_S : SInstWideDSPWide<"svsubwt", "sil", "aarch64_sve_ssubwt">; defm SVSUBWT_U : SInstWideDSPWide<"svsubwt", "UsUiUl", "aarch64_sve_usubwt">; -def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; -def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; - -def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone>; -def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone>; -def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone>; -def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone>; - -def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; -def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; +def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; + +def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone, "", [IsStreamingCompatible]>; +def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone, "", [IsStreamingCompatible]>; + +def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; +def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [IsStreamingCompatible], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Narrowing DSP operations let TargetGuard = "sve2" in { -def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">; -def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">; -def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">; -def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">; -def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">; -def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">; -def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">; -def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">; - -def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">; -def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">; -def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">; -def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">; -def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">; -def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">; -def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">; -def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">; - -def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; -def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; - -def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; -def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [IsStreamingCompatible]>; +def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [IsStreamingCompatible]>; +def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [IsStreamingCompatible]>; +def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [IsStreamingCompatible]>; +def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [IsStreamingCompatible]>; +def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [IsStreamingCompatible]>; +def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [IsStreamingCompatible]>; +def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [IsStreamingCompatible]>; + +def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [IsStreamingCompatible]>; +def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [IsStreamingCompatible]>; +def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [IsStreamingCompatible]>; +def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [IsStreamingCompatible]>; +def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [IsStreamingCompatible]>; +def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [IsStreamingCompatible]>; +def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [IsStreamingCompatible]>; +def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [IsStreamingCompatible]>; + +def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; +def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [IsStreamingCompatible], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; + +def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; +def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [IsStreamingCompatible], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Unary narrowing operations let TargetGuard = "sve2" in { -def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb">; -def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb">; -def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb">; +def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb", [IsStreamingCompatible]>; +def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb", [IsStreamingCompatible]>; +def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb", [IsStreamingCompatible]>; -def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt">; -def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt">; -def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt">; +def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt", [IsStreamingCompatible]>; +def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt", [IsStreamingCompatible]>; +def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1802,18 +1802,18 @@ def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", " // SVE2 - Polynomial arithmetic let TargetGuard = "sve2" in { -def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">; -def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">; -def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">; -def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">; -def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul">; -def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul">; -def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone>; -def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone>; -def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">; -def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">; -def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone>; -def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone>; +def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [IsStreamingCompatible]>; +def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [IsStreamingCompatible]>; +def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [IsStreamingCompatible]>; +def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [IsStreamingCompatible]>; +def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul", [IsStreamingCompatible]>; +def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul", [IsStreamingCompatible]>; +def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [IsStreamingCompatible]>; +def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [IsStreamingCompatible]>; +def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone, "", [IsStreamingCompatible]>; +def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone, "", [IsStreamingCompatible]>; def SVPMULLT_PAIR : SInst<"svpmullt_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">; def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">; } @@ -1822,8 +1822,8 @@ def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", Mer // SVE2 - Complex integer dot product let TargetGuard = "sve2" in { -def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>; -def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [], [ImmCheck<4, ImmCheckComplexRotAll90>, +def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRotAll90>]>; +def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [IsStreamingCompatible], [ImmCheck<4, ImmCheckComplexRotAll90>, ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; } @@ -1831,27 +1831,27 @@ def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch // SVE2 - Floating-point widening multiply-accumulate let TargetGuard = "sve2" in { -def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb">; -def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb">; -def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt">; -def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt">; -def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb">; -def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb">; -def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; -def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt">; -def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt">; -def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb", [IsStreamingCompatible]>; +def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb", [IsStreamingCompatible]>; +def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt", [IsStreamingCompatible]>; +def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt", [IsStreamingCompatible]>; +def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb", [IsStreamingCompatible]>; +def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb", [IsStreamingCompatible]>; +def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; +def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt", [IsStreamingCompatible]>; +def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt", [IsStreamingCompatible]>; +def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Floating-point integer binary logarithm let TargetGuard = "sve2" in { -def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb">; -def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb">; -def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb">; +def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb", [IsStreamingCompatible]>; +def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb", [IsStreamingCompatible]>; +def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1873,32 +1873,32 @@ def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve //////////////////////////////////////////////////////////////////////////////// // SVE2 - Contiguous conflict detection let TargetGuard = "sve2" in { -def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW]>; -def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>; -def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW]>; -def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW]>; +def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW, IsStreamingCompatible]>; -def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW]>; -def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>; -def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW]>; -def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW]>; +def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW, IsStreamingCompatible]>; } let TargetGuard = "sve2,bf16" in { -def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>; -def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>; +def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, IsStreamingCompatible]>; +def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// // SVE2 - Extended table lookup/permute let TargetGuard = "sve2" in { -def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone>; -def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx">; +def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone, "", [IsStreamingCompatible]>; +def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx", [IsStreamingCompatible]>; } let TargetGuard = "sve2,bf16" in { -def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone>; -def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx">; +def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone, "", [IsStreamingCompatible]>; +def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx", [IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 20228da15ade8..a4f8fc1845b1c 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -13851,6 +13851,7 @@ class Sema final { bool CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool ParseSVEImmChecks(CallExpr *TheCall, SmallVector, 3> &ImmChecks); + bool CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckCDEBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, CallExpr *TheCall); bool CheckARMCoprocessorImmediate(const TargetInfo &TI, const Expr *CoprocArg, diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index d4a40b850ceaf..49be88051e654 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -3156,7 +3156,6 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, const FunctionDecl *FD, ArmStreamingType BuiltinType) { ArmStreamingType FnType = getArmStreamingFnType(FD); - if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) { S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) << TheCall->getSourceRange() << "streaming"; @@ -3168,9 +3167,53 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, << TheCall->getSourceRange() << "streaming compatible"; return; } + + if (FnType == ArmNonStreaming && BuiltinType == ArmStreaming) { + S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) + << TheCall->getSourceRange() << "non-streaming"; + } +} + +bool Sema::CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { + if (const FunctionDecl *FD = getCurFunctionDecl()) { + std::optional BuiltinType; + + switch (BuiltinID) { +#define GET_SME_STREAMING_ATTRS +#include "clang/Basic/arm_sme_streaming_attrs.inc" +#undef GET_SME_STREAMING_ATTRS + } + + if (BuiltinType) + checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType); + } + + // Range check SME intrinsics that take immediate values. + SmallVector, 3> ImmChecks; + + switch (BuiltinID) { + default: + return false; +#define GET_SME_IMMEDIATE_CHECK +#include "clang/Basic/arm_sme_sema_rangechecks.inc" +#undef GET_SME_IMMEDIATE_CHECK + } + + return ParseSVEImmChecks(TheCall, ImmChecks); } bool Sema::CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { + if (const FunctionDecl *FD = getCurFunctionDecl()) { + std::optional BuiltinType; + + switch (BuiltinID) { +#define GET_SVE_STREAMING_ATTRS +#include "clang/Basic/arm_sve_streaming_attrs.inc" +#undef GET_SVE_STREAMING_ATTRS + } + if (BuiltinType) + checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType); + } // Range check SVE intrinsics that take immediate values. SmallVector, 3> ImmChecks; @@ -3180,9 +3223,6 @@ bool Sema::CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { #define GET_SVE_IMMEDIATE_CHECK #include "clang/Basic/arm_sve_sema_rangechecks.inc" #undef GET_SVE_IMMEDIATE_CHECK -#define GET_SME_IMMEDIATE_CHECK -#include "clang/Basic/arm_sme_sema_rangechecks.inc" -#undef GET_SME_IMMEDIATE_CHECK } return ParseSVEImmChecks(TheCall, ImmChecks); @@ -3569,6 +3609,9 @@ bool Sema::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, if (CheckSVEBuiltinFunctionCall(BuiltinID, TheCall)) return true; + if (CheckSMEBuiltinFunctionCall(BuiltinID, TheCall)) + return true; + // For intrinsics which take an immediate value as part of the instruction, // range check them here. unsigned i = 0, l = 0, u = 0; diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c index e444321be41b2..ee6c1c9dd566b 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) { +void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za32, _u32, _m)(0, pn, pm, zn); } @@ -50,7 +50,7 @@ void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) { +void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za32, _u32, _m)(3, pn, pm, zn); } @@ -70,7 +70,7 @@ void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) { +void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za32, _s32, _m)(0, pn, pm, zn); } @@ -90,7 +90,7 @@ void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) { +void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za32, _s32, _m)(3, pn, pm, zn); } @@ -110,7 +110,7 @@ void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) { +void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za32, _u32, _m)(0, pn, pm, zn); } @@ -130,7 +130,7 @@ void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) { +void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za32, _u32, _m)(3, pn, pm, zn); } @@ -150,7 +150,7 @@ void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) { +void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za32, _s32, _m)(0, pn, pm, zn); } @@ -170,7 +170,7 @@ void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) { +void test_svaddva_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za32, _s32, _m)(3, pn, pm, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c index 4b2f71d607eb6..254ea89d22c50 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) { +void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za64, _u64, _m)(0, pn, pm, zn); } @@ -50,7 +50,7 @@ void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) { +void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za64, _u64, _m)(7, pn, pm, zn); } @@ -70,7 +70,7 @@ void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) { +void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za64, _s64, _m)(0, pn, pm, zn); } @@ -90,7 +90,7 @@ void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) { +void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddha_za64, _s64, _m)(7, pn, pm, zn); } @@ -110,7 +110,7 @@ void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) { +void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za64, _u64, _m)(0, pn, pm, zn); } @@ -130,7 +130,7 @@ void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) { +void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za64, _u64, _m)(7, pn, pm, zn); } @@ -150,7 +150,7 @@ void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) { +void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za64, _s64, _m)(0, pn, pm, zn); } @@ -170,7 +170,7 @@ void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) { +void test_svaddva_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svaddva_za64, _s64, _m)(7, pn, pm, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c index 88206eec76b97..b90c9be4a6e09 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) { +void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -42,7 +42,7 @@ void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) { +void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za32, _u8, _m)(0, pn, pm, zn, zm); } @@ -62,7 +62,7 @@ void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.wide.nxv8bf16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) { +void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za32, _bf16, _m)(0, pn, pm, zn, zm); } @@ -82,7 +82,7 @@ void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.wide.nxv8f16(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) { +void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za32, _f16, _m)(1, pn, pm, zn, zm); } @@ -102,7 +102,7 @@ void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) { +void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za32, _f32, _m)(1, pn, pm, zn, zm); } @@ -118,7 +118,7 @@ void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) { +void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming { SME_ACLE_FUNC(svsumopa_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -134,7 +134,7 @@ void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) { +void test_svusmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming { SME_ACLE_FUNC(svusmopa_za32, _u8, _m)(0, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c index 1a1ab1e00d0ec..a56ce4d17f126 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smopa.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) { +void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za64, _s16, _m)(7, pn, pm, zn, zm); } @@ -50,7 +50,7 @@ void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umopa.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) { +void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za64, _u16, _m)(0, pn, pm, zn, zm); } @@ -70,7 +70,7 @@ void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.nxv2f64(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) { +void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming { SME_ACLE_FUNC(svmopa_za64, _f64, _m)(7, pn, pm, zn, zm); } @@ -90,7 +90,7 @@ void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumopa.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) { +void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming { SME_ACLE_FUNC(svsumopa_za64, _s16, _m)(0, pn, pm, zn, zm); } @@ -110,7 +110,7 @@ void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmopa.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) { +void test_svusmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming { SME_ACLE_FUNC(svusmopa_za64, _u16, _m)(7, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c index 1633cc6ed32e0..abcf4c2e698d7 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) { +void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -42,7 +42,7 @@ void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) { +void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za32, _u8, _m)(0, pn, pm, zn, zm); } @@ -62,7 +62,7 @@ void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.wide.nxv8bf16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) { +void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za32, _bf16, _m)(0, pn, pm, zn, zm); } @@ -82,7 +82,7 @@ void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.wide.nxv8f16(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) { +void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za32, _f16, _m)(1, pn, pm, zn, zm); } @@ -102,7 +102,7 @@ void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.nxv4f32(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) { +void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za32, _f32, _m)(1, pn, pm, zn, zm); } @@ -118,7 +118,7 @@ void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) { +void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming { SME_ACLE_FUNC(svsumops_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -134,7 +134,7 @@ void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) { +void test_svusmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming { SME_ACLE_FUNC(svusmops_za32, _u8, _m)(0, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c index b17df30c0f851..b26b9e4e51e05 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smops.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) { +void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za64, _s16, _m)(7, pn, pm, zn, zm); } @@ -50,7 +50,7 @@ void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umops.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) { +void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za64, _u16, _m)(0, pn, pm, zn, zm); } @@ -70,7 +70,7 @@ void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.nxv2f64(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) { +void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming { SME_ACLE_FUNC(svmops_za64, _f64, _m)(7, pn, pm, zn, zm); } @@ -90,7 +90,7 @@ void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumops.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) { +void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming { SME_ACLE_FUNC(svsumops_za64, _s16, _m)(0, pn, pm, zn, zm); } @@ -110,7 +110,7 @@ void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmops.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) { +void test_svusmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming { SME_ACLE_FUNC(svusmops_za64, _u16, _m)(7, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c index d63900129f994..a15599d186a87 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice_base); } @@ -44,7 +44,7 @@ svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) { // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice); } @@ -63,7 +63,7 @@ svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 0, slice_base); } @@ -83,7 +83,7 @@ svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 1, slice); } @@ -102,7 +102,7 @@ svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 0, slice_base); } @@ -122,7 +122,7 @@ svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 3, slice); } @@ -141,7 +141,7 @@ svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 0, slice_base); } @@ -161,7 +161,7 @@ svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 7, slice); } @@ -178,7 +178,7 @@ svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice_base); } @@ -196,7 +196,7 @@ svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice); } @@ -215,7 +215,7 @@ svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 0, slice_base); } @@ -235,7 +235,7 @@ svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 1, slice); } @@ -254,7 +254,7 @@ svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 0, slice_base); } @@ -274,7 +274,7 @@ svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 3, slice); } @@ -293,7 +293,7 @@ svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 0, slice_base); } @@ -313,7 +313,7 @@ svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 7, slice); } @@ -332,7 +332,7 @@ svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 0, slice_base); } @@ -352,7 +352,7 @@ svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 1, slice); } @@ -371,7 +371,7 @@ svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 0, slice_base); } @@ -391,7 +391,7 @@ svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 1, slice); } @@ -410,7 +410,7 @@ svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 0, slice_base); } @@ -430,7 +430,7 @@ svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 3, slice); } @@ -449,7 +449,7 @@ svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 0, slice_base); } @@ -469,7 +469,7 @@ svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 7, slice); } @@ -486,7 +486,7 @@ svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 0, slice_base); } @@ -502,7 +502,7 @@ svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 15, slice_base); } @@ -520,7 +520,7 @@ svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 0, slice_base); } @@ -538,7 +538,7 @@ svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 15, slice_base); } @@ -556,7 +556,7 @@ svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 0, slice_base); } @@ -574,7 +574,7 @@ svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 15, slice_base); } @@ -592,7 +592,7 @@ svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 0, slice_base); } @@ -610,7 +610,7 @@ svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 15, slice_base); } @@ -626,7 +626,7 @@ svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 0, slice_base); } @@ -642,7 +642,7 @@ svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 15, slice_base); } @@ -660,7 +660,7 @@ svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 0, slice_base); } @@ -678,7 +678,7 @@ svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 15, slice_base); } @@ -696,7 +696,7 @@ svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 0, slice_base); } @@ -714,7 +714,7 @@ svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 15, slice_base); } @@ -732,7 +732,7 @@ svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 0, slice_base); } @@ -750,7 +750,7 @@ svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 15, slice_base); } @@ -768,7 +768,7 @@ svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 0, slice_base); } @@ -786,7 +786,7 @@ svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 15, slice_base); } @@ -804,7 +804,7 @@ svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 0, slice_base); } @@ -822,7 +822,7 @@ svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t s // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 15, slice_base); } @@ -840,7 +840,7 @@ svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 0, slice_base); } @@ -858,7 +858,7 @@ svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 15, slice_base); } @@ -876,7 +876,7 @@ svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 0, slice_base); } @@ -894,7 +894,7 @@ svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 15, slice_base); } @@ -910,7 +910,7 @@ svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice_base); } @@ -928,7 +928,7 @@ svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice); } @@ -947,7 +947,7 @@ svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 0, slice_base); } @@ -967,7 +967,7 @@ svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 1, slice); } @@ -986,7 +986,7 @@ svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 0, slice_base); } @@ -1006,7 +1006,7 @@ svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 3, slice); } @@ -1025,7 +1025,7 @@ svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 0, slice_base); } @@ -1045,7 +1045,7 @@ svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 7, slice); } @@ -1062,7 +1062,7 @@ svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice_base); } @@ -1080,7 +1080,7 @@ svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice); } @@ -1099,7 +1099,7 @@ svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 0, slice_base); } @@ -1119,7 +1119,7 @@ svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 1, slice); } @@ -1138,7 +1138,7 @@ svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 0, slice_base); } @@ -1158,7 +1158,7 @@ svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 3, slice); } @@ -1177,7 +1177,7 @@ svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 0, slice_base); } @@ -1197,7 +1197,7 @@ svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 7, slice); } @@ -1216,7 +1216,7 @@ svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 0, slice_base); } @@ -1236,7 +1236,7 @@ svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8f16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 1, slice); } @@ -1255,7 +1255,7 @@ svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 0, slice_base); } @@ -1275,7 +1275,7 @@ svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 1, slice); } @@ -1294,7 +1294,7 @@ svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 0, slice_base); } @@ -1314,7 +1314,7 @@ svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4f32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 3, slice); } @@ -1333,7 +1333,7 @@ svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 0, slice_base); } @@ -1353,7 +1353,7 @@ svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2f64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 7, slice); } @@ -1370,7 +1370,7 @@ svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 0, slice_base); } @@ -1386,7 +1386,7 @@ svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) { +svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 15, slice_base); } @@ -1404,7 +1404,7 @@ svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 0, slice_base); } @@ -1422,7 +1422,7 @@ svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) { +svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 15, slice_base); } @@ -1440,7 +1440,7 @@ svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 0, slice_base); } @@ -1458,7 +1458,7 @@ svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) { +svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 15, slice_base); } @@ -1476,7 +1476,7 @@ svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 0, slice_base); } @@ -1494,7 +1494,7 @@ svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) { +svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 15, slice_base); } @@ -1510,7 +1510,7 @@ svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 0, slice_base); } @@ -1526,7 +1526,7 @@ svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) { +svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 15, slice_base); } @@ -1544,7 +1544,7 @@ svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 0, slice_base); } @@ -1562,7 +1562,7 @@ svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) { +svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 15, slice_base); } @@ -1580,7 +1580,7 @@ svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 0, slice_base); } @@ -1598,7 +1598,7 @@ svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) { +svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 15, slice_base); } @@ -1616,7 +1616,7 @@ svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 0, slice_base); } @@ -1634,7 +1634,7 @@ svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) { +svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 15, slice_base); } @@ -1652,7 +1652,7 @@ svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 0, slice_base); } @@ -1670,7 +1670,7 @@ svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8f16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 15, slice_base); } @@ -1688,7 +1688,7 @@ svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 0, slice_base); } @@ -1706,7 +1706,7 @@ svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t s // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) { +svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 15, slice_base); } @@ -1724,7 +1724,7 @@ svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 0, slice_base); } @@ -1742,7 +1742,7 @@ svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4f32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) { +svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 15, slice_base); } @@ -1760,7 +1760,7 @@ svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 0, slice_base); } @@ -1778,7 +1778,7 @@ svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2f64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) { +svfloat64_t test_svread_ver_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 15, slice_base); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c index 3f11aa8c5eb2e..2cc338add314b 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice_base, pg, zn); } @@ -44,7 +44,7 @@ void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice, pg, zn); } @@ -63,7 +63,7 @@ void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(0, slice_base, pg, zn); } @@ -83,7 +83,7 @@ void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(1, slice, pg, zn); } @@ -102,7 +102,7 @@ void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(0, slice_base, pg, zn); } @@ -122,7 +122,7 @@ void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(3, slice, pg, zn); } @@ -141,7 +141,7 @@ void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(0, slice_base, pg, zn); } @@ -161,7 +161,7 @@ void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(7, slice, pg, zn); } @@ -178,7 +178,7 @@ void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice_base, pg, zn); } @@ -196,7 +196,7 @@ void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice, pg, zn); } @@ -215,7 +215,7 @@ void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(0, slice_base, pg, zn); } @@ -235,7 +235,7 @@ void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(1, slice, pg, zn); } @@ -254,7 +254,7 @@ void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(0, slice_base, pg, zn); } @@ -274,7 +274,7 @@ void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(3, slice, pg, zn); } @@ -293,7 +293,7 @@ void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(0, slice_base, pg, zn); } @@ -313,7 +313,7 @@ void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(7, slice, pg, zn); } @@ -332,7 +332,7 @@ void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(0, slice_base, pg, zn); } @@ -352,7 +352,7 @@ void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(1, slice, pg, zn); } @@ -371,7 +371,7 @@ void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(0, slice_base, pg, zn); } @@ -391,7 +391,7 @@ void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(1, slice, pg, zn); } @@ -410,7 +410,7 @@ void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(0, slice_base, pg, zn); } @@ -430,7 +430,7 @@ void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(3, slice, pg, zn); } @@ -449,7 +449,7 @@ void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(0, slice_base, pg, zn); } @@ -469,7 +469,7 @@ void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(7, slice, pg, zn); } @@ -486,7 +486,7 @@ void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(0, slice_base, pg, zn); } @@ -502,7 +502,7 @@ void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(15, slice_base, pg, zn); } @@ -520,7 +520,7 @@ void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(0, slice_base, pg, zn); } @@ -538,7 +538,7 @@ void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(15, slice_base, pg, zn); } @@ -556,7 +556,7 @@ void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(0, slice_base, pg, zn); } @@ -574,7 +574,7 @@ void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(15, slice_base, pg, zn); } @@ -592,7 +592,7 @@ void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(0, slice_base, pg, zn); } @@ -610,7 +610,7 @@ void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(15, slice_base, pg, zn); } @@ -626,7 +626,7 @@ void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(0, slice_base, pg, zn); } @@ -642,7 +642,7 @@ void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(15, slice_base, pg, zn); } @@ -660,7 +660,7 @@ void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(0, slice_base, pg, zn); } @@ -678,7 +678,7 @@ void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(15, slice_base, pg, zn); } @@ -696,7 +696,7 @@ void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(0, slice_base, pg, zn); } @@ -714,7 +714,7 @@ void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(15, slice_base, pg, zn); } @@ -732,7 +732,7 @@ void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(0, slice_base, pg, zn); } @@ -750,7 +750,7 @@ void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(15, slice_base, pg, zn); } @@ -768,7 +768,7 @@ void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(0, slice_base, pg, zn); } @@ -786,7 +786,7 @@ void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(15, slice_base, pg, zn); } @@ -804,7 +804,7 @@ void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(0, slice_base, pg, zn); } @@ -822,7 +822,7 @@ void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(15, slice_base, pg, zn); } @@ -840,7 +840,7 @@ void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(0, slice_base, pg, zn); } @@ -858,7 +858,7 @@ void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(15, slice_base, pg, zn); } @@ -876,7 +876,7 @@ void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(0, slice_base, pg, zn); } @@ -894,7 +894,7 @@ void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(15, slice_base, pg, zn); } @@ -910,7 +910,7 @@ void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice_base, pg, zn); } @@ -928,7 +928,7 @@ void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice, pg, zn); } @@ -947,7 +947,7 @@ void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(0, slice_base, pg, zn); } @@ -967,7 +967,7 @@ void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(1, slice, pg, zn); } @@ -986,7 +986,7 @@ void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(0, slice_base, pg, zn); } @@ -1006,7 +1006,7 @@ void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(3, slice, pg, zn); } @@ -1025,7 +1025,7 @@ void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(0, slice_base, pg, zn); } @@ -1045,7 +1045,7 @@ void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(7, slice, pg, zn); } @@ -1062,7 +1062,7 @@ void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice_base, pg, zn); } @@ -1080,7 +1080,7 @@ void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice, pg, zn); } @@ -1099,7 +1099,7 @@ void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(0, slice_base, pg, zn); } @@ -1119,7 +1119,7 @@ void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(1, slice, pg, zn); } @@ -1138,7 +1138,7 @@ void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(0, slice_base, pg, zn); } @@ -1158,7 +1158,7 @@ void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(3, slice, pg, zn); } @@ -1177,7 +1177,7 @@ void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(0, slice_base, pg, zn); } @@ -1197,7 +1197,7 @@ void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(7, slice, pg, zn); } @@ -1216,7 +1216,7 @@ void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(0, slice_base, pg, zn); } @@ -1236,7 +1236,7 @@ void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(1, slice, pg, zn); } @@ -1255,7 +1255,7 @@ void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(0, slice_base, pg, zn); } @@ -1275,7 +1275,7 @@ void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(1, slice, pg, zn); } @@ -1294,7 +1294,7 @@ void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(0, slice_base, pg, zn); } @@ -1314,7 +1314,7 @@ void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(3, slice, pg, zn); } @@ -1333,7 +1333,7 @@ void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(0, slice_base, pg, zn); } @@ -1353,7 +1353,7 @@ void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(7, slice, pg, zn); } @@ -1370,7 +1370,7 @@ void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(0, slice_base, pg, zn); } @@ -1386,7 +1386,7 @@ void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) { +void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(15, slice_base, pg, zn); } @@ -1404,7 +1404,7 @@ void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(0, slice_base, pg, zn); } @@ -1422,7 +1422,7 @@ void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) { +void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(15, slice_base, pg, zn); } @@ -1440,7 +1440,7 @@ void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(0, slice_base, pg, zn); } @@ -1458,7 +1458,7 @@ void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) { +void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(15, slice_base, pg, zn); } @@ -1476,7 +1476,7 @@ void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(0, slice_base, pg, zn); } @@ -1494,7 +1494,7 @@ void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) { +void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(15, slice_base, pg, zn); } @@ -1510,7 +1510,7 @@ void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(0, slice_base, pg, zn); } @@ -1526,7 +1526,7 @@ void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) { +void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(15, slice_base, pg, zn); } @@ -1544,7 +1544,7 @@ void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(0, slice_base, pg, zn); } @@ -1562,7 +1562,7 @@ void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) { +void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(15, slice_base, pg, zn); } @@ -1580,7 +1580,7 @@ void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(0, slice_base, pg, zn); } @@ -1598,7 +1598,7 @@ void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) { +void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(15, slice_base, pg, zn); } @@ -1616,7 +1616,7 @@ void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(0, slice_base, pg, zn); } @@ -1634,7 +1634,7 @@ void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) { +void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(15, slice_base, pg, zn); } @@ -1652,7 +1652,7 @@ void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(0, slice_base, pg, zn); } @@ -1670,7 +1670,7 @@ void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) { +void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(15, slice_base, pg, zn); } @@ -1688,7 +1688,7 @@ void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(0, slice_base, pg, zn); } @@ -1706,7 +1706,7 @@ void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) { +void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(15, slice_base, pg, zn); } @@ -1724,7 +1724,7 @@ void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(0, slice_base, pg, zn); } @@ -1742,7 +1742,7 @@ void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) { +void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(15, slice_base, pg, zn); } @@ -1760,7 +1760,7 @@ void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(0, slice_base, pg, zn); } @@ -1778,7 +1778,7 @@ void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) { +void test_svwrite_ver_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(15, slice_base, pg, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c index e77e09c443518..361a9e82a3adb 100644 --- a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c +++ b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c @@ -5,6 +5,8 @@ // REQUIRES: aarch64-registered-target #include "arm_neon.h" +#include "arm_sme_draft_spec_subject_to_change.h" +#include "arm_sve.h" int16x8_t incompat_neon_sm(int16x8_t splat) __arm_streaming { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} @@ -20,3 +22,78 @@ int16x8_t incompat_neon_smc(int16x8_t splat) __arm_streaming_compatible { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming compatible function}} return (int16x8_t)__builtin_neon_vqaddq_v((int8x16_t)splat, (int8x16_t)splat, 33); } + +void incompat_sme_smc(svbool_t pg, void const *ptr) __arm_streaming_compatible __arm_shared_za { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming compatible function}} + return __builtin_sme_svld1_hor_za128(0, 0, pg, ptr); +} + +svuint32_t incompat_sve_sm(svbool_t pg, svuint32_t a, int16_t b) __arm_streaming { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} + return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); +} + +__arm_locally_streaming svuint32_t incompat_sve_ls(svbool_t pg, svuint32_t a, int64_t b) { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} + return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); +} + +svuint32_t incompat_sve_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming compatible function}} + return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); +} + +svuint32_t incompat_sve2_sm(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} + return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); +} + +__arm_locally_streaming svuint32_t incompat_sve2_ls(svbool_t pg, svuint32_t a, int64_t b) { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} + return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); +} + +svuint32_t incompat_sve2_smc(svbool_t pg, svuint32_t a, int64_t b) __arm_streaming_compatible { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming compatible function}} + return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); +} + +void incompat_sme_sm(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_shared_za { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a non-streaming function}} + svmops_za32_f32_m(0, pn, pm, zn, zm); +} + +svfloat64_t streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming { + // expected-no-warning + return svadd_n_f64_m(pg, a, b); +} + +__arm_locally_streaming svfloat64_t locally_streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) { + // expected-no-warning + return svadd_n_f64_m(pg, a, b); +} + +svfloat64_t streaming_compatible_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_streaming_compatible { + // expected-no-warning + return svadd_n_f64_m(pg, a, b); +} + +svint16_t streaming_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming { + // expected-no-warning + return svmul_lane_s16(op1, op2, 0); +} + +__arm_locally_streaming svint16_t locally_streaming_caller_sve2(svint16_t op1, svint16_t op2) { + // expected-no-warning + return svmul_lane_s16(op1, op2, 0); +} + +svint16_t streaming_compatible_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming_compatible { + // expected-no-warning + return svmul_lane_s16(op1, op2, 0); +} + +svbool_t streaming_caller_ptrue(void) __arm_streaming { + // expected-no-warning + return svand_z(svptrue_b16(), svptrue_pat_b16(SV_ALL), svptrue_pat_b16(SV_VL4)); +} diff --git a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp index 1faa5638c801c..47c7210206b05 100644 --- a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp +++ b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp @@ -12,7 +12,7 @@ #include -void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) { +void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 0]}} SVE_ACLE_FUNC(svld1_hor_za8,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 1 is outside the valid range [0, 0]}} @@ -32,7 +32,7 @@ void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) { SVE_ACLE_FUNC(svwrite_ver_za8, _s8, _m,)(1, slice, pg, svundef_s8()); } -void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) { +void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}} SVE_ACLE_FUNC(svld1_hor_za16,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}} @@ -52,7 +52,7 @@ void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) { SVE_ACLE_FUNC(svwrite_ver_za16, _s16, _m,)(2, slice, pg, svundef_s16()); } -void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) { +void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}} SVE_ACLE_FUNC(svld1_hor_za32,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}} @@ -90,7 +90,7 @@ void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) { SVE_ACLE_FUNC(svusmops_za32, _u8, _m,)(-1, pg, pg, svundef_u8(), svundef_s8()); } -void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) { +void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}} SVE_ACLE_FUNC(svld1_hor_za64,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 8 is outside the valid range [0, 7]}} @@ -133,7 +133,7 @@ void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) { SVE_ACLE_FUNC(svmops_za64, _f64, _m,)(-1, pg, pg, svundef_f64(), svundef_f64()); } -void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) { +void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 15]}} SVE_ACLE_FUNC(svld1_hor_za128,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}} @@ -153,14 +153,14 @@ void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) { SVE_ACLE_FUNC(svwrite_ver_za128, _s8, _m,)(16, slice, pg, svundef_s8()); } -void test_range_0_255(svbool_t pg, void *ptr) { +void test_range_0_255(svbool_t pg, void *ptr) __arm_streaming { // expected-error@+1 {{argument value 256 is outside the valid range [0, 255]}} SVE_ACLE_FUNC(svzero_mask_za,,,)(256); // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 255]}} SVE_ACLE_FUNC(svzero_mask_za,,,)(-1); } -void test_constant(uint64_t u64, svbool_t pg, void *ptr) { +void test_constant(uint64_t u64, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svld1_hor_za8,,,)(u64, u64, pg, ptr); // expected-error {{argument to 'svld1_hor_za8' must be a constant integer}} SVE_ACLE_FUNC(svst1_hor_za32,,,)(u64, 0, pg, ptr); // expected-error {{argument to 'svst1_hor_za32' must be a constant integer}} SVE_ACLE_FUNC(svld1_hor_vnum_za8,,,)(u64, 0, pg, ptr, u64); // expected-error {{argument to 'svld1_hor_vnum_za8' must be a constant integer}} diff --git a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c index 2de6d9f6877f0..7cfe9fdfbd24f 100644 --- a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c +++ b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c @@ -6,20 +6,21 @@ #include __attribute__((target("sme"))) -void test_sme(svbool_t pg, void *ptr) { +void test_sme(svbool_t pg, void *ptr) __arm_streaming { svld1_hor_za8(0, 0, pg, ptr); } __attribute__((target("arch=armv8-a+sme"))) -void test_arch_sme(svbool_t pg, void *ptr) { +void test_arch_sme(svbool_t pg, void *ptr) __arm_streaming { svld1_hor_vnum_za32(0, 0, pg, ptr, 0); } __attribute__((target("+sme"))) -void test_plus_sme(svbool_t pg, void *ptr) { +void test_plus_sme(svbool_t pg, void *ptr) __arm_streaming { svst1_ver_za16(0, 0, pg, ptr); } +__attribute__((target("+sme"))) void undefined(svbool_t pg, void *ptr) { - svst1_ver_vnum_za64(0, 0, pg, ptr, 0); // expected-error {{'svst1_ver_vnum_za64' needs target feature sme}} + svst1_ver_vnum_za64(0, 0, pg, ptr, 0); // expected-warning {{builtin call has undefined behaviour when called from a non-streaming function}} } diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp index e5f79ba99c5c8..53334016c180a 100644 --- a/clang/utils/TableGen/NeonEmitter.cpp +++ b/clang/utils/TableGen/NeonEmitter.cpp @@ -550,6 +550,8 @@ class NeonEmitter { void createIntrinsic(Record *R, SmallVectorImpl &Out); void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl &Defs); + void genStreamingSVECompatibleList(raw_ostream &OS, + SmallVectorImpl &Defs); void genOverloadTypeCheckCode(raw_ostream &OS, SmallVectorImpl &Defs); void genIntrinsicRangeCheckCode(raw_ostream &OS, @@ -2041,6 +2043,30 @@ void NeonEmitter::genBuiltinsDef(raw_ostream &OS, OS << "#endif\n\n"; } +void NeonEmitter::genStreamingSVECompatibleList( + raw_ostream &OS, SmallVectorImpl &Defs) { + OS << "#ifdef GET_NEON_STREAMING_COMPAT_FLAG\n"; + + std::set Emitted; + for (auto *Def : Defs) { + // If the def has a body (that is, it has Operation DAGs), it won't call + // __builtin_neon_* so we don't need to generate a definition for it. + if (Def->hasBody()) + continue; + + std::string Name = Def->getMangledName(); + if (Emitted.find(Name) != Emitted.end()) + continue; + + // FIXME: We should make exceptions here for some NEON builtins that are + // permitted in streaming mode. + OS << "case NEON::BI__builtin_neon_" << Name + << ": BuiltinType = ArmNonStreaming; break;\n"; + Emitted.insert(Name); + } + OS << "#endif\n\n"; +} + /// Generate the ARM and AArch64 overloaded type checking code for /// SemaChecking.cpp, checking for unique builtin declarations. void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS, @@ -2224,6 +2250,8 @@ void NeonEmitter::runHeader(raw_ostream &OS) { // Generate ARM overloaded type checking code for SemaChecking.cpp genOverloadTypeCheckCode(OS, Defs); + genStreamingSVECompatibleList(OS, Defs); + // Generate ARM range checking code for shift/lane immediates. genIntrinsicRangeCheckCode(OS, Defs); } diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp index 9361b99506377..a59b7099d5adf 100644 --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -379,6 +379,9 @@ class SVEEmitter { /// Emit all the information needed to map builtin -> LLVM IR intrinsic. void createSMECodeGenMap(raw_ostream &o); + /// Create a table for a builtin's requirement for PSTATE.SM. + void createStreamingAttrs(raw_ostream &o, ACLEKind Kind); + /// Emit all the range checks for the immediates. void createSMERangeChecks(raw_ostream &o); @@ -1702,6 +1705,51 @@ void SVEEmitter::createSMERangeChecks(raw_ostream &OS) { OS << "#endif\n\n"; } +void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { + std::vector RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + StringRef ExtensionKind; + switch (Kind) { + case ACLEKind::SME: + ExtensionKind = "SME"; + break; + case ACLEKind::SVE: + ExtensionKind = "SVE"; + break; + } + + OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n"; + + llvm::StringMap> StreamingMap; + + uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming"); + uint64_t IsStreamingCompatibleFlag = + getEnumValueForFlag("IsStreamingCompatible"); + for (auto &Def : Defs) { + if (Def->isFlagSet(IsStreamingFlag)) + StreamingMap["ArmStreaming"].insert(Def->getMangledName()); + else if (Def->isFlagSet(IsStreamingCompatibleFlag)) + StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName()); + else + StreamingMap["ArmNonStreaming"].insert(Def->getMangledName()); + } + + for (auto BuiltinType : StreamingMap.keys()) { + for (auto Name : StreamingMap[BuiltinType]) { + OS << "case " << ExtensionKind << "::BI__builtin_" + << ExtensionKind.lower() << "_"; + OS << Name << ":\n"; + } + OS << " BuiltinType = " << BuiltinType << ";\n"; + OS << " break;\n"; + } + + OS << "#endif\n\n"; +} + namespace clang { void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createHeader(OS); @@ -1723,6 +1771,10 @@ void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createTypeFlags(OS); } +void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE); +} + void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createSMEHeader(OS); } @@ -1739,4 +1791,7 @@ void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createSMERangeChecks(OS); } +void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME); +} } // End namespace clang diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp index 3ad46b95984ec..9043d90d7cb42 100644 --- a/clang/utils/TableGen/TableGen.cpp +++ b/clang/utils/TableGen/TableGen.cpp @@ -86,10 +86,12 @@ enum ActionType { GenArmSveBuiltinCG, GenArmSveTypeFlags, GenArmSveRangeChecks, + GenArmSveStreamingAttrs, GenArmSmeHeader, GenArmSmeBuiltins, GenArmSmeBuiltinCG, GenArmSmeRangeChecks, + GenArmSmeStreamingAttrs, GenArmCdeHeader, GenArmCdeBuiltinDef, GenArmCdeBuiltinSema, @@ -246,6 +248,8 @@ cl::opt Action( "Generate arm_sve_typeflags.inc for clang"), clEnumValN(GenArmSveRangeChecks, "gen-arm-sve-sema-rangechecks", "Generate arm_sve_sema_rangechecks.inc for clang"), + clEnumValN(GenArmSveStreamingAttrs, "gen-arm-sve-streaming-attrs", + "Generate arm_sve_streaming_attrs.inc for clang"), clEnumValN(GenArmSmeHeader, "gen-arm-sme-header", "Generate arm_sme.h for clang"), clEnumValN(GenArmSmeBuiltins, "gen-arm-sme-builtins", @@ -254,6 +258,8 @@ cl::opt Action( "Generate arm_sme_builtin_cg_map.inc for clang"), clEnumValN(GenArmSmeRangeChecks, "gen-arm-sme-sema-rangechecks", "Generate arm_sme_sema_rangechecks.inc for clang"), + clEnumValN(GenArmSmeStreamingAttrs, "gen-arm-sme-streaming-attrs", + "Generate arm_sme_streaming_attrs.inc for clang"), clEnumValN(GenArmMveHeader, "gen-arm-mve-header", "Generate arm_mve.h for clang"), clEnumValN(GenArmMveBuiltinDef, "gen-arm-mve-builtin-def", @@ -494,6 +500,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenArmSveRangeChecks: EmitSveRangeChecks(Records, OS); break; + case GenArmSveStreamingAttrs: + EmitSveStreamingAttrs(Records, OS); + break; case GenArmSmeHeader: EmitSmeHeader(Records, OS); break; @@ -506,6 +515,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenArmSmeRangeChecks: EmitSmeRangeChecks(Records, OS); break; + case GenArmSmeStreamingAttrs: + EmitSmeStreamingAttrs(Records, OS); + break; case GenArmCdeHeader: EmitCdeHeader(Records, OS); break; diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h index ef255612f4b8b..6ec51776d637c 100644 --- a/clang/utils/TableGen/TableGenBackends.h +++ b/clang/utils/TableGen/TableGenBackends.h @@ -105,11 +105,13 @@ void EmitSveBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveTypeFlags(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveRangeChecks(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSveStreamingAttrs(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeRangeChecks(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSmeStreamingAttrs(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveBuiltinDef(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); From 2f82662ce901c6666fceb9c6c5e0de216a1c9667 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Mon, 18 Dec 2023 17:39:22 +0800 Subject: [PATCH 087/884] [PowerPC] Let base implementation decide if MI is rematerizable by default (#75772) If MI is not PPC specific instructions, let base implementation decide if MI is rematerizable. This can fix failure in #75570 after #75271 . --- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index d0a6cced1b197..4dfd0358a16fd 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -1067,9 +1067,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable( const MachineInstr &MI) const { switch (MI.getOpcode()) { default: - // This function should only be called for opcodes with the ReMaterializable - // flag set. - llvm_unreachable("Unknown rematerializable operation!"); + // Let base implementaion decide. break; case PPC::LI: case PPC::LI8: From 49b0e6dcc296792b577ae8f0f674e61a0929b99d Mon Sep 17 00:00:00 2001 From: David Sherwood <57997763+david-arm@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:41:54 +0000 Subject: [PATCH 088/884] [LoopVectorize] Enable hoisting of runtime checks by default (#71538) With commit https://reviews.llvm.org/D152366 I introduced functionality that permitted the hoisting of runtime memory checks from a vectorised inner loop to the preheader of the next outer-most loop. This is useful for benchmarks like SPEC2017's x264 where the inner loop is vectorised and only has a small trip count. In such cases the runtime memory checks become expensive and since the checks never fail in the case of x264 it makes sense to do this. However, this behaviour was controlled by the flag -hoist-runtime-checks which was off by default. This patch enables this flag by default for all targets, since I believe this is a generally beneficial thing to do. I have tested this with SPEC2017 and I see 2.3% and 2.6% improvements with x264 on neoverse-v1 and neoverse-n1, respectively. Similarly, I saw slight improvements in the overall geomean on both machines. The only other notable changes were a 1% drop in the roms benchmark, which was compensated for by a 1% improvement in fotonik3d. --- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 2 +- .../invariant-store-vectorization.ll | 83 ++++++++--------- .../multiple-strides-vectorization.ll | 90 ++++++++++++++++--- .../runtime-checks-difference.ll | 2 +- .../LoopVectorize/runtime-checks-hoist.ll | 2 +- 5 files changed, 124 insertions(+), 55 deletions(-) diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 0894560fd0789..89666018d9251 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -143,7 +143,7 @@ static cl::opt HoistRuntimeChecks( "hoist-runtime-checks", cl::Hidden, cl::desc( "Hoist inner loop runtime memory checks to outer loop if possible"), - cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(false)); + cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true)); bool VectorizerParams::HoistRuntimeChecks; bool VectorizerParams::isInterleaveForced() { diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll index 9e36649bcf73d..c4d7ef8e949b0 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll @@ -346,74 +346,75 @@ define i32 @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly ; CHECK-NEXT: [[CMP20:%.*]] = icmp eq i32 [[ITR:%.*]], 0 ; CHECK-NEXT: br i1 [[CMP20]], label [[FOR_END10:%.*]], label [[FOR_COND1_PREHEADER_PREHEADER:%.*]] ; CHECK: for.cond1.preheader.preheader: -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 4 -; CHECK-NEXT: [[INVARIANT_GEP5:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 4 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[ITR]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[VAR1:%.*]], i64 [[TMP2]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP3]], i64 4 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[VAR2:%.*]], i64 4 ; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]] ; CHECK: for.cond1.preheader: ; CHECK-NEXT: [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT24:%.*]], [[FOR_INC8:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_PREHEADER]] ] ; CHECK-NEXT: [[J_022:%.*]] = phi i32 [ [[J_1_LCSSA:%.*]], [[FOR_INC8]] ], [ 0, [[FOR_COND1_PREHEADER_PREHEADER]] ] -; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV23]], 2 -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[VAR1]], i64 [[TMP0]] -; CHECK-NEXT: [[GEP6:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP5]], i64 [[TMP0]] ; CHECK-NEXT: [[CMP218:%.*]] = icmp ult i32 [[J_022]], [[ITR]] ; CHECK-NEXT: br i1 [[CMP218]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC8]] ; CHECK: for.body3.lr.ph: ; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[VAR1]], i64 [[INDVARS_IV23]] -; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[J_022]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[J_022]] to i64 ; CHECK-NEXT: [[ARRAYIDX5_PROMOTED:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = xor i32 [[J_022]], -1 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], [[ITR]] -; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 -; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[J_022]], -1 +; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], [[ITR]] +; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64 +; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP6]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP1]], 2 -; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[VAR2]], i64 [[TMP6]] -; CHECK-NEXT: [[TMP7:%.*]] = xor i32 [[J_022]], -1 -; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[ITR]] -; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP8]] to i64 -; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP1]], [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2 -; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP11]] -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[GEP6]] +; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP4]], 2 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[VAR2]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = xor i32 [[J_022]], -1 +; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], [[ITR]] +; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64 +; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i64 [[TMP4]], [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP13]], 2 +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP14]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP3]], [[VAR1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP5]], 8589934588 -; CHECK-NEXT: [[IND_END:%.*]] = add nuw nsw i64 [[N_VEC]], [[TMP1]] -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> , i32 [[ARRAYIDX5_PROMOTED]], i64 0 -; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr i32, ptr [[VAR2]], i64 [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP8]], 8589934588 +; CHECK-NEXT: [[IND_END:%.*]] = add nuw nsw i64 [[N_VEC]], [[TMP4]] +; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> , i32 [[ARRAYIDX5_PROMOTED]], i64 0 +; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr i32, ptr [[VAR2]], i64 [[TMP4]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP12]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP15]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[INVARIANT_GEP]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[GEP]], align 4, !alias.scope !23 -; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[TMP13]], +; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP17]] = add <4 x i32> [[TMP16]], ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] +; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP14]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[DOTLCSSA]]) -; CHECK-NEXT: store i32 [[TMP16]], ptr [[ARRAYIDX5]], align 4 -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]] +; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP17]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[DOTLCSSA]]) +; CHECK-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX5]], align 4 +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_INC8_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP1]], [[FOR_BODY3_LR_PH]] ], [ [[TMP1]], [[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ [[ARRAYIDX5_PROMOTED]], [[FOR_BODY3_LR_PH]] ], [ [[ARRAYIDX5_PROMOTED]], [[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP4]], [[FOR_BODY3_LR_PH]] ], [ [[TMP4]], [[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[ARRAYIDX5_PROMOTED]], [[FOR_BODY3_LR_PH]] ], [ [[ARRAYIDX5_PROMOTED]], [[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label [[FOR_BODY3:%.*]] ; CHECK: for.body3: -; CHECK-NEXT: [[TMP17:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP19:%.*]], [[FOR_BODY3]] ] +; CHECK-NEXT: [[TMP20:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP22:%.*]], [[FOR_BODY3]] ] ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[VAR2]], i64 [[INDVARS_IV]] -; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP17]], [[TMP18]] -; CHECK-NEXT: [[TMP19]] = add nsw i32 [[ADD]], 1 -; CHECK-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX5]], align 4 +; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP20]], [[TMP21]] +; CHECK-NEXT: [[TMP22]] = add nsw i32 [[ADD]], 1 +; CHECK-NEXT: store i32 [[TMP22]], ptr [[ARRAYIDX5]], align 4 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[ITR]] diff --git a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll index 47104645c18a6..fc6dcc3d278c9 100644 --- a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S < %s | FileCheck %s +; RUN: opt -passes=loop-vectorize -force-vector-width=4 -hoist-runtime-checks=false -S < %s | FileCheck %s --check-prefix=CHECK +; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S < %s | FileCheck %s --check-prefix=CHECK-HOIST ; This is the test case from PR26314. ; When we were retrying dependence checking with memchecks only, @@ -31,29 +32,29 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 { ; CHECK-LABEL: @Test( ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2 ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 256 -; CHECK-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 [[TMP1]] +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 [[TMP1]] ; CHECK-NEXT: br label [[DOTOUTER_PREHEADER:%.*]] ; CHECK: .outer.preheader: ; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ] ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[I]], 7 ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], 256 -; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP4]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP2]], [[TMP3]] -; CHECK-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP5]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP5]] ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[I]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 128 -; CHECK-NEXT: [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP7]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP6]], 132 -; CHECK-NEXT: [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP8]] +; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[OBJ]], i64 0, i32 1, i64 [[I]] ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[UGLYGEP]], [[UGLYGEP2]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[UGLYGEP1]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; CHECK-NEXT: [[BOUND05:%.*]] = icmp ult ptr [[UGLYGEP]], [[UGLYGEP4]] -; CHECK-NEXT: [[BOUND16:%.*]] = icmp ult ptr [[UGLYGEP3]], [[UGLYGEP1]] +; CHECK-NEXT: [[BOUND05:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP4]] +; CHECK-NEXT: [[BOUND16:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP1]] ; CHECK-NEXT: [[FOUND_CONFLICT7:%.*]] = and i1 [[BOUND05]], [[BOUND16]] ; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT7]] ; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] @@ -103,7 +104,74 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 { ; CHECK-NEXT: store i32 [[TMP25]], ptr [[TMP23]], align 4 ; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1 ; CHECK-NEXT: [[EXITCOND_INNER:%.*]] = icmp eq i64 [[J_NEXT]], [[Z]] -; CHECK-NEXT: br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP11:![0-9]+]] +; +; CHECK-HOIST-LABEL: @Test( +; CHECK-HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 256 +; CHECK-HOIST-NEXT: [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2 +; CHECK-HOIST-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 4224 +; CHECK-HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP2]] +; CHECK-HOIST-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP1]] +; CHECK-HOIST-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[OBJ]], i64 128 +; CHECK-HOIST-NEXT: br label [[DOTOUTER_PREHEADER:%.*]] +; CHECK-HOIST: .outer.preheader: +; CHECK-HOIST-NEXT: [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ] +; CHECK-HOIST-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[OBJ]], i64 0, i32 1, i64 [[I]] +; CHECK-HOIST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4 +; CHECK-HOIST-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] +; CHECK-HOIST: vector.memcheck: +; CHECK-HOIST-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] +; CHECK-HOIST-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OBJ]], [[SCEVGEP1]] +; CHECK-HOIST-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-HOIST-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SCEVGEP3]], [[SCEVGEP1]] +; CHECK-HOIST-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK-HOIST: vector.ph: +; CHECK-HOIST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[Z]], 4 +; CHECK-HOIST-NEXT: [[N_VEC:%.*]] = sub i64 [[Z]], [[N_MOD_VF]] +; CHECK-HOIST-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK-HOIST: vector.body: +; CHECK-HOIST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-HOIST-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 +; CHECK-HOIST-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[OBJ]], i64 0, i32 0, i64 [[TMP4]] +; CHECK-HOIST-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 +; CHECK-HOIST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !alias.scope !0 +; CHECK-HOIST-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope !3 +; CHECK-HOIST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0 +; CHECK-HOIST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-HOIST-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]] +; CHECK-HOIST-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[OBJ]], i64 0, i32 2, i64 [[I]], i64 [[TMP4]] +; CHECK-HOIST-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0 +; CHECK-HOIST-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !alias.scope !5, !noalias !7 +; CHECK-HOIST-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP8]], [[WIDE_LOAD5]] +; CHECK-HOIST-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP10]], align 4, !alias.scope !5, !noalias !7 +; CHECK-HOIST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-HOIST-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-HOIST-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-HOIST: middle.block: +; CHECK-HOIST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[Z]], [[N_VEC]] +; CHECK-HOIST-NEXT: br i1 [[CMP_N]], label [[DOTOUTER]], label [[SCALAR_PH]] +; CHECK-HOIST: scalar.ph: +; CHECK-HOIST-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTOUTER_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ] +; CHECK-HOIST-NEXT: br label [[DOTINNER:%.*]] +; CHECK-HOIST: .exit: +; CHECK-HOIST-NEXT: ret void +; CHECK-HOIST: .outer: +; CHECK-HOIST-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 +; CHECK-HOIST-NEXT: [[EXITCOND_OUTER:%.*]] = icmp eq i64 [[I_NEXT]], 32 +; CHECK-HOIST-NEXT: br i1 [[EXITCOND_OUTER]], label [[DOTEXIT:%.*]], label [[DOTOUTER_PREHEADER]] +; CHECK-HOIST: .inner: +; CHECK-HOIST-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[J_NEXT:%.*]], [[DOTINNER]] ] +; CHECK-HOIST-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[OBJ]], i64 0, i32 0, i64 [[J]] +; CHECK-HOIST-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4 +; CHECK-HOIST-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-HOIST-NEXT: [[TMP16:%.*]] = add nsw i32 [[TMP15]], [[TMP14]] +; CHECK-HOIST-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[OBJ]], i64 0, i32 2, i64 [[I]], i64 [[J]] +; CHECK-HOIST-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; CHECK-HOIST-NEXT: [[TMP19:%.*]] = add nsw i32 [[TMP16]], [[TMP18]] +; CHECK-HOIST-NEXT: store i32 [[TMP19]], ptr [[TMP17]], align 4 +; CHECK-HOIST-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1 +; CHECK-HOIST-NEXT: [[EXITCOND_INNER:%.*]] = icmp eq i64 [[J_NEXT]], [[Z]] +; CHECK-HOIST-NEXT: br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP11:![0-9]+]] ; br label %.outer.preheader diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll index 876d351fb1538..5fe1686fa7b5d 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s +; RUN: opt %s -passes=loop-vectorize -hoist-runtime-checks=false -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll index 7892db23d5282..0b9b592627c65 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; REQUIRES: asserts -; RUN: opt < %s -hoist-runtime-checks -p 'loop-vectorize' -force-vector-interleave=1 -S \ +; RUN: opt < %s -p 'loop-vectorize' -force-vector-interleave=1 -S \ ; RUN: -force-vector-width=4 -debug-only=loop-accesses,loop-vectorize,loop-utils 2> %t | FileCheck %s ; RUN: cat %t | FileCheck %s --check-prefix=DEBUG From d01be3c63109986627c1c029d6d0130f76a63a2f Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 18 Dec 2023 01:50:55 -0800 Subject: [PATCH 089/884] [sanitizer] Disable InstallAtForkHandler on Solaris, NetBSD (#75659) Handlers need missing FutexWait implementation. Reported in #75290. --- compiler-rt/lib/asan/asan_posix.cpp | 3 +++ compiler-rt/lib/lsan/lsan_posix.cpp | 3 +++ .../test/sanitizer_common/TestCases/Posix/fork_threaded.c | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/compiler-rt/lib/asan/asan_posix.cpp b/compiler-rt/lib/asan/asan_posix.cpp index 206551b6ef910..0d2e4fe017677 100644 --- a/compiler-rt/lib/asan/asan_posix.cpp +++ b/compiler-rt/lib/asan/asan_posix.cpp @@ -149,6 +149,9 @@ void PlatformTSDDtor(void *tsd) { #endif void InstallAtForkHandler() { +# if SANITIZER_SOLARIS || SANITIZER_NETBSD + return; // FIXME: Implement FutexWait. +# endif auto before = []() { if (CAN_SANITIZE_LEAKS) { __lsan::LockGlobal(); diff --git a/compiler-rt/lib/lsan/lsan_posix.cpp b/compiler-rt/lib/lsan/lsan_posix.cpp index 3677f0141a2f0..e0c1899edcd2c 100644 --- a/compiler-rt/lib/lsan/lsan_posix.cpp +++ b/compiler-rt/lib/lsan/lsan_posix.cpp @@ -101,6 +101,9 @@ void InstallAtExitCheckLeaks() { } void InstallAtForkHandler() { +# if SANITIZER_SOLARIS || SANITIZER_NETBSD + return; // FIXME: Implement FutexWait. +# endif auto before = []() { LockGlobal(); LockThreads(); diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c index 6c5102049ae8e..5c6a8743cdb58 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c +++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c @@ -12,6 +12,10 @@ // FIXME: False stack overflow report // UNSUPPORTED: android && asan +// FIXME: Requires `FutexWait` implementation. See __asan::InstallAtForkHandler. +// UNSUPPORTED: target={{.*solaris.*}} +// UNSUPPORTED: target={{.*netbsd.*}} + // Forking in multithread environment is unsupported. However we already have // some workarounds, and will add more, so this is the test. // The test try to check two things: From d777504355dac9a2a8a7bbba6001d05b936868b2 Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Mon, 18 Dec 2023 11:14:46 +0100 Subject: [PATCH 090/884] [MLIR][OpenMP][Offload] Lower target update op to DeviceRT (#75159) Adds support for lowring `UpdateDataOp` to the DeviceRT. This reuses the existing utils used by other device directive. --- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 24 ++++++++++-- mlir/test/Target/LLVMIR/omptarget-llvm.mlir | 38 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 4f6200d29a70a..088e7ae4231be 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -1915,6 +1915,23 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, mapOperands = exitDataOp.getMapOperands(); return success(); }) + .Case([&](omp::UpdateDataOp updateDataOp) { + if (updateDataOp.getNowait()) + return failure(); + + if (auto ifExprVar = updateDataOp.getIfExpr()) + ifCond = moduleTranslation.lookupValue(ifExprVar); + + if (auto devId = updateDataOp.getDevice()) + if (auto constOp = + dyn_cast(devId.getDefiningOp())) + if (auto intAttr = dyn_cast(constOp.getValue())) + deviceID = intAttr.getInt(); + + RTLFn = llvm::omp::OMPRTL___tgt_target_data_update_mapper; + mapOperands = updateDataOp.getMotionOperands(); + return success(); + }) .Default([&](Operation *op) { return op->emitError("unsupported OpenMP operation: ") << op->getName(); @@ -2748,9 +2765,10 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation( .Case([&](omp::ThreadprivateOp) { return convertOmpThreadprivate(*op, builder, moduleTranslation); }) - .Case([&](auto op) { - return convertOmpTargetData(op, builder, moduleTranslation); - }) + .Case( + [&](auto op) { + return convertOmpTargetData(op, builder, moduleTranslation); + }) .Case([&](omp::TargetOp) { return convertOmpTarget(*op, builder, moduleTranslation); }) diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir index 9221b410d766e..b089d47f795df 100644 --- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir @@ -441,3 +441,41 @@ llvm.func @_QPopenmp_target_use_dev_both() { // CHECK: ret void // ----- + +llvm.func @_QPopenmp_target_data_update() { + %0 = llvm.mlir.constant(1 : i64) : i64 + %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr + %2 = omp.map_info var_ptr(%1 : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""} + omp.target_data map_entries(%2 : !llvm.ptr) { + %3 = llvm.mlir.constant(99 : i32) : i32 + llvm.store %3, %1 : i32, !llvm.ptr + omp.terminator + } + + omp.target_update_data motion_entries(%2 : !llvm.ptr) + + llvm.return +} + +// CHECK-LABEL: define void @_QPopenmp_target_data_update + +// CHECK-DAG: %[[OFFLOAD_BASEPTRS:.*]] = alloca [1 x ptr], align 8 +// CHECK-DAG: %[[OFFLOAD_PTRS:.*]] = alloca [1 x ptr], align 8 +// CHECK-DAG: %[[INT_ALLOCA:.*]] = alloca i32, i64 1, align 4 +// CHECK-DAG: %[[OFFLOAD_MAPPERS:.*]] = alloca [1 x ptr], align 8 + +// CHECK: call void @__tgt_target_data_begin_mapper +// CHECK: store i32 99, ptr %[[INT_ALLOCA]], align 4 +// CHECK: call void @__tgt_target_data_end_mapper + +// CHECK: %[[BASEPTRS_VAL:.*]] = getelementptr inbounds [1 x ptr], ptr %[[OFFLOAD_BASEPTRS]], i32 0, i32 0 +// CHECK: store ptr %[[INT_ALLOCA]], ptr %[[BASEPTRS_VAL]], align 8 +// CHECK: %[[PTRS_VAL:.*]] = getelementptr inbounds [1 x ptr], ptr %[[OFFLOAD_PTRS]], i32 0, i32 0 +// CHECK: store ptr %[[INT_ALLOCA]], ptr %[[PTRS_VAL]], align 8 +// CHECK: %[[MAPPERS_VAL:.*]] = getelementptr inbounds [1 x ptr], ptr %[[OFFLOAD_MAPPERS]], i64 0, i64 0 +// CHECK: store ptr null, ptr %[[MAPPERS_VAL]], align 8 +// CHECK: %[[BASEPTRS_VAL_2:.*]] = getelementptr inbounds [1 x ptr], ptr %[[OFFLOAD_BASEPTRS]], i32 0, i32 0 +// CHECK: %[[PTRS_VAL_2:.*]] = getelementptr inbounds [1 x ptr], ptr %[[OFFLOAD_PTRS]], i32 0, i32 0 +// CHECK: call void @__tgt_target_data_update_mapper(ptr @2, i64 -1, i32 1, ptr %[[BASEPTRS_VAL_2]], ptr %[[PTRS_VAL_2]], ptr @{{.*}}, ptr @{{.*}}, ptr @{{.*}}, ptr null) + +// CHECK: ret void From 9e3d915d8ebf86e24c9ff58766be8e7c6aa7b0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Mon, 18 Dec 2023 12:31:05 +0200 Subject: [PATCH 091/884] [llvm-windres] Change the interpretation of --preprocessor to match Binutils 2.36 (#75391) Binutils 2.36 had a somewhat controversial change in how the --preprocessor option was handled in GNU windres; previously, the option was interpreted as a part of the command string, potentially containing multiple arguments (which even was hinted at in the documentation). In Binutils 2.36, this was changed to interpret the --preprocessor argument as one argument (possibly containing spaces) pointing at the preprocessor executable. The existing behaviour where implicit arguments like -E -xc -DRC_INVOKED are dropped if --preprocessor is specified, was kept. This was a breaking change for some users of GNU windres, see https://sourceware.org/git/?p=binutils-gdb.git;a=commitdiff;h=21c33bcbe36377abf01614fb1b9be439a3b6de20, https://sourceware.org/bugzilla/show_bug.cgi?id=27594, and https://sourceware.org/git/?p=binutils-gdb.git;a=commitdiff;h=5edb8e3f5ad8d74a83fc0df7f6e4514eed0aa77f. As multiple years have passed since, the behaviour change seems to be here to stay, and any users of the previous form of the option have been forced to avoid this construct. Thus update llvm-windres to match the new way Binutils of handling this option. One construct for specifying the path to the preprocessor, which works both before and after binutils 2.36 (and this change in llvm-windres) is to specify options like this: --preprocessor path/to/executable --preprocessor-arg -E --preprocessor-arg -xc -DRC_INVOKED --- llvm/test/tools/llvm-rc/windres-preproc.test | 9 +++- llvm/tools/llvm-rc/llvm-rc.cpp | 44 +++----------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/llvm/test/tools/llvm-rc/windres-preproc.test b/llvm/test/tools/llvm-rc/windres-preproc.test index 74e888614aa2b..13f82299a074b 100644 --- a/llvm/test/tools/llvm-rc/windres-preproc.test +++ b/llvm/test/tools/llvm-rc/windres-preproc.test @@ -6,8 +6,8 @@ ; RUN: llvm-windres -### --include-dir %p/incdir1 --include %p/incdir2 "-DFOO1=\\\"foo bar\\\"" -UFOO2 -D FOO3 --preprocessor-arg "-DFOO4=\\\"baz baz\\\"" -DFOO5=\"bar\" %p/Inputs/empty.rc %t.res | FileCheck %s --check-prefix=CHECK1 ; RUN: llvm-windres -### --include-dir %p/incdir1 --include %p/incdir2 "-DFOO1=\"foo bar\"" -UFOO2 -D FOO3 --preprocessor-arg "-DFOO4=\"baz baz\"" "-DFOO5=bar" %p/Inputs/empty.rc %t.res --use-temp-file | FileCheck %s --check-prefix=CHECK1 ; CHECK1: {{^}} "clang" "--driver-mode=gcc" "-target" "{{.*}}-{{.*}}{{mingw32|windows-gnu}}" "-E" "-xc" "-DRC_INVOKED" "-I" "{{.*}}incdir1" "-I" "{{.*}}incdir2" "-D" "FOO1=\"foo bar\"" "-U" "FOO2" "-D" "FOO3" "-DFOO4=\"baz baz\"" "-D" "FOO5=bar" "{{.*}}empty.rc" "-o" "{{.*}}preproc-{{.*}}.rc"{{$}} -; RUN: llvm-windres -### --preprocessor "i686-w64-mingw32-gcc -E -DFOO=\\\"foo\\ bar\\\"" %p/Inputs/empty.rc %t.res | FileCheck %s --check-prefix=CHECK2 -; CHECK2: {{^}} "{{.*}}i686-w64-mingw32-gcc" "-E" "-DFOO=\"foo bar\"" "{{.*}}empty.rc" "-o" "{{.*}}preproc-{{.*}}.rc"{{$}} +; RUN: llvm-windres -### --preprocessor "i686-w64-mingw32-gcc" --preprocessor-arg -E "-DFOO=\\\"foo bar\\\"" %p/Inputs/empty.rc %t.res | FileCheck %s --check-prefix=CHECK2 +; CHECK2: {{^}} "{{.*}}i686-w64-mingw32-gcc" "-E" "-D" "FOO=\"foo bar\"" "{{.*}}empty.rc" "-o" "{{.*}}preproc-{{.*}}.rc"{{$}} ;; Test resolving the --preprocessor executable from PATH @@ -22,3 +22,8 @@ ; RUN: not llvm-windres --preprocessor intentionally-missing-executable %p/Inputs/empty.rc %t.res 2>&1 | FileCheck %s --check-prefix=CHECK4 ; CHECK4: llvm-rc: Preprocessing failed: Executable "intentionally-missing-executable" doesn't exist! + +;; Test --preprocessor with an argument with spaces. + +; RUN: llvm-windres -### --preprocessor "path with spaces/gcc" %p/Inputs/empty.rc %t.res | FileCheck %s --check-prefix=CHECK5 +; CHECK5: {{^}} "path with spaces/gcc" "{{.*}}empty.rc" "-o" "{{.*}}preproc-{{.*}}.rc"{{$}} diff --git a/llvm/tools/llvm-rc/llvm-rc.cpp b/llvm/tools/llvm-rc/llvm-rc.cpp index 27fb0309e0ee5..78ab96492acc7 100644 --- a/llvm/tools/llvm-rc/llvm-rc.cpp +++ b/llvm/tools/llvm-rc/llvm-rc.cpp @@ -209,7 +209,7 @@ struct RcOptions { bool Preprocess = true; bool PrintCmdAndExit = false; std::string Triple; - std::vector PreprocessCmd; + std::optional Preprocessor; std::vector PreprocessArgs; std::string InputFile; @@ -229,7 +229,7 @@ struct RcOptions { void preprocess(StringRef Src, StringRef Dst, const RcOptions &Opts, const char *Argv0) { std::string Clang; - if (Opts.PrintCmdAndExit || !Opts.PreprocessCmd.empty()) { + if (Opts.PrintCmdAndExit || Opts.Preprocessor) { Clang = "clang"; } else { ErrorOr ClangOrErr = findClang(Argv0, Opts.Triple); @@ -249,10 +249,9 @@ void preprocess(StringRef Src, StringRef Dst, const RcOptions &Opts, Clang, "--driver-mode=gcc", "-target", Opts.Triple, "-E", "-xc", "-DRC_INVOKED"}; std::string PreprocessorExecutable; - if (!Opts.PreprocessCmd.empty()) { + if (Opts.Preprocessor) { Args.clear(); - for (const auto &S : Opts.PreprocessCmd) - Args.push_back(S); + Args.push_back(*Opts.Preprocessor); if (!sys::fs::can_execute(Args[0])) { if (auto P = sys::findProgramByName(Args[0])) { PreprocessorExecutable = *P; @@ -342,36 +341,6 @@ std::string unescape(StringRef S) { return Out; } -std::vector unescapeSplit(StringRef S) { - std::vector OutArgs; - std::string Out; - bool InQuote = false; - for (int I = 0, E = S.size(); I < E; I++) { - if (S[I] == '\\') { - if (I + 1 < E) - Out.push_back(S[++I]); - else - fatalError("Unterminated escape"); - continue; - } - if (S[I] == '"') { - InQuote = !InQuote; - continue; - } - if (S[I] == ' ' && !InQuote) { - OutArgs.push_back(Out); - Out.clear(); - continue; - } - Out.push_back(S[I]); - } - if (InQuote) - fatalError("Unterminated quote"); - if (!Out.empty()) - OutArgs.push_back(Out); - return OutArgs; -} - RcOptions parseWindresOptions(ArrayRef ArgsArr, ArrayRef InputArgsArray, std::string Prefix) { @@ -506,11 +475,8 @@ RcOptions parseWindresOptions(ArrayRef ArgsArr, break; } } - // TODO: If --use-temp-file is set, we shouldn't be unescaping - // the --preprocessor argument either, only splitting it. if (InputArgs.hasArg(WINDRES_preprocessor)) - Opts.PreprocessCmd = - unescapeSplit(InputArgs.getLastArgValue(WINDRES_preprocessor)); + Opts.Preprocessor = InputArgs.getLastArgValue(WINDRES_preprocessor); Opts.Params.CodePage = CpWin1252; // Different default if (InputArgs.hasArg(WINDRES_codepage)) { From 428660cfb986dd0a59cd2a16972c5f7109080522 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Mon, 18 Dec 2023 10:52:42 +0000 Subject: [PATCH 092/884] [GitHub] Don't indent comment that revision has passed the formatting check Due to the way the f string was written, the text ended up with 4 spaces at the start. 4 space indent in Markdown means plain text, which is not what we intend here. --- llvm/utils/git/code-format-helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py index 697a27ab82a75..849ae996f7339 100755 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -131,8 +131,8 @@ def run(self, changed_files: List[str], args: FormatArgs) -> bool: if diff is None: if should_update_gh: comment_text = f""" - :white_check_mark: With the latest revision this PR passed the {self.friendly_name}. - """ +:white_check_mark: With the latest revision this PR passed the {self.friendly_name}. +""" self.update_pr(comment_text, args, create_new=False) return True elif len(diff) > 0: From 6deb5d4e440f34e6838eaf617198caea7ce721c5 Mon Sep 17 00:00:00 2001 From: Dominik Adamski Date: Mon, 18 Dec 2023 11:59:35 +0100 Subject: [PATCH 093/884] [NFC][OpenMP][MLIR] Verify if empty workshare loop is lowered correctly (#75518) Check if workshare loop without loop body is lowered correctly i.e.: 1) null pointer is passed to OpenMP device RTL function as a parameter which denotes loop function body aggregated parameters 2) Outlined loop function body has only one parameter - loop counter --- mlir/test/Target/LLVMIR/omptarget-wsloop.mlir | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir index ba641be4dada1..220eb85b3483e 100644 --- a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir @@ -15,6 +15,16 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo } llvm.return } + + llvm.func @target_empty_wsloop(){ + %loop_ub = llvm.mlir.constant(9 : i32) : i32 + %loop_lb = llvm.mlir.constant(0 : i32) : i32 + %loop_step = llvm.mlir.constant(1 : i32) : i32 + omp.wsloop for (%loop_cnt) : i32 = (%loop_lb) to (%loop_ub) inclusive step (%loop_step) { + omp.yield + } + llvm.return + } } // CHECK: define void @[[FUNC0:.*]](ptr %[[ARG0:.*]]) @@ -31,3 +41,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo // CHECK: %[[GEP3:.*]] = getelementptr [10 x i32], ptr %[[LOADGEP]], i32 0, i32 %[[TMP2:.*]] // CHECK: store i32 %[[VAL0:.*]], ptr %[[GEP3]], align 4 +// CHECK: define void @[[FUNC_EMPTY_WSLOOP:.*]]() +// CHECK: call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB2:[0-9]+]] to ptr), ptr @[[LOOP_EMPTY_BODY_FN:.*]], ptr null, i32 10, i32 %[[NUM_THREADS:.*]], i32 0) + +// CHECK: define internal void @[[LOOP_EMPTY_BODY_FN]](i32 %[[LOOP_CNT:.*]]) From 1faa1cd02d569c1e58303ab4bd9d082aecda2a52 Mon Sep 17 00:00:00 2001 From: Momchil Velikov Date: Mon, 18 Dec 2023 11:03:39 +0000 Subject: [PATCH 094/884] [Clang][SVE2.1] Add intrinsics for `WHILEcc` resulting in predicate pair (#75107) Add intrinsics of the form: svboolx2_t svwhile_b{8,16,32,64}_[{s,u}64]_x2([u]int64_t, [u]int64_t); and their overloaded variants as specified in https://github.com/ARM-software/acle/pull/257 --- clang/include/clang/Basic/arm_sve.td | 12 + .../acle_sve2p1_while_x2.c | 879 ++++++++++++++++++ 2 files changed, 891 insertions(+) create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_x2.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 98434c5c53e28..a429a3c5fe378 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1341,6 +1341,18 @@ def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNon def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>; } +let TargetGuard = "sve2p1|sme2" in { + def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege_x2">; + def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt_x2">; + def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi_x2">; + def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs_x2">; + def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele_x2">; + def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt_x2">; + def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo_x2">; + def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels_x2">; + +} + //////////////////////////////////////////////////////////////////////////////// // SVE2 - Uniform DSP operations diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_x2.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_x2.c new file mode 100644 index 0000000000000..acead9be3f01d --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_x2.c @@ -0,0 +1,879 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -o /dev/null %s +#include + +#ifdef SVE_OVERLOADED_FORMS +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3 +#endif + +// CHECK-LABEL: define dso_local @test_svwhilege_b8_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilege_b8_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0:[0-9]+]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilege_b8_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b8,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b8_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilege_b8_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilege_b8_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b8,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b16_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b16_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b16_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b16,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b16_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b16_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b16_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b16,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b32_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b32_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b32_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b32,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b32_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b32_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b32_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b32,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b64_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b64_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilege.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b64_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b64,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilege_b64_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilege_b64_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehs.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilege_b64_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilege_b64,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b8_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilegt_b8_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilegt_b8_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b8,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b8_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilegt_b8_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilegt_b8_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b8,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b16_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b16_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b16_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b16,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b16_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b16_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b16_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b16,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b32_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b32_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b32_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b32,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b32_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b32_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b32_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b32,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b64_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b64_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilegt.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b64_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b64,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilegt_b64_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilegt_b64_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilehi.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilegt_b64_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilegt_b64,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b8_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilele_b8_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilele_b8_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b8,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b8_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilele_b8_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilele_b8_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b8,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b16_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b16_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b16_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b16,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b16_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b16_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b16_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b16,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b32_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b32_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b32_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b32,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b32_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b32_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b32_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b32,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b64_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b64_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilele.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b64_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b64,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilele_b64_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilele_b64_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilels.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilele_b64_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilele_b64,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b8_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilelt_b8_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilelt_b8_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b8,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b8_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: define dso_local @_Z21test_svwhilelt_b8_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv16i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svwhilelt_b8_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b8,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b16_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b16_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b16_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b16,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b16_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b16_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv8i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b16_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b16,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b32_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b32_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b32_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b32,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b32_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b32_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv4i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b32_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b32,_u64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b64_s64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b64_s64ll( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelt.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b64_s64(int64_t op1, int64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b64,_s64,_x2)(op1, op2); +} + +// CHECK-LABEL: define dso_local @test_svwhilelt_b64_u64( +// CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: define dso_local @_Z22test_svwhilelt_b64_u64mm( +// CPP-CHECK-SAME: i64 noundef [[OP1:%.*]], i64 noundef [[OP2:%.*]]) #[[ATTR0]] { +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.whilelo.x2.nxv2i1(i64 [[OP1]], i64 [[OP2]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svwhilelt_b64_u64(uint64_t op1, uint64_t op2) { + return SVE_ACLE_FUNC(svwhilelt_b64,_u64,_x2)(op1, op2); +} From a34db9bdefe6d60b46f485aa1105ef6c95542e16 Mon Sep 17 00:00:00 2001 From: Jakub Chlanda Date: Mon, 18 Dec 2023 12:07:22 +0100 Subject: [PATCH 095/884] [AMDGPU][NFC] Simplify needcopysign logic (#75176) This was caught by coverity, reported as: `dead_error_condition`. Since the conditional revolves around `CF`, it is guaranteed to be null in the else clause, hence making the second part of the statement redundant. --- llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp index 0c21382e5c225..f03e6b8915b13 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp @@ -1050,8 +1050,7 @@ bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, CF->isNegative(); } else { needlog = true; - needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && - (!CF || CF->isNegative()); + needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; } } else { ConstantDataVector *CDV = dyn_cast(opr0); From a4deb14e353cd8c543f78979ce986f98023d36dd Mon Sep 17 00:00:00 2001 From: Kiran Chandramohan Date: Mon, 18 Dec 2023 11:19:32 +0000 Subject: [PATCH 096/884] [Flang][OpenMP] Add check-dag to private-clause-fixes test --- .../OpenMP/parallel-private-clause-fixes.f90 | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 index d713129675791..8533106b7ac48 100644 --- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 +++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 @@ -4,20 +4,20 @@ ! CHECK-LABEL: multiple_private_fix ! CHECK-SAME: %[[GAMA:.*]]: !fir.ref {fir.bindc_name = "gama"} -! CHECK: %[[GAMA_DECL:.*]]:2 = hlfir.declare %[[GAMA]] {uniq_name = "_QFmultiple_private_fixEgama"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_private_fixEi"} -! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFmultiple_private_fixEj"} -! CHECK: %[[J_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"} -! CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[GAMA_DECL:.*]]:2 = hlfir.declare %[[GAMA]] {uniq_name = "_QFmultiple_private_fixEgama"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_private_fixEi"} +! CHECK-DAG: %[[I_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFmultiple_private_fixEj"} +! CHECK-DAG: %[[J_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"} +! CHECK-DAG: %[[X_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: omp.parallel { -! CHECK: %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} -! CHECK: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! CHECK: %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFmultiple_private_fixEj"} -! CHECK: %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! CHECK: %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned -! CHECK: %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK-DAG: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFmultiple_private_fixEj"} +! CHECK-DAG: %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK-DAG: %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned +! CHECK-DAG: %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[ONE:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_3:.*]] = fir.load %[[GAMA_DECL]]#0 : !fir.ref ! CHECK: %[[VAL_5:.*]] = arith.constant 1 : i32 From 7c1d8c74e8576c12da526f589c802faa91bc410a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 12:20:54 +0100 Subject: [PATCH 097/884] [ValueTracking] Add test for non-zero sub via known non equal (NFC) --- .../Analysis/ValueTracking/known-non-zero.ll | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/llvm/test/Analysis/ValueTracking/known-non-zero.ll b/llvm/test/Analysis/ValueTracking/known-non-zero.ll index dbec47ea0ae26..d8dbd1a1b0bda 100644 --- a/llvm/test/Analysis/ValueTracking/known-non-zero.ll +++ b/llvm/test/Analysis/ValueTracking/known-non-zero.ll @@ -1218,3 +1218,19 @@ define <2 x i1> @cmp_excludes_zero_with_nonsplat_vec_fail(<2 x i8> %a, <2 x i8> ret <2 x i1> %r } +define i1 @sub_via_non_eq(i8 %x, i8 %y) { +; CHECK-LABEL: @sub_via_non_eq( +; CHECK-NEXT: [[NE:%.*]] = icmp ne i8 [[X:%.*]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[NE]]) +; CHECK-NEXT: [[SHL:%.*]] = shl nuw i8 [[X]], 3 +; CHECK-NEXT: [[SUB:%.*]] = sub i8 [[X]], [[SHL]] +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[SUB]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %ne = icmp ne i8 %x, 0 + call void @llvm.assume(i1 %ne) + %shl = shl nuw i8 %x, 3 + %sub = sub i8 %x, %shl + %cmp = icmp eq i8 %sub, 0 + ret i1 %cmp +} From 337504683efa0b898bcf69b4e5be67d73dbaf180 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 09:58:23 +0100 Subject: [PATCH 098/884] [ValueTracking] Use isKnownNonEqual() in isNonZeroSub() (x - y) != 0 is true iff x != y, so use the isKnownNonEqual() helper, which knows some additional tricks. --- llvm/lib/Analysis/ValueTracking.cpp | 11 ++--------- llvm/test/Analysis/ValueTracking/known-non-zero.ll | 5 +---- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 9ae05a4b5ccc7..45fdd4eda47d7 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -2368,19 +2368,12 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth, static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth, const SimplifyQuery &Q, unsigned BitWidth, Value *X, Value *Y) { + // TODO: Move this case into isKnownNonEqual(). if (auto *C = dyn_cast(X)) if (C->isNullValue() && isKnownNonZero(Y, DemandedElts, Depth, Q)) return true; - KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q); - if (XKnown.isUnknown()) - return false; - KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q); - // If X != Y then X - Y is non zero. - std::optional ne = KnownBits::ne(XKnown, YKnown); - // If we are unable to compute if X != Y, we won't be able to do anything - // computing the knownbits of the sub expression so just return here. - return ne && *ne; + return ::isKnownNonEqual(X, Y, Depth, Q); } static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts, diff --git a/llvm/test/Analysis/ValueTracking/known-non-zero.ll b/llvm/test/Analysis/ValueTracking/known-non-zero.ll index d8dbd1a1b0bda..c8e17f8dcc69f 100644 --- a/llvm/test/Analysis/ValueTracking/known-non-zero.ll +++ b/llvm/test/Analysis/ValueTracking/known-non-zero.ll @@ -1222,10 +1222,7 @@ define i1 @sub_via_non_eq(i8 %x, i8 %y) { ; CHECK-LABEL: @sub_via_non_eq( ; CHECK-NEXT: [[NE:%.*]] = icmp ne i8 [[X:%.*]], 0 ; CHECK-NEXT: call void @llvm.assume(i1 [[NE]]) -; CHECK-NEXT: [[SHL:%.*]] = shl nuw i8 [[X]], 3 -; CHECK-NEXT: [[SUB:%.*]] = sub i8 [[X]], [[SHL]] -; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[SUB]], 0 -; CHECK-NEXT: ret i1 [[CMP]] +; CHECK-NEXT: ret i1 false ; %ne = icmp ne i8 %x, 0 call void @llvm.assume(i1 %ne) From 465ecf872ec6c09fe610bf161c54031f6a372e95 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 12:36:19 +0100 Subject: [PATCH 099/884] [InstCombine] Rename UndefElts -> PoisonElts (NFC) In line with updated shufflevector semantics, this represents the poison elements rather than undef elements now. This commit is a pure rename, without any logic changes. --- .../InstCombine/InstCombineCalls.cpp | 20 +-- .../InstCombine/InstCombineInternal.h | 2 +- .../InstCombine/InstCombineSelect.cpp | 4 +- .../InstCombineSimplifyDemanded.cpp | 122 +++++++++--------- .../InstCombine/InstCombineVectorOps.cpp | 17 +-- .../InstCombine/InstructionCombining.cpp | 10 +- 6 files changed, 88 insertions(+), 87 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 1539fa9a3269e..c496f9c7419b5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -357,9 +357,9 @@ Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) { // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask); - APInt UndefElts(DemandedElts.getBitWidth(), 0); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts)) + APInt PoisonElts(DemandedElts.getBitWidth(), 0); + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, + PoisonElts)) return replaceOperand(II, 0, V); return nullptr; @@ -439,12 +439,12 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) { // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask); - APInt UndefElts(DemandedElts.getBitWidth(), 0); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts)) + APInt PoisonElts(DemandedElts.getBitWidth(), 0); + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, + PoisonElts)) return replaceOperand(II, 0, V); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts, UndefElts)) + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts, + PoisonElts)) return replaceOperand(II, 1, V); return nullptr; @@ -1526,9 +1526,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // support. if (auto *IIFVTy = dyn_cast(II->getType())) { auto VWidth = IIFVTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, PoisonElts)) { if (V != II) return replaceInstUsesWith(*II, V); return II; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 1d50fa9b6bf74..f86db698ef8f1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -550,7 +550,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final bool SimplifyDemandedInstructionBits(Instruction &Inst, KnownBits &Known); Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, - APInt &UndefElts, unsigned Depth = 0, + APInt &PoisonElts, unsigned Depth = 0, bool AllowMultipleUsers = false) override; /// Canonicalize the position of binops relative to shufflevector. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 2dda46986f0fd..20bf00344b144 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2440,9 +2440,9 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) { return nullptr; unsigned NumElts = VecTy->getNumElements(); - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); APInt AllOnesEltMask(APInt::getAllOnes(NumElts)); - if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, PoisonElts)) { if (V != &Sel) return replaceInstUsesWith(Sel, V); return &Sel; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 2490f5b9b97eb..a8ed6fe1432d9 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1319,8 +1319,8 @@ Value *InstCombinerImpl::simplifyShrShlDemandedBits( } /// The specified value produces a vector with any number of elements. -/// This method analyzes which elements of the operand are undef or poison and -/// returns that information in UndefElts. +/// This method analyzes which elements of the operand are poison and +/// returns that information in PoisonElts. /// /// DemandedElts contains the set of elements that are actually used by the /// caller, and by default (AllowMultipleUsers equals false) the value is @@ -1333,7 +1333,7 @@ Value *InstCombinerImpl::simplifyShrShlDemandedBits( /// returned. This returns null if no change was made. Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, - APInt &UndefElts, + APInt &PoisonElts, unsigned Depth, bool AllowMultipleUsers) { // Cannot analyze scalable type. The number of vector elements is not a @@ -1347,16 +1347,16 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (match(V, m_Undef())) { // If the entire vector is undef or poison, just return this info. - UndefElts = EltMask; + PoisonElts = EltMask; return nullptr; } if (DemandedElts.isZero()) { // If nothing is demanded, provide poison. - UndefElts = EltMask; + PoisonElts = EltMask; return PoisonValue::get(V->getType()); } - UndefElts = 0; + PoisonElts = 0; if (auto *C = dyn_cast(V)) { // Check if this is identity. If so, return 0 since we are not simplifying @@ -1370,7 +1370,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, for (unsigned i = 0; i != VWidth; ++i) { if (!DemandedElts[i]) { // If not demanded, set to poison. Elts.push_back(Poison); - UndefElts.setBit(i); + PoisonElts.setBit(i); continue; } @@ -1379,7 +1379,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, Elts.push_back(Elt); if (isa(Elt)) // Already poison. - UndefElts.setBit(i); + PoisonElts.setBit(i); } // If we changed the constant, return it. @@ -1400,7 +1400,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // They'll be handled when it's their turn to be visited by // the main instcombine process. if (Depth != 0) - // TODO: Just compute the UndefElts information recursively. + // TODO: Just compute the PoisonElts information recursively. return nullptr; // Conservatively assume that all elements are needed. @@ -1422,8 +1422,8 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } }; - APInt UndefElts2(VWidth, 0); - APInt UndefElts3(VWidth, 0); + APInt PoisonElts2(VWidth, 0); + APInt PoisonElts3(VWidth, 0); switch (I->getOpcode()) { default: break; @@ -1449,17 +1449,17 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (i == 0 ? match(I->getOperand(i), m_Undef()) : match(I->getOperand(i), m_Poison())) { // If the entire vector is undefined, just return this info. - UndefElts = EltMask; + PoisonElts = EltMask; return nullptr; } if (I->getOperand(i)->getType()->isVectorTy()) { - APInt UndefEltsOp(VWidth, 0); - simplifyAndSetOp(I, i, DemandedElts, UndefEltsOp); + APInt PoisonEltsOp(VWidth, 0); + simplifyAndSetOp(I, i, DemandedElts, PoisonEltsOp); // gep(x, undef) is not undef, so skip considering idx ops here // Note that we could propagate poison, but we can't distinguish between // undef & poison bits ATM if (i == 0) - UndefElts |= UndefEltsOp; + PoisonElts |= PoisonEltsOp; } } @@ -1472,7 +1472,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!Idx) { // Note that we can't propagate undef elt info, because we don't know // which elt is getting updated. - simplifyAndSetOp(I, 0, DemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts2); break; } @@ -1487,7 +1487,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // was extracted from the same index in another vector with the same type, // replace this insert with that other vector. // Note: This is attempted before the call to simplifyAndSetOp because that - // may change UndefElts to a value that does not match with Vec. + // may change PoisonElts to a value that does not match with Vec. Value *Vec; if (PreInsertDemandedElts == 0 && match(I->getOperand(1), @@ -1496,7 +1496,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return Vec; } - simplifyAndSetOp(I, 0, PreInsertDemandedElts, UndefElts); + simplifyAndSetOp(I, 0, PreInsertDemandedElts, PoisonElts); // If this is inserting an element that isn't demanded, remove this // insertelement. @@ -1506,7 +1506,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } // The inserted element is defined. - UndefElts.clearBit(IdxNo); + PoisonElts.clearBit(IdxNo); break; } case Instruction::ShuffleVector: { @@ -1525,12 +1525,12 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, MadeChange = true; } APInt LeftDemanded(OpWidth, 1); - APInt LHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts); - if (LHSUndefElts[0]) - UndefElts = EltMask; + APInt LHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts); + if (LHSPoisonElts[0]) + PoisonElts = EltMask; else - UndefElts.clearAllBits(); + PoisonElts.clearAllBits(); break; } @@ -1549,11 +1549,11 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } } - APInt LHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts); + APInt LHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts); - APInt RHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 1, RightDemanded, RHSUndefElts); + APInt RHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 1, RightDemanded, RHSPoisonElts); // If this shuffle does not change the vector length and the elements // demanded by this shuffle are an identity mask, then this shuffle is @@ -1579,7 +1579,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return Shuffle->getOperand(0); } - bool NewUndefElts = false; + bool NewPoisonElts = false; unsigned LHSIdx = -1u, LHSValIdx = -1u; unsigned RHSIdx = -1u, RHSValIdx = -1u; bool LHSUniform = true; @@ -1587,23 +1587,23 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, for (unsigned i = 0; i < VWidth; i++) { unsigned MaskVal = Shuffle->getMaskValue(i); if (MaskVal == -1u) { - UndefElts.setBit(i); + PoisonElts.setBit(i); } else if (!DemandedElts[i]) { - NewUndefElts = true; - UndefElts.setBit(i); + NewPoisonElts = true; + PoisonElts.setBit(i); } else if (MaskVal < OpWidth) { - if (LHSUndefElts[MaskVal]) { - NewUndefElts = true; - UndefElts.setBit(i); + if (LHSPoisonElts[MaskVal]) { + NewPoisonElts = true; + PoisonElts.setBit(i); } else { LHSIdx = LHSIdx == -1u ? i : OpWidth; LHSValIdx = LHSValIdx == -1u ? MaskVal : OpWidth; LHSUniform = LHSUniform && (MaskVal == i); } } else { - if (RHSUndefElts[MaskVal - OpWidth]) { - NewUndefElts = true; - UndefElts.setBit(i); + if (RHSPoisonElts[MaskVal - OpWidth]) { + NewPoisonElts = true; + PoisonElts.setBit(i); } else { RHSIdx = RHSIdx == -1u ? i : OpWidth; RHSValIdx = RHSValIdx == -1u ? MaskVal - OpWidth : OpWidth; @@ -1646,11 +1646,11 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return New; } } - if (NewUndefElts) { + if (NewPoisonElts) { // Add additional discovered undefs. SmallVector Elts; for (unsigned i = 0; i < VWidth; ++i) { - if (UndefElts[i]) + if (PoisonElts[i]) Elts.push_back(PoisonMaskElem); else Elts.push_back(Shuffle->getMaskValue(i)); @@ -1665,12 +1665,12 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // on the current demanded elements. SelectInst *Sel = cast(I); if (Sel->getCondition()->getType()->isVectorTy()) { - // TODO: We are not doing anything with UndefElts based on this call. + // TODO: We are not doing anything with PoisonElts based on this call. // It is overwritten below based on the other select operands. If an // element of the select condition is known undef, then we are free to // choose the output value from either arm of the select. If we know that // one of those values is undef, then the output can be undef. - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); } // Next, see if we can transform the arms of the select. @@ -1692,12 +1692,12 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } } - simplifyAndSetOp(I, 1, DemandedLHS, UndefElts2); - simplifyAndSetOp(I, 2, DemandedRHS, UndefElts3); + simplifyAndSetOp(I, 1, DemandedLHS, PoisonElts2); + simplifyAndSetOp(I, 2, DemandedRHS, PoisonElts3); // Output elements are undefined if the element from each arm is undefined. // TODO: This can be improved. See comment in select condition handling. - UndefElts = UndefElts2 & UndefElts3; + PoisonElts = PoisonElts2 & PoisonElts3; break; } case Instruction::BitCast: { @@ -1706,7 +1706,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!VTy) break; unsigned InVWidth = cast(VTy)->getNumElements(); APInt InputDemandedElts(InVWidth, 0); - UndefElts2 = APInt(InVWidth, 0); + PoisonElts2 = APInt(InVWidth, 0); unsigned Ratio; if (VWidth == InVWidth) { @@ -1735,25 +1735,25 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, break; } - simplifyAndSetOp(I, 0, InputDemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, InputDemandedElts, PoisonElts2); if (VWidth == InVWidth) { - UndefElts = UndefElts2; + PoisonElts = PoisonElts2; } else if ((VWidth % InVWidth) == 0) { // If the number of elements in the output is a multiple of the number of // elements in the input then an output element is undef if the // corresponding input element is undef. for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx) - if (UndefElts2[OutIdx / Ratio]) - UndefElts.setBit(OutIdx); + if (PoisonElts2[OutIdx / Ratio]) + PoisonElts.setBit(OutIdx); } else if ((InVWidth % VWidth) == 0) { // If the number of elements in the input is a multiple of the number of // elements in the output then an output element is undef if all of the // corresponding input elements are undef. for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx) { - APInt SubUndef = UndefElts2.lshr(OutIdx * Ratio).zextOrTrunc(Ratio); + APInt SubUndef = PoisonElts2.lshr(OutIdx * Ratio).zextOrTrunc(Ratio); if (SubUndef.popcount() == Ratio) - UndefElts.setBit(OutIdx); + PoisonElts.setBit(OutIdx); } } else { llvm_unreachable("Unimp"); @@ -1762,7 +1762,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } case Instruction::FPTrunc: case Instruction::FPExt: - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); break; case Instruction::Call: { @@ -1785,18 +1785,18 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, DemandedPassThrough.clearBit(i); } if (II->getIntrinsicID() == Intrinsic::masked_gather) - simplifyAndSetOp(II, 0, DemandedPtrs, UndefElts2); - simplifyAndSetOp(II, 3, DemandedPassThrough, UndefElts3); + simplifyAndSetOp(II, 0, DemandedPtrs, PoisonElts2); + simplifyAndSetOp(II, 3, DemandedPassThrough, PoisonElts3); // Output elements are undefined if the element from both sources are. // TODO: can strengthen via mask as well. - UndefElts = UndefElts2 & UndefElts3; + PoisonElts = PoisonElts2 & PoisonElts3; break; } default: { // Handle target specific intrinsics std::optional V = targetSimplifyDemandedVectorEltsIntrinsic( - *II, DemandedElts, UndefElts, UndefElts2, UndefElts3, + *II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3, simplifyAndSetOp); if (V) return *V; @@ -1859,17 +1859,17 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return ShufBO; } - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); - simplifyAndSetOp(I, 1, DemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); + simplifyAndSetOp(I, 1, DemandedElts, PoisonElts2); // Output elements are undefined if both are undefined. Consider things // like undef & 0. The result is known zero, not undef. - UndefElts &= UndefElts2; + PoisonElts &= PoisonElts2; } // If we've proven all of the lanes undef, return an undef value. // TODO: Intersect w/demanded lanes - if (UndefElts.isAllOnes()) + if (PoisonElts.isAllOnes()) return UndefValue::get(I->getType()); return MadeChange ? I : nullptr; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c8b58c51d4e6e..c381d25011f68 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -581,20 +581,20 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // If the input vector has a single use, simplify it based on this use // property. if (SrcVec->hasOneUse()) { - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); APInt DemandedElts(NumElts, 0); DemandedElts.setBit(IndexC->getZExtValue()); if (Value *V = - SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts)) + SimplifyDemandedVectorElts(SrcVec, DemandedElts, PoisonElts)) return replaceOperand(EI, 0, V); } else { // If the input vector has multiple uses, simplify it based on a union // of all elements used. APInt DemandedElts = findDemandedEltsByAllUsers(SrcVec); if (!DemandedElts.isAllOnes()) { - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); if (Value *V = SimplifyDemandedVectorElts( - SrcVec, DemandedElts, UndefElts, 0 /* Depth */, + SrcVec, DemandedElts, PoisonElts, 0 /* Depth */, true /* AllowMultipleUsers */)) { if (V != SrcVec) { Worklist.addValue(SrcVec); @@ -1713,9 +1713,10 @@ Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) { if (auto VecTy = dyn_cast(VecOp->getType())) { unsigned VWidth = VecTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(&IE, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&IE, AllOnesEltMask, + PoisonElts)) { if (V != &IE) return replaceInstUsesWith(IE, V); return &IE; @@ -2855,9 +2856,9 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { if (Instruction *I = foldCastShuffle(SVI, Builder)) return I; - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, PoisonElts)) { if (V != &SVI) return replaceInstUsesWith(SVI, V); return &SVI; diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index a7ddadc25de43..94f60719b78ca 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -173,14 +173,14 @@ std::optional InstCombiner::targetSimplifyDemandedUseBitsIntrinsic( } std::optional InstCombiner::targetSimplifyDemandedVectorEltsIntrinsic( - IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, - APInt &UndefElts3, + IntrinsicInst &II, APInt DemandedElts, APInt &PoisonElts, + APInt &PoisonElts2, APInt &PoisonElts3, std::function SimplifyAndSetOp) { // Handle target specific intrinsics if (II.getCalledFunction()->isTargetIntrinsic()) { return TTI.simplifyDemandedVectorEltsIntrinsic( - *this, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, + *this, II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3, SimplifyAndSetOp); } return std::nullopt; @@ -2241,10 +2241,10 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { // compile-time. if (auto *GEPFVTy = dyn_cast(GEPType)) { auto VWidth = GEPFVTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask, - UndefElts)) { + PoisonElts)) { if (V != &GEP) return replaceInstUsesWith(GEP, V); return &GEP; From 42239d2e96c5683a2973c39ea7ea20b616218b66 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Mon, 18 Dec 2023 12:37:29 +0100 Subject: [PATCH 100/884] [clang] Fix CTAD not respect default template arguments that were added after the definition. (#75569) Fixes https://github.com/llvm/llvm-project/issues/69987 --- clang/docs/ReleaseNotes.rst | 3 +++ clang/lib/Sema/SemaTemplate.cpp | 28 ++++++++++++++++------------ clang/test/SemaTemplate/ctad.cpp | 10 ++++++++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 26ba4f8f72508..2e32f8b36d23d 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -688,6 +688,9 @@ Bug Fixes in This Version (`#62157 `_) and (`#64885 `_) and (`#65568 `_) +- Fix an issue where clang doesn't respect detault template arguments that + are added in a later redeclaration for CTAD. + Fixes (#69987 `_) Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index f10abeaba0d45..5fcc39ec70052 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -1824,6 +1824,15 @@ static void SetNestedNameSpecifier(Sema &S, TagDecl *T, T->setQualifierInfo(SS.getWithLocInContext(S.Context)); } +// Returns the template parameter list with all default template argument +// information. +static TemplateParameterList *GetTemplateParameterList(TemplateDecl *TD) { + // Make sure we get the template parameter list from the most + // recent declaration, since that is the only one that is guaranteed to + // have all the default template argument information. + return cast(TD->getMostRecentDecl())->getTemplateParameters(); +} + DeclResult Sema::CheckClassTemplate( Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc, @@ -2061,13 +2070,13 @@ DeclResult Sema::CheckClassTemplate( if (!(TUK == TUK_Friend && CurContext->isDependentContext()) && CheckTemplateParameterList( TemplateParams, - PrevClassTemplate - ? PrevClassTemplate->getMostRecentDecl()->getTemplateParameters() - : nullptr, + PrevClassTemplate ? GetTemplateParameterList(PrevClassTemplate) + : nullptr, (SS.isSet() && SemanticContext && SemanticContext->isRecord() && SemanticContext->isDependentContext()) ? TPC_ClassTemplateMember - : TUK == TUK_Friend ? TPC_FriendClassTemplate : TPC_ClassTemplate, + : TUK == TUK_Friend ? TPC_FriendClassTemplate + : TPC_ClassTemplate, SkipBody)) Invalid = true; @@ -2298,7 +2307,7 @@ struct ConvertConstructorToDeductionGuideTransform { // -- The template parameters are the template parameters of the class // template followed by the template parameters (including default // template arguments) of the constructor, if any. - TemplateParameterList *TemplateParams = Template->getTemplateParameters(); + TemplateParameterList *TemplateParams = GetTemplateParameterList(Template); if (FTD) { TemplateParameterList *InnerParams = FTD->getTemplateParameters(); SmallVector AllParams; @@ -2424,7 +2433,7 @@ struct ConvertConstructorToDeductionGuideTransform { Params.push_back(NewParam); } - return buildDeductionGuide(Template->getTemplateParameters(), nullptr, + return buildDeductionGuide(GetTemplateParameterList(Template), nullptr, ExplicitSpecifier(), TSI, Loc, Loc, Loc); } @@ -5956,12 +5965,7 @@ bool Sema::CheckTemplateArgumentList( // template. TemplateArgumentListInfo NewArgs = TemplateArgs; - // Make sure we get the template parameter list from the most - // recent declaration, since that is the only one that is guaranteed to - // have all the default template argument information. - TemplateParameterList *Params = - cast(Template->getMostRecentDecl()) - ->getTemplateParameters(); + TemplateParameterList *Params = GetTemplateParameterList(Template); SourceLocation RAngleLoc = NewArgs.getRAngleLoc(); diff --git a/clang/test/SemaTemplate/ctad.cpp b/clang/test/SemaTemplate/ctad.cpp index 4d836839d8c34..388ed7d4cced1 100644 --- a/clang/test/SemaTemplate/ctad.cpp +++ b/clang/test/SemaTemplate/ctad.cpp @@ -44,3 +44,13 @@ namespace Access { }; D z = {Z(), {}}; } + +namespace GH69987 { +template struct X {}; +template struct X; +X x; + +template struct Y { Y(T); }; +template struct Y ; +Y y(1); +}; From 9c18f031968aabc7698e9ab0ea94da3b665d4d5a Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Mon, 18 Dec 2023 12:43:10 +0100 Subject: [PATCH 101/884] [libc++] Adds headers to FTM. (#75699) These feature-test macros had no headers listed in their associated papers. This adds the expected headers. Fixes https://github.com/llvm/llvm-project/issues/75577 --- libcxx/include/version | 4 +- .../.version.compile.pass.cpp | 105 ------------------ .../format.version.compile.pass.cpp | 34 ++++++ .../numeric.version.compile.pass.cpp | 46 +++++++- .../generate_feature_test_macro_components.py | 10 +- 5 files changed, 83 insertions(+), 116 deletions(-) delete mode 100644 libcxx/test/std/language.support/support.limits/support.limits.general/.version.compile.pass.cpp diff --git a/libcxx/include/version b/libcxx/include/version index a91c344c43609..7aa002e257b0a 100644 --- a/libcxx/include/version +++ b/libcxx/include/version @@ -93,7 +93,7 @@ __cpp_lib_expected 202211L __cpp_lib_filesystem 201703L __cpp_lib_format 202106L __cpp_lib_format_ranges 202207L -__cpp_lib_format_uchar 202311L <> +__cpp_lib_format_uchar 202311L __cpp_lib_formatters 202302L __cpp_lib_forward_like 202207L __cpp_lib_freestanding_algorithm 202311L @@ -188,7 +188,7 @@ __cpp_lib_remove_cvref 201711L __cpp_lib_result_of_sfinae 201210L __cpp_lib_robust_nonmodifying_seq_ops 201304L __cpp_lib_sample 201603L -__cpp_lib_saturation_arithmetic 202311L <> +__cpp_lib_saturation_arithmetic 202311L __cpp_lib_scoped_lock 201703L __cpp_lib_semaphore 201907L __cpp_lib_shared_mutex 201505L diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/.version.compile.pass.cpp deleted file mode 100644 index 2486985cefaca..0000000000000 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/.version.compile.pass.cpp +++ /dev/null @@ -1,105 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// WARNING: This test was generated by generate_feature_test_macro_components.py -// and should not be edited manually. -// -// clang-format off - -// <> - -// Test the feature test macros defined by <> - -/* Constant Value - __cpp_lib_format_uchar 202311L [C++20] - __cpp_lib_saturation_arithmetic 202311L [C++26] -*/ - -#include <> -#include "test_macros.h" - -#if TEST_STD_VER < 14 - -# ifdef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should not be defined before c++20" -# endif - -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" -# endif - -#elif TEST_STD_VER == 14 - -# ifdef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should not be defined before c++20" -# endif - -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" -# endif - -#elif TEST_STD_VER == 17 - -# ifdef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should not be defined before c++20" -# endif - -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" -# endif - -#elif TEST_STD_VER == 20 - -# ifndef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should be defined in c++20" -# endif -# if __cpp_lib_format_uchar != 202311L -# error "__cpp_lib_format_uchar should have the value 202311L in c++20" -# endif - -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" -# endif - -#elif TEST_STD_VER == 23 - -# ifndef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should be defined in c++23" -# endif -# if __cpp_lib_format_uchar != 202311L -# error "__cpp_lib_format_uchar should have the value 202311L in c++23" -# endif - -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" -# endif - -#elif TEST_STD_VER > 23 - -# ifndef __cpp_lib_format_uchar -# error "__cpp_lib_format_uchar should be defined in c++26" -# endif -# if __cpp_lib_format_uchar != 202311L -# error "__cpp_lib_format_uchar should have the value 202311L in c++26" -# endif - -# if !defined(_LIBCPP_VERSION) -# ifndef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should be defined in c++26" -# endif -# if __cpp_lib_saturation_arithmetic != 202311L -# error "__cpp_lib_saturation_arithmetic should have the value 202311L in c++26" -# endif -# else // _LIBCPP_VERSION -# ifdef __cpp_lib_saturation_arithmetic -# error "__cpp_lib_saturation_arithmetic should not be defined because it is unimplemented in libc++!" -# endif -# endif - -#endif // TEST_STD_VER > 23 - diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/format.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/format.version.compile.pass.cpp index 4e2c50483c229..aa7b2f4286827 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/format.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/format.version.compile.pass.cpp @@ -18,6 +18,7 @@ /* Constant Value __cpp_lib_format 202106L [C++20] __cpp_lib_format_ranges 202207L [C++23] + __cpp_lib_format_uchar 202311L [C++20] */ #include @@ -33,6 +34,10 @@ # error "__cpp_lib_format_ranges should not be defined before c++23" # endif +# ifdef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should not be defined before c++20" +# endif + #elif TEST_STD_VER == 14 # ifdef __cpp_lib_format @@ -43,6 +48,10 @@ # error "__cpp_lib_format_ranges should not be defined before c++23" # endif +# ifdef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should not be defined before c++20" +# endif + #elif TEST_STD_VER == 17 # ifdef __cpp_lib_format @@ -53,6 +62,10 @@ # error "__cpp_lib_format_ranges should not be defined before c++23" # endif +# ifdef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should not be defined before c++20" +# endif + #elif TEST_STD_VER == 20 # if !defined(_LIBCPP_VERSION) @@ -72,6 +85,13 @@ # error "__cpp_lib_format_ranges should not be defined before c++23" # endif +# ifndef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should be defined in c++20" +# endif +# if __cpp_lib_format_uchar != 202311L +# error "__cpp_lib_format_uchar should have the value 202311L in c++20" +# endif + #elif TEST_STD_VER == 23 # if !defined(_LIBCPP_VERSION) @@ -94,6 +114,13 @@ # error "__cpp_lib_format_ranges should have the value 202207L in c++23" # endif +# ifndef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should be defined in c++23" +# endif +# if __cpp_lib_format_uchar != 202311L +# error "__cpp_lib_format_uchar should have the value 202311L in c++23" +# endif + #elif TEST_STD_VER > 23 # if !defined(_LIBCPP_VERSION) @@ -116,5 +143,12 @@ # error "__cpp_lib_format_ranges should have the value 202207L in c++26" # endif +# ifndef __cpp_lib_format_uchar +# error "__cpp_lib_format_uchar should be defined in c++26" +# endif +# if __cpp_lib_format_uchar != 202311L +# error "__cpp_lib_format_uchar should have the value 202311L in c++26" +# endif + #endif // TEST_STD_VER > 23 diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/numeric.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/numeric.version.compile.pass.cpp index 60004b06c5ff5..b510eefc69a5d 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/numeric.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/numeric.version.compile.pass.cpp @@ -15,12 +15,13 @@ // Test the feature test macros defined by -/* Constant Value - __cpp_lib_constexpr_numeric 201911L [C++20] - __cpp_lib_gcd_lcm 201606L [C++17] - __cpp_lib_interpolate 201902L [C++20] - __cpp_lib_parallel_algorithm 201603L [C++17] - __cpp_lib_ranges_iota 202202L [C++23] +/* Constant Value + __cpp_lib_constexpr_numeric 201911L [C++20] + __cpp_lib_gcd_lcm 201606L [C++17] + __cpp_lib_interpolate 201902L [C++20] + __cpp_lib_parallel_algorithm 201603L [C++17] + __cpp_lib_ranges_iota 202202L [C++23] + __cpp_lib_saturation_arithmetic 202311L [C++26] */ #include @@ -48,6 +49,10 @@ # error "__cpp_lib_ranges_iota should not be defined before c++23" # endif +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" +# endif + #elif TEST_STD_VER == 14 # ifdef __cpp_lib_constexpr_numeric @@ -70,6 +75,10 @@ # error "__cpp_lib_ranges_iota should not be defined before c++23" # endif +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" +# endif + #elif TEST_STD_VER == 17 # ifdef __cpp_lib_constexpr_numeric @@ -104,6 +113,10 @@ # error "__cpp_lib_ranges_iota should not be defined before c++23" # endif +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" +# endif + #elif TEST_STD_VER == 20 # ifndef __cpp_lib_constexpr_numeric @@ -144,6 +157,10 @@ # error "__cpp_lib_ranges_iota should not be defined before c++23" # endif +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" +# endif + #elif TEST_STD_VER == 23 # ifndef __cpp_lib_constexpr_numeric @@ -193,6 +210,10 @@ # endif # endif +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined before c++26" +# endif + #elif TEST_STD_VER > 23 # ifndef __cpp_lib_constexpr_numeric @@ -242,5 +263,18 @@ # endif # endif +# if !defined(_LIBCPP_VERSION) +# ifndef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should be defined in c++26" +# endif +# if __cpp_lib_saturation_arithmetic != 202311L +# error "__cpp_lib_saturation_arithmetic should have the value 202311L in c++26" +# endif +# else // _LIBCPP_VERSION +# ifdef __cpp_lib_saturation_arithmetic +# error "__cpp_lib_saturation_arithmetic should not be defined because it is unimplemented in libc++!" +# endif +# endif + #endif // TEST_STD_VER > 23 diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index 6a30324397883..70e18b6dde006 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -486,7 +486,9 @@ def add_version_header(tc): "values": { "c++20": 202311 # DR P2909R4 Fix formatting of code units as integers }, - "headers": [""], # Note not in format + "headers": [ + "format" # TODO verify this entry since the paper was underspecified. + ], }, { "name": "__cpp_lib_formatters", @@ -660,7 +662,7 @@ def add_version_header(tc): }, { "name": "__cpp_lib_ios_noreplace", - "values": { "c++23": 202207 }, + "values": {"c++23": 202207}, "headers": ["ios"], }, { @@ -1010,7 +1012,9 @@ def add_version_header(tc): { "name": "__cpp_lib_saturation_arithmetic", "values": {"c++26": 202311}, # P0543R3 Saturation arithmetic - "headers": [""], # Note not in + "headers": [ + "numeric" # TODO verify this entry since the paper was underspecified. + ], "unimplemented": True, }, { From 3b08b3340d5adbd001cd1b328f6fd6ecf5d92f4a Mon Sep 17 00:00:00 2001 From: Alex Bradbury Date: Mon, 18 Dec 2023 11:45:21 +0000 Subject: [PATCH 102/884] [RISCV][test] Fix lifetime bug with Module in test As pointed out by @s-barannikov in post-commit review of #72356, the module is destroyed at the end of the constructor. --- llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp index b4c96a9c2a62c..5ef86bb3f7b46 100644 --- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp +++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp @@ -31,6 +31,7 @@ class RISCVInstrInfoTest : public testing::TestWithParam { std::unique_ptr ST; std::unique_ptr MMI; std::unique_ptr MF; + std::unique_ptr M; static void SetUpTestSuite() { LLVMInitializeRISCVTargetInfo(); @@ -49,10 +50,10 @@ class RISCVInstrInfoTest : public testing::TestWithParam { CodeGenOptLevel::Default))); Ctx = std::make_unique(); - Module M("Module", *Ctx); - M.setDataLayout(TM->createDataLayout()); + M = std::make_unique("Module", *Ctx); + M->setDataLayout(TM->createDataLayout()); auto *FType = FunctionType::get(Type::getVoidTy(*Ctx), false); - auto *F = Function::Create(FType, GlobalValue::ExternalLinkage, "Test", &M); + auto *F = Function::Create(FType, GlobalValue::ExternalLinkage, "Test", *M); MMI = std::make_unique(TM.get()); ST = std::make_unique( From a00c4220be6a9ce17e2a0b88191bdd08e65bf1ad Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Mon, 18 Dec 2023 12:44:25 +0100 Subject: [PATCH 103/884] [SystemZ] Fix complex address matching when i128 is legal Complex address matching currently handles truncations, under the assumption that those are no-ops. This is no longer true when i128 is legal. Change the code to only handle actual no-op truncations. Fixes https://github.com/llvm/llvm-project/issues/75708 Fixes https://github.com/llvm/llvm-project/issues/75714 --- .../Target/SystemZ/SystemZISelDAGToDAG.cpp | 3 +- llvm/test/CodeGen/SystemZ/addr-04.ll | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/SystemZ/addr-04.ll diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index e5e1e91916f32..c7d8591c5bdf6 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -465,7 +465,8 @@ bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM, bool IsBase) const { SDValue N = IsBase ? AM.Base : AM.Index; unsigned Opcode = N.getOpcode(); - if (Opcode == ISD::TRUNCATE) { + // Look through no-op truncations. + if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) { N = N.getOperand(0); Opcode = N.getOpcode(); } diff --git a/llvm/test/CodeGen/SystemZ/addr-04.ll b/llvm/test/CodeGen/SystemZ/addr-04.ll new file mode 100644 index 0000000000000..245623fea9b9b --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/addr-04.ll @@ -0,0 +1,46 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; Test complex addresses with base or index truncated from 128 bit. +; +; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s + +; Shift amount with base truncated from 128 bit to 32 bit. +define void @f1(i128 %x, i32 %y, ptr %px, ptr %py) { +; CHECK-LABEL: f1: +; CHECK: # %bb.0: +; CHECK-NEXT: larl %r1, .LCPI0_0 +; CHECK-NEXT: vl %v0, 0(%r2), 3 +; CHECK-NEXT: vl %v1, 0(%r1), 3 +; CHECK-NEXT: vaq %v0, %v0, %v1 +; CHECK-NEXT: vlgvf %r1, %v0, 3 +; CHECK-NEXT: srl %r3, 0(%r1) +; CHECK-NEXT: vst %v0, 0(%r4), 3 +; CHECK-NEXT: st %r3, 0(%r5) +; CHECK-NEXT: br %r14 + %x1 = add i128 %x, 1 + store i128 %x1, ptr %px, align 8 + %amt = trunc i128 %x1 to i32 + %y1 = lshr i32 %y, %amt + store i32 %y1, ptr %py, align 4 + ret void +} + +; Memory address with index truncated from 128 bit to 64 bit. +define i8 @f2(ptr %base, ptr %p) { +; CHECK-LABEL: f2: +; CHECK: # %bb.0: +; CHECK-NEXT: larl %r1, .LCPI1_0 +; CHECK-NEXT: vl %v0, 0(%r3), 3 +; CHECK-NEXT: vl %v1, 0(%r1), 3 +; CHECK-NEXT: vaq %v0, %v0, %v1 +; CHECK-NEXT: vlgvg %r1, %v0, 1 +; CHECK-NEXT: vst %v0, 0(%r3), 3 +; CHECK-NEXT: lb %r2, 0(%r1,%r2) +; CHECK-NEXT: br %r14 + %idx = load i128, ptr %p, align 8 + %inc = add nsw i128 %idx, 1 + store i128 %inc, ptr %p, align 8 + %idxprom = trunc i128 %inc to i64 + %arrayidx = getelementptr inbounds i8, ptr %base, i64 %idxprom + %res = load i8, ptr %arrayidx, align 1 + ret i8 %res +} From df3ddd78f640ebec74151028d919904c6cf9ecdd Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Mon, 18 Dec 2023 11:51:13 +0000 Subject: [PATCH 104/884] CGBuiltin - fix gcc Wunused-variable warning. NFC. --- clang/lib/CodeGen/CGBuiltin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index c96f86a823a46..c42b885289c4c 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -942,7 +942,7 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, : Builder.CreateZExtOrTrunc(FAMSize, ResType); Value *Res = FAMSize; - if (const auto *DRE = dyn_cast(Base)) { + if (isa(Base)) { // The whole struct is specificed in the __bdos. const RecordDecl *OuterRD = CountedByFD->getDeclContext()->getOuterLexicalRecordContext(); From 2f8178806770b549c054b02ae3556b454cdeb722 Mon Sep 17 00:00:00 2001 From: Serge Pavlov Date: Mon, 18 Dec 2023 18:57:36 +0700 Subject: [PATCH 105/884] [ARM][FPEnv] Lowering of fpmode intrinsics (#74054) LLVM intrinsics `get_fpmode`, `set_fpmode` and `reset_fpmode` operate control modes, the bits of FP environment that affect FP operations. On ARM these bits are in FPSCR together with the status bits. The implementation of these intrinsics produces code close to that of functions `fegetmode` and `fesetmode` from GLIBC. Pull request: https://github.com/llvm/llvm-project/pull/74054 --- llvm/lib/Target/ARM/ARMISelLowering.cpp | 58 +++++++++++++++ llvm/lib/Target/ARM/ARMISelLowering.h | 10 +++ llvm/lib/Target/ARM/ARMInstrVFP.td | 1 + llvm/test/CodeGen/ARM/fpenv.ll | 98 +++++++++++++++++++++++++ 4 files changed, 167 insertions(+) diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index db63facca870f..d00b7853816e1 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -1415,6 +1415,9 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setOperationAction(ISD::GET_FPENV, MVT::i32, Legal); setOperationAction(ISD::SET_FPENV, MVT::i32, Legal); setOperationAction(ISD::RESET_FPENV, MVT::Other, Legal); + setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal); + setOperationAction(ISD::SET_FPMODE, MVT::i32, Custom); + setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom); } // We want to custom lower some of our intrinsics. @@ -6447,6 +6450,57 @@ SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op, return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); } +SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op->getOperand(0); + SDValue Mode = Op->getOperand(1); + + // Generate nodes to build: + // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits) + SDValue Ops[] = {Chain, + DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)}; + SDValue FPSCR = + DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); + Chain = FPSCR.getValue(1); + FPSCR = FPSCR.getValue(0); + + SDValue FPSCRMasked = + DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR, + DAG.getConstant(ARM::FPStatusBits, DL, MVT::i32)); + SDValue InputMasked = + DAG.getNode(ISD::AND, DL, MVT::i32, Mode, + DAG.getConstant(~ARM::FPStatusBits, DL, MVT::i32)); + FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCRMasked, InputMasked); + + SDValue Ops2[] = { + Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR}; + return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); +} + +SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op->getOperand(0); + + // To get the default FP mode all control bits are cleared: + // FPSCR = FPSCR & (FPStatusBits | FPReservedBits) + SDValue Ops[] = {Chain, + DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)}; + SDValue FPSCR = + DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); + Chain = FPSCR.getValue(1); + FPSCR = FPSCR.getValue(0); + + SDValue FPSCRMasked = DAG.getNode( + ISD::AND, DL, MVT::i32, FPSCR, + DAG.getConstant(ARM::FPStatusBits | ARM::FPReservedBits, DL, MVT::i32)); + SDValue Ops2[] = {Chain, + DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), + FPSCRMasked}; + return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); +} + static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) { SDLoc dl(N); @@ -10557,6 +10611,10 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget); case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG); case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG); + case ISD::SET_FPMODE: + return LowerSET_FPMODE(Op, DAG); + case ISD::RESET_FPMODE: + return LowerRESET_FPMODE(Op, DAG); case ISD::MUL: return LowerMUL(Op, DAG); case ISD::SDIV: if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index 6c2b92de7a1df..f398b01f41862 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -375,6 +375,14 @@ class VectorType; // Bit position of rounding mode bits in FPSCR. const unsigned RoundingBitsPos = 22; + + // Bits of floating-point status. These are NZCV flags, QC bit and cumulative + // FP exception bits. + const unsigned FPStatusBits = 0xf800009f; + + // Some bits in the FPSCR are not yet defined. They must be preserved when + // modifying the contents. + const unsigned FPReservedBits = 0x00006060; } // namespace ARM /// Define some predicates that are used for node matching. @@ -835,6 +843,8 @@ class VectorType; SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerSET_FPMODE(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerRESET_FPMODE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerConstantFP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST) const; SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td index 800527bcf756c..55d3efbd9b9a2 100644 --- a/llvm/lib/Target/ARM/ARMInstrVFP.td +++ b/llvm/lib/Target/ARM/ARMInstrVFP.td @@ -2675,6 +2675,7 @@ def : Pat<(get_fpenv), (VMRS)>; def : Pat<(set_fpenv GPRnopc:$Rt), (VMSR GPRnopc:$Rt)>; def : Pat<(reset_fpenv), (VMSR (MOVi 0))>, Requires<[IsARM]>; def : Pat<(reset_fpenv), (VMSR (tMOVi8 0))>, Requires<[IsThumb]>; +def : Pat<(get_fpmode), (VMRS)>; //===----------------------------------------------------------------------===// // Assembler aliases. diff --git a/llvm/test/CodeGen/ARM/fpenv.ll b/llvm/test/CodeGen/ARM/fpenv.ll index 40db627ebb3c2..f5d87170d9153 100644 --- a/llvm/test/CodeGen/ARM/fpenv.ll +++ b/llvm/test/CodeGen/ARM/fpenv.ll @@ -142,9 +142,107 @@ entry: ret void } +define i32 @get_fpmode_01() #0 { +; CHECK-LABEL: get_fpmode_01: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r11, lr} +; CHECK-NEXT: push {r11, lr} +; CHECK-NEXT: .pad #8 +; CHECK-NEXT: sub sp, sp, #8 +; CHECK-NEXT: add r0, sp, #4 +; CHECK-NEXT: bl fegetmode +; CHECK-NEXT: ldr r0, [sp, #4] +; CHECK-NEXT: add sp, sp, #8 +; CHECK-NEXT: pop {r11, lr} +; CHECK-NEXT: mov pc, lr +entry: + %fpenv = call i32 @llvm.get.fpmode.i32() + ret i32 %fpenv +} + +define i32 @get_fpmode_02() nounwind { +; CHECK-LABEL: get_fpmode_02: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmrs r0, fpscr +; CHECK-NEXT: mov pc, lr +entry: + %fpenv = call i32 @llvm.get.fpmode.i32() + ret i32 %fpenv +} + +define void @set_fpmode_01(i32 %fpmode) #0 { +; CHECK-LABEL: set_fpmode_01: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r11, lr} +; CHECK-NEXT: push {r11, lr} +; CHECK-NEXT: .pad #8 +; CHECK-NEXT: sub sp, sp, #8 +; CHECK-NEXT: str r0, [sp, #4] +; CHECK-NEXT: add r0, sp, #4 +; CHECK-NEXT: bl fesetmode +; CHECK-NEXT: add sp, sp, #8 +; CHECK-NEXT: pop {r11, lr} +; CHECK-NEXT: mov pc, lr +entry: + call void @llvm.set.fpmode.i32(i32 %fpmode) + ret void +} + +define void @set_fpmode_02(i32 %fpmode) nounwind { +; CHECK-LABEL: set_fpmode_02: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmrs r1, fpscr +; CHECK-NEXT: mvn r2, #159 +; CHECK-NEXT: sub r2, r2, #-134217728 +; CHECK-NEXT: and r0, r0, r2 +; CHECK-NEXT: mov r2, #159 +; CHECK-NEXT: orr r2, r2, #-134217728 +; CHECK-NEXT: and r1, r1, r2 +; CHECK-NEXT: orr r0, r1, r0 +; CHECK-NEXT: vmsr fpscr, r0 +; CHECK-NEXT: mov pc, lr +entry: + call void @llvm.set.fpmode.i32(i32 %fpmode) + ret void +} + +define void @reset_fpmode_01() #0 { +; CHECK-LABEL: reset_fpmode_01: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r11, lr} +; CHECK-NEXT: push {r11, lr} +; CHECK-NEXT: mvn r0, #0 +; CHECK-NEXT: bl fesetmode +; CHECK-NEXT: pop {r11, lr} +; CHECK-NEXT: mov pc, lr +entry: + call void @llvm.reset.fpmode() + ret void +} + +define void @reset_fpmode_02() nounwind { +; CHECK-LABEL: reset_fpmode_02: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmrs r0, fpscr +; CHECK-NEXT: ldr r1, .LCPI16_0 +; CHECK-NEXT: and r0, r0, r1 +; CHECK-NEXT: vmsr fpscr, r0 +; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: .p2align 2 +; CHECK-NEXT: @ %bb.1: +; CHECK-NEXT: .LCPI16_0: +; CHECK-NEXT: .long 4160774399 @ 0xf80060ff +entry: + call void @llvm.reset.fpmode() + ret void +} + attributes #0 = { nounwind "use-soft-float"="true" } declare void @llvm.set.rounding(i32) declare i32 @llvm.get.fpenv.i32() declare void @llvm.set.fpenv.i32(i32 %fpenv) declare void @llvm.reset.fpenv() +declare i32 @llvm.get.fpmode.i32() +declare void @llvm.set.fpmode.i32(i32 %fpmode) +declare void @llvm.reset.fpmode() From dea16ebd2613a4a218c53045270fc4fcc9b427ad Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Mon, 18 Dec 2023 11:58:42 +0000 Subject: [PATCH 106/884] [LLVM][IR] Replace ConstantInt's specialisation of getType() with getIntegerType(). (#75217) The specialisation will not be valid when ConstantInt gains native support for vector types. This is largely a mechanical change but with extra attention paid to constant folding, InstCombineVectorOps.cpp, LoopFlatten.cpp and Verifier.cpp to remove the need to call `getIntegerType()`. Co-authored-by: Nikita Popov --- clang/lib/CodeGen/CGBuiltin.cpp | 7 ++++--- llvm/include/llvm/IR/Constants.h | 7 +++---- llvm/lib/Analysis/InstructionSimplify.cpp | 2 +- llvm/lib/IR/ConstantFold.cpp | 2 +- llvm/lib/IR/Verifier.cpp | 11 ++++++----- .../Hexagon/HexagonLoopIdiomRecognition.cpp | 6 +++--- llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 15 ++++++++------- .../InstCombine/InstCombineVectorOps.cpp | 4 ++-- llvm/lib/Transforms/Scalar/ConstantHoisting.cpp | 5 ++--- llvm/lib/Transforms/Scalar/LoopFlatten.cpp | 5 ++--- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 8 ++++---- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 2 +- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index c42b885289c4c..4eb1686f09506 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3214,7 +3214,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); ConstantInt *AlignmentCI = cast(AlignmentValue); if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment)) - AlignmentCI = ConstantInt::get(AlignmentCI->getType(), + AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(), llvm::Value::MaximumAlignment); emitAlignmentAssumption(PtrValue, Ptr, @@ -17034,7 +17034,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Value *Op1 = EmitScalarExpr(E->getArg(1)); ConstantInt *AlignmentCI = cast(Op0); if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment)) - AlignmentCI = ConstantInt::get(AlignmentCI->getType(), + AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(), llvm::Value::MaximumAlignment); emitAlignmentAssumption(Op1, E->getArg(1), @@ -17272,7 +17272,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Op0, llvm::FixedVectorType::get(ConvertType(E->getType()), 2)); if (getTarget().isLittleEndian()) - Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue()); + Index = + ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue()); return Builder.CreateExtractElement(Unpacked, Index); } diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index 0b9f89830b79c..b5dcc7fbc1d92 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -171,10 +171,9 @@ class ConstantInt final : public ConstantData { /// Determine if this constant's value is same as an unsigned char. bool equalsInt(uint64_t V) const { return Val == V; } - /// getType - Specialize the getType() method to always return an IntegerType, - /// which reduces the amount of casting needed in parts of the compiler. - /// - inline IntegerType *getType() const { + /// Variant of the getType() method to always return an IntegerType, which + /// reduces the amount of casting needed in parts of the compiler. + inline IntegerType *getIntegerType() const { return cast(Value::getType()); } diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 2a45acf63aa2c..5beac5547d65e 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6079,7 +6079,7 @@ static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, Type *Int32Ty = Type::getInt32Ty(Ptr->getContext()); auto *OffsetConstInt = dyn_cast(Offset); - if (!OffsetConstInt || OffsetConstInt->getType()->getBitWidth() > 64) + if (!OffsetConstInt || OffsetConstInt->getBitWidth() > 64) return nullptr; APInt OffsetInt = OffsetConstInt->getValue().sextOrTrunc( diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp index d499d74f7ba01..7fdc35e7fca09 100644 --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -868,7 +868,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, } if (GVAlign > 1) { - unsigned DstWidth = CI2->getType()->getBitWidth(); + unsigned DstWidth = CI2->getBitWidth(); unsigned SrcWidth = std::min(DstWidth, Log2(GVAlign)); APInt BitsNotSet(APInt::getLowBitsSet(DstWidth, SrcWidth)); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 8aba28026306a..aeaca21a99cc5 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -2296,10 +2296,9 @@ void Verifier::verifyFunctionMetadata( Check(isa(MD->getOperand(0)), "expected a constant operand for !kcfi_type", MD); Constant *C = cast(MD->getOperand(0))->getValue(); - Check(isa(C), + Check(isa(C) && isa(C->getType()), "expected a constant integer operand for !kcfi_type", MD); - IntegerType *Type = cast(C)->getType(); - Check(Type->getBitWidth() == 32, + Check(cast(C)->getBitWidth() == 32, "expected a 32-bit integer constant operand for !kcfi_type", MD); } } @@ -5690,8 +5689,10 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "vector of ints"); auto *Op3 = cast(Call.getArgOperand(2)); - Check(Op3->getType()->getBitWidth() <= 32, - "third argument of [us][mul|div]_fix[_sat] must fit within 32 bits"); + Check(Op3->getType()->isIntegerTy(), + "third operand of [us][mul|div]_fix[_sat] must be an int type"); + Check(Op3->getBitWidth() <= 32, + "third operand of [us][mul|div]_fix[_sat] must fit within 32 bits"); if (ID == Intrinsic::smul_fix || ID == Intrinsic::smul_fix_sat || ID == Intrinsic::sdiv_fix || ID == Intrinsic::sdiv_fix_sat) { diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp index 51ef72b873a51..7777ae23e8aec 100644 --- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp +++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp @@ -1062,7 +1062,7 @@ void PolynomialMultiplyRecognize::promoteTo(Instruction *In, // Promote immediates. for (unsigned i = 0, n = In->getNumOperands(); i != n; ++i) { if (ConstantInt *CI = dyn_cast(In->getOperand(i))) - if (CI->getType()->getBitWidth() < DestBW) + if (CI->getBitWidth() < DestBW) In->setOperand(i, ConstantInt::get(DestTy, CI->getZExtValue())); } } @@ -1577,7 +1577,7 @@ Value *PolynomialMultiplyRecognize::generate(BasicBlock::iterator At, static bool hasZeroSignBit(const Value *V) { if (const auto *CI = dyn_cast(V)) - return (CI->getType()->getSignBit() & CI->getSExtValue()) == 0; + return CI->getValue().isNonNegative(); const Instruction *I = dyn_cast(V); if (!I) return false; @@ -1688,7 +1688,7 @@ void PolynomialMultiplyRecognize::setupPreSimplifier(Simplifier &S) { if (I->getOpcode() != Instruction::Or) return nullptr; ConstantInt *Msb = dyn_cast(I->getOperand(1)); - if (!Msb || Msb->getZExtValue() != Msb->getType()->getSignBit()) + if (!Msb || !Msb->getValue().isSignMask()) return nullptr; if (!hasZeroSignBit(I->getOperand(0))) return nullptr; diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index b2665161c090d..2c880316e0a1c 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -3763,7 +3763,7 @@ struct AAKernelInfoFunction : AAKernelInfo { ConstantInt *ExecModeC = KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC); ConstantInt *AssumedExecModeC = ConstantInt::get( - ExecModeC->getType(), + ExecModeC->getIntegerType(), ExecModeC->getSExtValue() | OMP_TGT_EXEC_MODE_GENERIC_SPMD); if (ExecModeC->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD) SPMDCompatibilityTracker.indicateOptimisticFixpoint(); @@ -3792,7 +3792,7 @@ struct AAKernelInfoFunction : AAKernelInfo { ConstantInt *MayUseNestedParallelismC = KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(KernelEnvC); ConstantInt *AssumedMayUseNestedParallelismC = ConstantInt::get( - MayUseNestedParallelismC->getType(), NestedParallelism); + MayUseNestedParallelismC->getIntegerType(), NestedParallelism); setMayUseNestedParallelismOfKernelEnvironment( AssumedMayUseNestedParallelismC); @@ -3801,7 +3801,7 @@ struct AAKernelInfoFunction : AAKernelInfo { KernelInfo::getUseGenericStateMachineFromKernelEnvironment( KernelEnvC); ConstantInt *AssumedUseGenericStateMachineC = - ConstantInt::get(UseGenericStateMachineC->getType(), false); + ConstantInt::get(UseGenericStateMachineC->getIntegerType(), false); setUseGenericStateMachineOfKernelEnvironment( AssumedUseGenericStateMachineC); } @@ -4280,8 +4280,9 @@ struct AAKernelInfoFunction : AAKernelInfo { // kernel is executed in. assert(ExecModeVal == OMP_TGT_EXEC_MODE_GENERIC && "Initially non-SPMD kernel has SPMD exec mode!"); - setExecModeOfKernelEnvironment(ConstantInt::get( - ExecModeC->getType(), ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD)); + setExecModeOfKernelEnvironment( + ConstantInt::get(ExecModeC->getIntegerType(), + ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD)); ++NumOpenMPTargetRegionKernelsSPMD; @@ -4332,7 +4333,7 @@ struct AAKernelInfoFunction : AAKernelInfo { // If not SPMD mode, indicate we use a custom state machine now. setUseGenericStateMachineOfKernelEnvironment( - ConstantInt::get(UseStateMachineC->getType(), false)); + ConstantInt::get(UseStateMachineC->getIntegerType(), false)); // If we don't actually need a state machine we are done here. This can // happen if there simply are no parallel regions. In the resulting kernel @@ -4658,7 +4659,7 @@ struct AAKernelInfoFunction : AAKernelInfo { KernelInfo::getMayUseNestedParallelismFromKernelEnvironment( AA.KernelEnvC); ConstantInt *NewMayUseNestedParallelismC = ConstantInt::get( - MayUseNestedParallelismC->getType(), AA.NestedParallelism); + MayUseNestedParallelismC->getIntegerType(), AA.NestedParallelism); AA.setMayUseNestedParallelismOfKernelEnvironment( NewMayUseNestedParallelismC); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c381d25011f68..bd5f608045cf1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -388,7 +388,7 @@ static APInt findDemandedEltsByAllUsers(Value *V) { /// arbitrarily pick 64 bit as our canonical type. The actual bitwidth doesn't /// matter, we just want a consistent type to simplify CSE. static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) { - const unsigned IndexBW = IndexC->getType()->getBitWidth(); + const unsigned IndexBW = IndexC->getBitWidth(); if (IndexBW == 64 || IndexC->getValue().getActiveBits() > 64) return nullptr; return ConstantInt::get(IndexC->getContext(), @@ -2640,7 +2640,7 @@ static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf, assert(NewInsIndex != -1 && "Did not fold shuffle with unused operand?"); // Index is updated to the potentially translated insertion lane. - IndexC = ConstantInt::get(IndexC->getType(), NewInsIndex); + IndexC = ConstantInt::get(IndexC->getIntegerType(), NewInsIndex); return true; }; diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp index 1fb9d7fff32f6..9e40d94dd73c7 100644 --- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp +++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp @@ -674,8 +674,7 @@ void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) { llvm::stable_sort(ConstCandVec, [](const ConstantCandidate &LHS, const ConstantCandidate &RHS) { if (LHS.ConstInt->getType() != RHS.ConstInt->getType()) - return LHS.ConstInt->getType()->getBitWidth() < - RHS.ConstInt->getType()->getBitWidth(); + return LHS.ConstInt->getBitWidth() < RHS.ConstInt->getBitWidth(); return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue()); }); @@ -890,7 +889,7 @@ bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) { Type *Ty = ConstInfo.BaseExpr->getType(); Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP); } else { - IntegerType *Ty = ConstInfo.BaseInt->getType(); + IntegerType *Ty = ConstInfo.BaseInt->getIntegerType(); Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP); } diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp index b1add3c42976f..eef94636578d8 100644 --- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp +++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp @@ -343,9 +343,8 @@ static bool verifyTripCount(Value *RHS, Loop *L, // If the RHS of the compare is equal to the backedge taken count we need // to add one to get the trip count. if (SCEVRHS == BackedgeTCExt || SCEVRHS == BackedgeTakenCount) { - ConstantInt *One = ConstantInt::get(ConstantRHS->getType(), 1); - Value *NewRHS = ConstantInt::get( - ConstantRHS->getContext(), ConstantRHS->getValue() + One->getValue()); + Value *NewRHS = ConstantInt::get(ConstantRHS->getContext(), + ConstantRHS->getValue() + 1); return setLoopComponents(NewRHS, TripCount, Increment, IterationInstructions); } diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 89494a7f64971..55e375670cc61 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -6293,7 +6293,7 @@ Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { } case BitMapKind: { // Type of the bitmap (e.g. i59). - IntegerType *MapTy = BitMap->getType(); + IntegerType *MapTy = BitMap->getIntegerType(); // Cast Index to the same type as the bitmap. // Note: The Index is <= the number of elements in the table, so @@ -6668,7 +6668,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, Value *TableIndex; ConstantInt *TableIndexOffset; if (UseSwitchConditionAsTableIndex) { - TableIndexOffset = ConstantInt::get(MaxCaseVal->getType(), 0); + TableIndexOffset = ConstantInt::get(MaxCaseVal->getIntegerType(), 0); TableIndex = SI->getCondition(); } else { TableIndexOffset = MinCaseVal; @@ -6752,7 +6752,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, // Get the TableIndex'th bit of the bitmask. // If this bit is 0 (meaning hole) jump to the default destination, // else continue with table lookup. - IntegerType *MapTy = TableMask->getType(); + IntegerType *MapTy = TableMask->getIntegerType(); Value *MaskIndex = Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex"); Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted"); @@ -6975,7 +6975,7 @@ static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder, // Replace each case with its trailing zeros number. for (auto &Case : SI->cases()) { auto *OrigValue = Case.getCaseValue(); - Case.setValue(ConstantInt::get(OrigValue->getType(), + Case.setValue(ConstantInt::get(OrigValue->getIntegerType(), OrigValue->getValue().countr_zero())); } diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index ec2692f58695d..905405e939882 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -720,7 +720,7 @@ static TypedAttr getScalarConstantAsAttr(OpBuilder &builder, // Convert scalar intergers. if (auto *constInt = dyn_cast(constScalar)) { return builder.getIntegerAttr( - IntegerType::get(context, constInt->getType()->getBitWidth()), + IntegerType::get(context, constInt->getBitWidth()), constInt->getValue()); } From 8f5628eaef3f8fe2f9007fb323808838ffdf8701 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Mon, 18 Dec 2023 13:26:38 +0100 Subject: [PATCH 107/884] Reapply "[bazel] Port 2e45326b088b3b2f5c8327f6d5e61bdd2845bbbe" This reverts commit 0911f237737839dd90e77c93dd865756275aba69. The corresponding Clang change re-landed in 945c645acb9670b7b866a4abd94bcc9b0ae5d87d --- .../llvm-project-overlay/clang/BUILD.bazel | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 564ee16df91b6..4e6020791b9ad 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -180,6 +180,20 @@ gentbl( ], ) +gentbl( + name = "basic_arm_sve_streaming_attrs_gen", + tbl_outs = [( + "-gen-arm-sve-streaming-attrs", + "include/clang/Basic/arm_sve_streaming_attrs.inc", + )], + tblgen = ":clang-tblgen", + td_file = "include/clang/Basic/arm_sve.td", + td_srcs = [ + "include/clang/Basic/arm_sve.td", + "include/clang/Basic/arm_sve_sme_incl.td", + ], +) + gentbl( name = "basic_arm_sme_builtins_gen", tbl_outs = [( @@ -222,6 +236,20 @@ gentbl( ], ) +gentbl( + name = "basic_arm_sme_streaming_attrs_gen", + tbl_outs = [( + "-gen-arm-sme-streaming-attrs", + "include/clang/Basic/arm_sme_streaming_attrs.inc", + )], + tblgen = ":clang-tblgen", + td_file = "include/clang/Basic/arm_sme.td", + td_srcs = [ + "include/clang/Basic/arm_sme.td", + "include/clang/Basic/arm_sve_sme_incl.td", + ], +) + gentbl( name = "basic_arm_mve_cg_gen", tbl_outs = [( @@ -1040,8 +1068,10 @@ cc_library( ":basic_arm_cde_sema_gen", ":basic_arm_sme_builtins_gen", ":basic_arm_sme_sema_rangechecks_gen", + ":basic_arm_sme_streaming_attrs_gen", ":basic_arm_sve_builtins_gen", ":basic_arm_sve_sema_rangechecks_gen", + ":basic_arm_sve_streaming_attrs_gen", ":basic_riscv_sifive_vector_builtin_sema_gen", ":basic_riscv_vector_builtin_sema_gen", ":edit", From 9727919a2e35d2114d52f33a7751696ae1595581 Mon Sep 17 00:00:00 2001 From: antoine moynault Date: Mon, 18 Dec 2023 13:52:21 +0100 Subject: [PATCH 108/884] =?UTF-8?q?Revert=20"[RISCV]=20Implement=20multi-l?= =?UTF-8?q?ib=20reuse=20rule=20for=20RISC-V=20bare-metal=20=E2=80=A6=20(#7?= =?UTF-8?q?5789)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …toolchain (#73765)" This reverts commit 111a2290650743b27f70f9b24618411e54493b59, as it broke several bots https://lab.llvm.org/buildbot/#/builders/245/builds/18162 https://lab.llvm.org/buildbot/#/builders/188/builds/39436 https://lab.llvm.org/buildbot/#/builders/187/builds/13723 https://lab.llvm.org/buildbot/#/builders/182/builds/8449 https://lab.llvm.org/buildbot/#/builders/198/builds/7438 https://lab.llvm.org/buildbot/#/builders/176/builds/7419 https://lab.llvm.org/buildbot/#/builders/186/builds/13781 https://lab.llvm.org/buildbot/#/builders/183/builds/18116 https://lab.llvm.org/buildbot/#/builders/197/builds/11410 https://lab.llvm.org/buildbot/#/builders/184/builds/8651 When reapplying, please take care of another commit that have been merged after this one: c7cdf3cd5d74 [clang] Use 'starts_with' instead of 'startswith' in Gnu.cpp (NFC) --- clang/lib/Driver/ToolChains/Gnu.cpp | 127 +----------------- .../riscv-toolchain-gcc-multilib-reuse.c | 81 ----------- 2 files changed, 1 insertion(+), 207 deletions(-) delete mode 100644 clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index 7f463ddd17d3d..835215a83c403 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -30,7 +30,6 @@ #include "llvm/Option/ArgList.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Path.h" -#include "llvm/Support/RISCVISAInfo.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/TargetParser.h" #include @@ -1716,129 +1715,6 @@ static void findCSKYMultilibs(const Driver &D, const llvm::Triple &TargetTriple, Result.Multilibs = CSKYMultilibs; } -/// Extend the multi-lib re-use selection mechanism for RISC-V. -/// This function will try to re-use multi-lib if they are compatible. -/// Definition of compatible: -/// - ABI must be the same. -/// - multi-lib is a subset of current arch, e.g. multi-lib=march=rv32im -/// is a subset of march=rv32imc. -/// - march that contains atomic extension can't reuse multi-lib that -/// doesn't have atomic, vice versa. e.g. multi-lib=march=rv32im and -/// march=rv32ima are not compatible, because software and hardware -/// atomic operation can't work together correctly. -static bool -selectRISCVMultilib(const MultilibSet &RISCVMultilibSet, StringRef Arch, - const Multilib::flags_list &Flags, - llvm::SmallVectorImpl &SelectedMultilibs) { - // Try to find the perfect matching multi-lib first. - if (RISCVMultilibSet.select(Flags, SelectedMultilibs)) - return true; - - Multilib::flags_list NewFlags; - std::vector NewMultilibs; - - llvm::Expected> ParseResult = - llvm::RISCVISAInfo::parseArchString( - Arch, /*EnableExperimentalExtension=*/true, - /*ExperimentalExtensionVersionCheck=*/false); - if (!ParseResult) { - // Ignore any error here, we assume it will be handled in another place. - consumeError(ParseResult.takeError()); - return false; - } - - auto &ISAInfo = *ParseResult; - - addMultilibFlag(ISAInfo->getXLen() == 32, "-m32", NewFlags); - addMultilibFlag(ISAInfo->getXLen() == 64, "-m64", NewFlags); - - // Collect all flags except march=* - for (StringRef Flag : Flags) { - if (Flag.starts_with("!march=") || Flag.starts_with("-march=")) - continue; - - NewFlags.push_back(Flag.str()); - } - - llvm::StringSet<> AllArchExts; - // Reconstruct multi-lib list, and break march option into separated - // extension. e.g. march=rv32im -> +i +m - for (const auto &M : RISCVMultilibSet) { - bool Skip = false; - - MultilibBuilder NewMultilib = - MultilibBuilder(M.gccSuffix(), M.osSuffix(), M.includeSuffix()); - for (StringRef Flag : M.flags()) { - // Add back all flags except -march. - if (!Flag.consume_front("-march=")) { - NewMultilib.flag(Flag); - continue; - } - - // Break down -march into individual extension. - llvm::Expected> MLConfigParseResult = - llvm::RISCVISAInfo::parseArchString( - Flag, /*EnableExperimentalExtension=*/true, - /*ExperimentalExtensionVersionCheck=*/false); - if (!MLConfigParseResult) { - // Ignore any error here, we assume it will handled in another place. - llvm::consumeError(MLConfigParseResult.takeError()); - - // We might get a parsing error if rv32e in the list, we could just skip - // that and process the rest of multi-lib configs. - Skip = true; - continue; - } - auto &MLConfigISAInfo = *MLConfigParseResult; - - const llvm::RISCVISAInfo::OrderedExtensionMap &MLConfigArchExts = - MLConfigISAInfo->getExtensions(); - for (auto MLConfigArchExt : MLConfigArchExts) { - auto ExtName = MLConfigArchExt.first; - NewMultilib.flag(Twine("-", ExtName).str()); - - if (AllArchExts.insert(ExtName).second) { - addMultilibFlag(ISAInfo->hasExtension(ExtName), - Twine("-", ExtName).str(), NewFlags); - } - } - - // Check the XLEN explicitly. - if (MLConfigISAInfo->getXLen() == 32) { - NewMultilib.flag("-m32"); - NewMultilib.flag("!m64"); - } else { - NewMultilib.flag("!m32"); - NewMultilib.flag("-m64"); - } - - // Atomic extension must be explicitly checked, soft and hard atomic - // operation never co-work correctly. - if (!MLConfigISAInfo->hasExtension("a")) - NewMultilib.flag("!a"); - } - - if (Skip) - continue; - - NewMultilibs.emplace_back(NewMultilib); - } - - // Build an internal used only multi-lib list, used for checking any - // compatible multi-lib. - MultilibSet NewRISCVMultilibs = - MultilibSetBuilder().Either(NewMultilibs).makeMultilibSet(); - - if (NewRISCVMultilibs.select(NewFlags, SelectedMultilibs)) - for (const Multilib &NewSelectedM : SelectedMultilibs) - for (const auto &M : RISCVMultilibSet) - // Look up the corresponding multi-lib entry in original multi-lib set. - if (M.gccSuffix() == NewSelectedM.gccSuffix()) - return true; - - return false; -} - static void findRISCVBareMetalMultilibs(const Driver &D, const llvm::Triple &TargetTriple, StringRef Path, const ArgList &Args, @@ -1890,8 +1766,7 @@ static void findRISCVBareMetalMultilibs(const Driver &D, } } - if (selectRISCVMultilib(RISCVMultilibs, MArch, Flags, - Result.SelectedMultilibs)) + if (RISCVMultilibs.select(Flags, Result.SelectedMultilibs)) Result.Multilibs = RISCVMultilibs; } diff --git a/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c b/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c deleted file mode 100644 index 1f8a5a8821edf..0000000000000 --- a/clang/test/Driver/riscv-toolchain-gcc-multilib-reuse.c +++ /dev/null @@ -1,81 +0,0 @@ -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32imc -mabi=ilp32 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMC-ILP32 %s -// GCC-MULTI-LIB-REUSE-RV32IMC-ILP32: rv32im/ilp32 -// GCC-MULTI-LIB-REUSE-RV32IMC-ILP32-NOT: {{^.+$}} - -// Check rv32imac won't reuse rv32im or rv32ic -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32imac -mabi=ilp32 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32 %s -// GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32: rv32imac/ilp32 -// GCC-MULTI-LIB-REUSE-RV32IMAC-ILP32--NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32iac -mabi=ilp32 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IAC-ILP32 %s -// GCC-MULTI-LIB-REUSE-RV32IAC-ILP32: rv32iac/ilp32 -// GCC-MULTI-LIB-REUSE-RV32IAC-ILP32-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32imafdc -mabi=ilp32f \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F %s -// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F: rv32imafc/ilp32f -// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32F-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32imafdc -mabi=ilp32d \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D %s -// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D: . -// GCC-MULTI-LIB-REUSE-RV32IMAFDC-ILP32D-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv64imafc -mabi=lp64 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64 %s -// GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64: rv64imac/lp64 -// GCC-MULTI-LIB-REUSE-RV64IMAFC-LP64-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32imafc_zfh -mabi=ilp32 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32 %s -// GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32: rv32imac/ilp32 -// GCC-MULTI-LIB-REUSE-RV32IMAFC_ZFH-ILP32-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv32i_zvkb -mabi=ilp32 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32 %s -// GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32: rv32i/ilp32 -// GCC-MULTI-LIB-REUSE-RV32I_ZVKB-ILP32-NOT: {{^.+$}} - -// RUN: %clang %s \ -// RUN: -target riscv64-unknown-elf \ -// RUN: --gcc-toolchain=%S/Inputs/multilib_riscv_elf_sdk \ -// RUN: --print-multi-directory \ -// RUN: -march=rv64imfc -mabi=lp64 \ -// RUN: | FileCheck -check-prefix=GCC-MULTI-LIB-REUSE-RV64IMFC-LP64 %s -// GCC-MULTI-LIB-REUSE-RV64IMFC-LP64: . -// GCC-MULTI-LIB-REUSE-RV64IMFC-LP64-NOT: {{^.+$}} From 482a37b86036f564a97ad66bbc7aca40fa9cee60 Mon Sep 17 00:00:00 2001 From: harishch4 Date: Mon, 18 Dec 2023 18:27:34 +0530 Subject: [PATCH 109/884] [Flang][OpenMp]Add testcase for threadprivate with blank common block (#74969) --- .../Parser/OpenMP/threadprivate-blank-common-block.f90 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 flang/test/Parser/OpenMP/threadprivate-blank-common-block.f90 diff --git a/flang/test/Parser/OpenMP/threadprivate-blank-common-block.f90 b/flang/test/Parser/OpenMP/threadprivate-blank-common-block.f90 new file mode 100644 index 0000000000000..6317258e6ec8d --- /dev/null +++ b/flang/test/Parser/OpenMP/threadprivate-blank-common-block.f90 @@ -0,0 +1,9 @@ +! RUN: not %flang_fc1 -fsyntax-only %s -fopenmp 2>&1 | FileCheck %s +! From Standard: A blank common block cannot appear in a threadprivate directive. + +program main + integer :: a + common//a + !CHECK: error: expected one of '$@ABCDEFGHIJKLMNOPQRSTUVWXYZ_' + !$omp threadprivate(//) + end From 74cf5254d284f0218db29f535e9ccbcc6a59a3a4 Mon Sep 17 00:00:00 2001 From: arrv-sc <150441443+arrv-sc@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:02:17 +0300 Subject: [PATCH 110/884] [llvm][Support] Add indirection to call correct validate(...) function (#71966) Previously "yamlize" overload for validatedMappingTraits was unconditionally calling "MappingTraits::validate" even if "MappingContextTraits" was passed to it. Therefore compilation failed when specifying "MappingContextTraits::validate()" --- llvm/include/llvm/Support/YAMLTraits.h | 17 +++++++++++++++-- llvm/unittests/Support/YAMLIOTest.cpp | 3 +++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index 99074105a5569..3b1f4bad57fcf 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -1058,6 +1058,19 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) { } } +namespace detail { + +template +std::string doValidate(IO &io, T &Val, Context &Ctx) { + return MappingContextTraits::validate(io, Val, Ctx); +} + +template std::string doValidate(IO &io, T &Val, EmptyContext &) { + return MappingTraits::validate(io, Val); +} + +} // namespace detail + template std::enable_if_t::value, void> yamlize(IO &io, T &Val, bool, Context &Ctx) { @@ -1066,7 +1079,7 @@ yamlize(IO &io, T &Val, bool, Context &Ctx) { else io.beginMapping(); if (io.outputting()) { - std::string Err = MappingTraits::validate(io, Val); + std::string Err = detail::doValidate(io, Val, Ctx); if (!Err.empty()) { errs() << Err << "\n"; assert(Err.empty() && "invalid struct trying to be written as yaml"); @@ -1074,7 +1087,7 @@ yamlize(IO &io, T &Val, bool, Context &Ctx) { } detail::doMapping(io, Val, Ctx); if (!io.outputting()) { - std::string Err = MappingTraits::validate(io, Val); + std::string Err = detail::doValidate(io, Val, Ctx); if (!Err.empty()) io.setError(Err); } diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp index 488746764ae65..401981f3841ee 100644 --- a/llvm/unittests/Support/YAMLIOTest.cpp +++ b/llvm/unittests/Support/YAMLIOTest.cpp @@ -2623,6 +2623,9 @@ template <> struct MappingContextTraits { ++Context.A; io.mapRequired("Context", Context.A); } + static std::string validate(IO &io, SimpleMap &sm, MappingContext &Context) { + return ""; + } }; template <> struct MappingTraits { From 82a1bffd34dab41a379d5854ffa84332bd6456d2 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Mon, 18 Dec 2023 14:03:57 +0100 Subject: [PATCH 111/884] [SelectionDAG] Do not crash on large integers in CheckInteger (#75787) The CheckInteger routine called from TableGen-generated selection logic uses getSExtValue - which will abort if the underlying APInt does not fit into an int64_t. This case is now triggered by the SystemZ back-end since i128 is a legal type on certain machines. While we do not have any regular instructions that take 128-bit immediates (like most other platforms), there are patterns in the .td files that recognize an i128 "xor ..., -1" as a "not". These patterns cause code to be generated that calls the CheckInteger routine on some i128-valued integer, which may trigger the assert. Fix by using trySExtValue instead. Fixes https://github.com/llvm/llvm-project/issues/75710 --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 2 +- llvm/test/CodeGen/SystemZ/xor-09.ll | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index a1cf4cbbee1b8..af49ef17a3f2d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2786,7 +2786,7 @@ CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, Val = decodeSignRotatedValue(Val); ConstantSDNode *C = dyn_cast(N); - return C && C->getSExtValue() == Val; + return C && C->getAPIntValue().trySExtValue() == Val; } LLVM_ATTRIBUTE_ALWAYS_INLINE static bool diff --git a/llvm/test/CodeGen/SystemZ/xor-09.ll b/llvm/test/CodeGen/SystemZ/xor-09.ll index d0287f7fdd77e..7b7aaa404c00f 100644 --- a/llvm/test/CodeGen/SystemZ/xor-09.ll +++ b/llvm/test/CodeGen/SystemZ/xor-09.ll @@ -15,3 +15,17 @@ define i128 @f1(i128 %a, i128 %b) { %res = xor i128 %a, %b ret i128 %res } + +; Verify that xor with a large constant does not crash. +define i128 @f2(i128 %x) { +; CHECK-LABEL: f2: +; CHECK: # %bb.0: +; CHECK-NEXT: larl %r1, .LCPI1_0 +; CHECK-NEXT: vl %v0, 0(%r3), 3 +; CHECK-NEXT: vl %v1, 0(%r1), 3 +; CHECK-NEXT: vx %v0, %v0, %v1 +; CHECK-NEXT: vst %v0, 0(%r2), 3 +; CHECK-NEXT: br %r14 + %res = xor i128 %x, 17440380254424117642 + ret i128 %res +} From 32a4e3fccaf304c8d541bdefdb1a7ef829f84c1c Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Mon, 18 Dec 2023 14:16:52 +0100 Subject: [PATCH 112/884] [mlir] support non-interprocedural dataflow analyses (#75583) The core implementation of the dataflow anlysis framework is interpocedural by design. While this offers better analysis precision, it also comes with additional cost as it takes longer for the analysis to reach the fixpoint state. Add a configuration mechanism to the dataflow solver to control whether it operates inteprocedurally or not to offer clients a choice. As a positive side effect, this change also adds hooks for explicitly processing external/opaque function calls in the dataflow analyses, e.g., based off of attributes present in the the function declaration or call operation such as alias scopes and modref available in the LLVM dialect. This change should not affect existing analyses and the default solver configuration remains interprocedural. Co-authored-by: Jacob Peng --- .../mlir/Analysis/DataFlow/DenseAnalysis.h | 40 +++-- .../mlir/Analysis/DataFlow/SparseAnalysis.h | 55 +++++++ .../include/mlir/Analysis/DataFlowFramework.h | 38 +++++ mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp | 42 ++++-- mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp | 57 ++++--- .../test-last-modified-callgraph.mlir | 125 +++++++++++++--- .../Analysis/DataFlow/test-next-access.mlir | 140 ++++++++++++++---- .../Analysis/DataFlow/test-written-to.mlir | 90 +++++++++-- .../TestDenseBackwardDataFlowAnalysis.cpp | 103 ++++++++++--- .../DataFlow/TestDenseDataFlowAnalysis.h | 96 +++++++++--- .../TestDenseForwardDataFlowAnalysis.cpp | 107 ++++++++++--- .../TestSparseBackwardDataFlowAnalysis.cpp | 49 +++++- 12 files changed, 771 insertions(+), 171 deletions(-) diff --git a/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h index 6a1335bab8bf6..088b6cd7d698f 100644 --- a/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h +++ b/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h @@ -27,8 +27,9 @@ namespace dataflow { // CallControlFlowAction //===----------------------------------------------------------------------===// -/// Indicates whether the control enters or exits the callee. -enum class CallControlFlowAction { EnterCallee, ExitCallee }; +/// Indicates whether the control enters, exits, or skips over the callee (in +/// the case of external functions). +enum class CallControlFlowAction { EnterCallee, ExitCallee, ExternalCallee }; //===----------------------------------------------------------------------===// // AbstractDenseLattice @@ -131,14 +132,21 @@ class AbstractDenseForwardDataFlowAnalysis : public DataFlowAnalysis { /// Propagate the dense lattice forward along the call control flow edge, /// which can be either entering or exiting the callee. Default implementation - /// just meets the states, meaning that operations implementing - /// `CallOpInterface` don't have any effect on the lattice that isn't already - /// expressed by the interface itself. + /// for enter and exit callee actions just meets the states, meaning that + /// operations implementing `CallOpInterface` don't have any effect on the + /// lattice that isn't already expressed by the interface itself. Default + /// implementation for the external callee action additionally sets the + /// "after" lattice to the entry state. virtual void visitCallControlFlowTransfer(CallOpInterface call, CallControlFlowAction action, const AbstractDenseLattice &before, AbstractDenseLattice *after) { join(after, before); + // Note that `setToEntryState` may be a "partial fixpoint" for some + // lattices, e.g., lattices that are lists of maps of other lattices will + // only set fixpoint for "known" lattices. + if (action == CallControlFlowAction::ExternalCallee) + setToEntryState(after); } /// Visit a program point within a region branch operation with predecessors @@ -155,7 +163,9 @@ class AbstractDenseForwardDataFlowAnalysis : public DataFlowAnalysis { /// Visit an operation for which the data flow is described by the /// `CallOpInterface`. - void visitCallOperation(CallOpInterface call, AbstractDenseLattice *after); + void visitCallOperation(CallOpInterface call, + const AbstractDenseLattice &before, + AbstractDenseLattice *after); }; //===----------------------------------------------------------------------===// @@ -361,14 +371,22 @@ class AbstractDenseBackwardDataFlowAnalysis : public DataFlowAnalysis { /// Propagate the dense lattice backwards along the call control flow edge, /// which can be either entering or exiting the callee. Default implementation - /// just meets the states, meaning that operations implementing - /// `CallOpInterface` don't have any effect on hte lattice that isn't already - /// expressed by the interface itself. + /// for enter and exit callee action just meets the states, meaning that + /// operations implementing `CallOpInterface` don't have any effect on the + /// lattice that isn't already expressed by the interface itself. Default + /// implementation for external callee action additional sets the result to + /// the exit (fixpoint) state. virtual void visitCallControlFlowTransfer(CallOpInterface call, CallControlFlowAction action, const AbstractDenseLattice &after, AbstractDenseLattice *before) { meet(before, after); + + // Note that `setToExitState` may be a "partial fixpoint" for some lattices, + // e.g., lattices that are lists of maps of other lattices will only + // set fixpoint for "known" lattices. + if (action == CallControlFlowAction::ExternalCallee) + setToExitState(before); } private: @@ -394,7 +412,9 @@ class AbstractDenseBackwardDataFlowAnalysis : public DataFlowAnalysis { /// otherwise, /// - meet that state with the state before the call-like op, or use the /// custom logic if overridden by concrete analyses. - void visitCallOperation(CallOpInterface call, AbstractDenseLattice *before); + void visitCallOperation(CallOpInterface call, + const AbstractDenseLattice &after, + AbstractDenseLattice *before); /// Symbol table for call-level control flow. SymbolTableCollection &symbolTable; diff --git a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h index 5a9a36159b56c..b65ac8bb1dec2 100644 --- a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h +++ b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h @@ -17,6 +17,7 @@ #include "mlir/Analysis/DataFlowFramework.h" #include "mlir/IR/SymbolTable.h" +#include "mlir/Interfaces/CallInterfaces.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" #include "llvm/ADT/SmallPtrSet.h" @@ -199,6 +200,12 @@ class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis { ArrayRef operandLattices, ArrayRef resultLattices) = 0; + /// The transfer function for calls to external functions. + virtual void visitExternalCallImpl( + CallOpInterface call, + ArrayRef argumentLattices, + ArrayRef resultLattices) = 0; + /// Given an operation with region control-flow, the lattices of the operands, /// and a region successor, compute the lattice values for block arguments /// that are not accounted for by the branching control flow (ex. the bounds @@ -271,6 +278,14 @@ class SparseForwardDataFlowAnalysis virtual void visitOperation(Operation *op, ArrayRef operands, ArrayRef results) = 0; + /// Visit a call operation to an externally defined function given the + /// lattices of its arguments. + virtual void visitExternalCall(CallOpInterface call, + ArrayRef argumentLattices, + ArrayRef resultLattices) { + setAllToEntryStates(resultLattices); + } + /// Given an operation with possible region control-flow, the lattices of the /// operands, and a region successor, compute the lattice values for block /// arguments that are not accounted for by the branching control flow (ex. @@ -321,6 +336,17 @@ class SparseForwardDataFlowAnalysis {reinterpret_cast(resultLattices.begin()), resultLattices.size()}); } + void visitExternalCallImpl( + CallOpInterface call, + ArrayRef argumentLattices, + ArrayRef resultLattices) override { + visitExternalCall( + call, + {reinterpret_cast(argumentLattices.begin()), + argumentLattices.size()}, + {reinterpret_cast(resultLattices.begin()), + resultLattices.size()}); + } void visitNonControlFlowArgumentsImpl( Operation *op, const RegionSuccessor &successor, ArrayRef argLattices, @@ -363,6 +389,11 @@ class AbstractSparseBackwardDataFlowAnalysis : public DataFlowAnalysis { Operation *op, ArrayRef operandLattices, ArrayRef resultLattices) = 0; + /// The transfer function for calls to external functions. + virtual void visitExternalCallImpl( + CallOpInterface call, ArrayRef operandLattices, + ArrayRef resultLattices) = 0; + // Visit operands on branch instructions that are not forwarded. virtual void visitBranchOperand(OpOperand &operand) = 0; @@ -444,6 +475,19 @@ class SparseBackwardDataFlowAnalysis virtual void visitOperation(Operation *op, ArrayRef operands, ArrayRef results) = 0; + /// Visit a call to an external function. This function is expected to set + /// lattice values of the call operands. By default, calls `visitCallOperand` + /// for all operands. + virtual void visitExternalCall(CallOpInterface call, + ArrayRef argumentLattices, + ArrayRef resultLattices) { + (void)argumentLattices; + (void)resultLattices; + for (OpOperand &operand : call->getOpOperands()) { + visitCallOperand(operand); + } + }; + protected: /// Get the lattice element for a value. StateT *getLatticeElement(Value value) override { @@ -474,6 +518,17 @@ class SparseBackwardDataFlowAnalysis {reinterpret_cast(resultLattices.begin()), resultLattices.size()}); } + + void visitExternalCallImpl( + CallOpInterface call, ArrayRef operandLattices, + ArrayRef resultLattices) override { + visitExternalCall( + call, + {reinterpret_cast(operandLattices.begin()), + operandLattices.size()}, + {reinterpret_cast(resultLattices.begin()), + resultLattices.size()}); + } }; } // end namespace dataflow diff --git a/mlir/include/mlir/Analysis/DataFlowFramework.h b/mlir/include/mlir/Analysis/DataFlowFramework.h index c27615b52a12b..541cdb1e237c1 100644 --- a/mlir/include/mlir/Analysis/DataFlowFramework.h +++ b/mlir/include/mlir/Analysis/DataFlowFramework.h @@ -175,6 +175,32 @@ struct ProgramPoint /// Forward declaration of the data-flow analysis class. class DataFlowAnalysis; +//===----------------------------------------------------------------------===// +// DataFlowConfig +//===----------------------------------------------------------------------===// + +/// Configuration class for data flow solver and child analyses. Follows the +/// fluent API pattern. +class DataFlowConfig { +public: + DataFlowConfig() = default; + + /// Set whether the solver should operate interpocedurally, i.e. enter the + /// callee body when available. Interprocedural analyses may be more precise, + /// but also more expensive as more states need to be computed and the + /// fixpoint convergence takes longer. + DataFlowConfig &setInterprocedural(bool enable) { + interprocedural = enable; + return *this; + } + + /// Return `true` if the solver operates interprocedurally, `false` otherwise. + bool isInterprocedural() const { return interprocedural; } + +private: + bool interprocedural = true; +}; + //===----------------------------------------------------------------------===// // DataFlowSolver //===----------------------------------------------------------------------===// @@ -195,6 +221,9 @@ class DataFlowAnalysis; /// TODO: Optimize the internal implementation of the solver. class DataFlowSolver { public: + explicit DataFlowSolver(const DataFlowConfig &config = DataFlowConfig()) + : config(config) {} + /// Load an analysis into the solver. Return the analysis instance. template AnalysisT *load(Args &&...args); @@ -236,7 +265,13 @@ class DataFlowSolver { /// dependent work items to the back of the queue. void propagateIfChanged(AnalysisState *state, ChangeResult changed); + /// Get the configuration of the solver. + const DataFlowConfig &getConfig() const { return config; } + private: + /// Configuration of the dataflow solver. + DataFlowConfig config; + /// The solver's work queue. Work items can be inserted to the front of the /// queue to be processed greedily, speeding up computations that otherwise /// quickly degenerate to quadratic due to propagation of state updates. @@ -423,6 +458,9 @@ class DataFlowAnalysis { return state; } + /// Return the configuration of the solver used for this analysis. + const DataFlowConfig &getSolverConfig() const { return solver.getConfig(); } + #if LLVM_ENABLE_ABI_BREAKING_CHECKS /// When compiling with debugging, keep a name for the analyis. StringRef debugName; diff --git a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp index a6c9f7d7da225..08d89d6db788c 100644 --- a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp +++ b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp @@ -54,12 +54,22 @@ LogicalResult AbstractDenseForwardDataFlowAnalysis::visit(ProgramPoint point) { } void AbstractDenseForwardDataFlowAnalysis::visitCallOperation( - CallOpInterface call, AbstractDenseLattice *after) { + CallOpInterface call, const AbstractDenseLattice &before, + AbstractDenseLattice *after) { + // Allow for customizing the behavior of calls to external symbols, including + // when the analysis is explicitly marked as non-interprocedural. + auto callable = + dyn_cast_if_present(call.resolveCallable()); + if (!getSolverConfig().isInterprocedural() || + (callable && !callable.getCallableRegion())) { + return visitCallControlFlowTransfer( + call, CallControlFlowAction::ExternalCallee, before, after); + } const auto *predecessors = getOrCreateFor(call.getOperation(), call); - // If not all return sites are known, then conservatively assume we can't - // reason about the data-flow. + // Otherwise, if not all return sites are known, then conservatively assume we + // can't reason about the data-flow. if (!predecessors->allPredecessorsKnown()) return setToEntryState(after); @@ -108,7 +118,7 @@ void AbstractDenseForwardDataFlowAnalysis::processOperation(Operation *op) { // If this is a call operation, then join its lattices across known return // sites. if (auto call = dyn_cast(op)) - return visitCallOperation(call, after); + return visitCallOperation(call, *before, after); // Invoke the operation transfer function. visitOperationImpl(op, *before, after); @@ -130,8 +140,10 @@ void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) { if (callable && callable.getCallableRegion() == block->getParent()) { const auto *callsites = getOrCreateFor(block, callable); // If not all callsites are known, conservatively mark all lattices as - // having reached their pessimistic fixpoints. - if (!callsites->allPredecessorsKnown()) + // having reached their pessimistic fixpoints. Do the same if + // interprocedural analysis is not enabled. + if (!callsites->allPredecessorsKnown() || + !getSolverConfig().isInterprocedural()) return setToEntryState(after); for (Operation *callsite : callsites->getKnownPredecessors()) { // Get the dense lattice before the callsite. @@ -267,18 +279,20 @@ LogicalResult AbstractDenseBackwardDataFlowAnalysis::visit(ProgramPoint point) { } void AbstractDenseBackwardDataFlowAnalysis::visitCallOperation( - CallOpInterface call, AbstractDenseLattice *before) { + CallOpInterface call, const AbstractDenseLattice &after, + AbstractDenseLattice *before) { // Find the callee. Operation *callee = call.resolveCallable(&symbolTable); auto callable = dyn_cast_or_null(callee); if (!callable) return setToExitState(before); - // No region means the callee is only declared in this module and we shouldn't - // assume anything about it. + // No region means the callee is only declared in this module. Region *region = callable.getCallableRegion(); - if (!region || region->empty()) - return setToExitState(before); + if (!region || region->empty() || !getSolverConfig().isInterprocedural()) { + return visitCallControlFlowTransfer( + call, CallControlFlowAction::ExternalCallee, after, before); + } // Call-level control flow specifies the data flow here. // @@ -324,7 +338,7 @@ void AbstractDenseBackwardDataFlowAnalysis::processOperation(Operation *op) { return visitRegionBranchOperation(op, branch, RegionBranchPoint::parent(), before); if (auto call = dyn_cast(op)) - return visitCallOperation(call, before); + return visitCallOperation(call, *after, before); // Invoke the operation transfer function. visitOperationImpl(op, *after, before); @@ -359,8 +373,10 @@ void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) { const auto *callsites = getOrCreateFor(block, callable); // If not all call sites are known, conservative mark all lattices as // having reached their pessimistic fix points. - if (!callsites->allPredecessorsKnown()) + if (!callsites->allPredecessorsKnown() || + !getSolverConfig().isInterprocedural()) { return setToExitState(before); + } for (Operation *callsite : callsites->getKnownPredecessors()) { const AbstractDenseLattice *after; diff --git a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp index 9f544d656df92..b47bba16fd902 100644 --- a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp +++ b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp @@ -116,8 +116,27 @@ void AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) { resultLattices); } - // The results of a call operation are determined by the callgraph. + // Grab the lattice elements of the operands. + SmallVector operandLattices; + operandLattices.reserve(op->getNumOperands()); + for (Value operand : op->getOperands()) { + AbstractSparseLattice *operandLattice = getLatticeElement(operand); + operandLattice->useDefSubscribe(this); + operandLattices.push_back(operandLattice); + } + if (auto call = dyn_cast(op)) { + // If the call operation is to an external function, attempt to infer the + // results from the call arguments. + auto callable = + dyn_cast_if_present(call.resolveCallable()); + if (!getSolverConfig().isInterprocedural() || + (callable && !callable.getCallableRegion())) { + return visitExternalCallImpl(call, operandLattices, resultLattices); + } + + // Otherwise, the results of a call operation are determined by the + // callgraph. const auto *predecessors = getOrCreateFor(op, call); // If not all return sites are known, then conservatively assume we can't // reason about the data-flow. @@ -129,15 +148,6 @@ void AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) { return; } - // Grab the lattice elements of the operands. - SmallVector operandLattices; - operandLattices.reserve(op->getNumOperands()); - for (Value operand : op->getOperands()) { - AbstractSparseLattice *operandLattice = getLatticeElement(operand); - operandLattice->useDefSubscribe(this); - operandLattices.push_back(operandLattice); - } - // Invoke the operation transfer function. visitOperationImpl(op, operandLattices, resultLattices); } @@ -168,8 +178,10 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) { const auto *callsites = getOrCreateFor(block, callable); // If not all callsites are known, conservatively mark all lattices as // having reached their pessimistic fixpoints. - if (!callsites->allPredecessorsKnown()) + if (!callsites->allPredecessorsKnown() || + !getSolverConfig().isInterprocedural()) { return setAllToEntryStates(argLattices); + } for (Operation *callsite : callsites->getKnownPredecessors()) { auto call = cast(callsite); for (auto it : llvm::zip(call.getArgOperands(), argLattices)) @@ -433,19 +445,26 @@ void AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) { // stored in `unaccounted`. BitVector unaccounted(op->getNumOperands(), true); + // If the call invokes an external function (or a function treated as + // external due to config), defer to the corresponding extension hook. + // By default, it just does `visitCallOperand` for all operands. OperandRange argOperands = call.getArgOperands(); MutableArrayRef argOpOperands = operandsToOpOperands(argOperands); Region *region = callable.getCallableRegion(); - if (region && !region->empty()) { - Block &block = region->front(); - for (auto [blockArg, argOpOperand] : - llvm::zip(block.getArguments(), argOpOperands)) { - meet(getLatticeElement(argOpOperand.get()), - *getLatticeElementFor(op, blockArg)); - unaccounted.reset(argOpOperand.getOperandNumber()); - } + if (!region || region->empty() || !getSolverConfig().isInterprocedural()) + return visitExternalCallImpl(call, operandLattices, resultLattices); + + // Otherwise, propagate information from the entry point of the function + // back to operands whenever possible. + Block &block = region->front(); + for (auto [blockArg, argOpOperand] : + llvm::zip(block.getArguments(), argOpOperands)) { + meet(getLatticeElement(argOpOperand.get()), + *getLatticeElementFor(op, blockArg)); + unaccounted.reset(argOpOperand.getOperandNumber()); } + // Handle the operands of the call op that aren't forwarded to any // arguments. for (int index : unaccounted.set_bits()) { diff --git a/mlir/test/Analysis/DataFlow/test-last-modified-callgraph.mlir b/mlir/test/Analysis/DataFlow/test-last-modified-callgraph.mlir index 709d787bb306b..a5eba43ac68ab 100644 --- a/mlir/test/Analysis/DataFlow/test-last-modified-callgraph.mlir +++ b/mlir/test/Analysis/DataFlow/test-last-modified-callgraph.mlir @@ -1,8 +1,32 @@ -// RUN: mlir-opt -test-last-modified --split-input-file %s 2>&1 | FileCheck %s +// RUN: mlir-opt -test-last-modified --split-input-file %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP,IP_ONLY +// RUN: mlir-opt -test-last-modified='assume-func-writes=true' \ +// RUN: --split-input-file %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP,IP_AW +// RUN: mlir-opt -test-last-modified='interprocedural=false' \ +// RUN: --split-input-file %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,LOCAL +// RUN: mlir-opt \ +// RUN: -test-last-modified='interprocedural=false assume-func-writes=true' \ +// RUN: --split-input-file %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,LC_AW + +// Check prefixes are as follows: +// 'check': common for all runs; +// 'ip': interprocedural runs; +// 'ip_aw': interpocedural runs assuming calls to external functions write to +// all arguments; +// 'ip_only': interprocedural runs not assuming calls writing; +// 'local': local (non-interprocedural) analysis not assuming calls writing; +// 'lc_aw': local analysis assuming external calls writing to all arguments. // CHECK-LABEL: test_tag: test_callsite -// CHECK: operand #0 -// CHECK-NEXT: - a +// IP: operand #0 +// IP-NEXT: - a +// LOCAL: operand #0 +// LOCAL-NEXT: - +// LC_AW: operand #0 +// LC_AW-NEXT: - func.func private @single_callsite_fn(%ptr: memref) -> memref { return {tag = "test_callsite"} %ptr : memref } @@ -16,8 +40,12 @@ func.func @test_callsite() { } // CHECK-LABEL: test_tag: test_return_site -// CHECK: operand #0 -// CHECK-NEXT: - b +// IP: operand #0 +// IP-NEXT: - b +// LOCAL: operand #0 +// LOCAL-NEXT: - +// LC_AW: operand #0 +// LC_AW-NEXT: - func.func private @single_return_site_fn(%ptr: memref) -> memref { %c0 = arith.constant 0 : i32 memref.store %c0, %ptr[] {tag_name = "b"} : memref @@ -25,9 +53,13 @@ func.func private @single_return_site_fn(%ptr: memref) -> memref { } // CHECK-LABEL: test_tag: test_multiple_callsites -// CHECK: operand #0 -// CHECK-NEXT: write0 -// CHECK-NEXT: write1 +// IP: operand #0 +// IP-NEXT: write0 +// IP-NEXT: write1 +// LOCAL: operand #0 +// LOCAL-NEXT: - +// LC_AW: operand #0 +// LC_AW-NEXT: - func.func @test_return_site(%ptr: memref) -> memref { %0 = func.call @single_return_site_fn(%ptr) : (memref) -> memref return {tag = "test_return_site"} %0 : memref @@ -46,9 +78,13 @@ func.func @test_multiple_callsites(%a: i32, %ptr: memref) -> memref { } // CHECK-LABEL: test_tag: test_multiple_return_sites -// CHECK: operand #0 -// CHECK-NEXT: return0 -// CHECK-NEXT: return1 +// IP: operand #0 +// IP-NEXT: return0 +// IP-NEXT: return1 +// LOCAL: operand #0 +// LOCAL-NEXT: - +// LC_AW: operand #0 +// LC_AW-NEXT: - func.func private @multiple_return_site_fn(%cond: i1, %a: i32, %ptr: memref) -> memref { cf.cond_br %cond, ^a, ^b @@ -69,8 +105,12 @@ func.func @test_multiple_return_sites(%cond: i1, %a: i32, %ptr: memref) -> // ----- // CHECK-LABEL: test_tag: after_call -// CHECK: operand #0 -// CHECK-NEXT: - write0 +// IP: operand #0 +// IP-NEXT: - write0 +// LOCAL: operand #0 +// LOCAL-NEXT: - +// LC_AW: operand #0 +// LC_AW-NEXT: - func.call func.func private @void_return(%ptr: memref) { return } @@ -98,17 +138,29 @@ func.func private @callee(%arg0: memref) -> memref { // "pre" -> "call" -> "callee" -> "post" // CHECK-LABEL: test_tag: call_and_store_before::enter_callee: -// CHECK: operand #0 -// CHECK: - call +// IP: operand #0 +// IP: - call +// LOCAL: operand #0 +// LOCAL: - +// LC_AW: operand #0 +// LC_AW: - + // CHECK: test_tag: exit_callee: // CHECK: operand #0 // CHECK: - callee + // CHECK: test_tag: before_call: // CHECK: operand #0 // CHECK: - pre + // CHECK: test_tag: after_call: -// CHECK: operand #0 -// CHECK: - callee +// IP: operand #0 +// IP: - callee +// LOCAL: operand #0 +// LOCAL: - +// LC_AW: operand #0 +// LC_AW: - call + // CHECK: test_tag: return: // CHECK: operand #0 // CHECK: - post @@ -138,17 +190,29 @@ func.func private @callee(%arg0: memref) -> memref { // "pre" -> "callee" -> "call" -> "post" // CHECK-LABEL: test_tag: call_and_store_after::enter_callee: -// CHECK: operand #0 -// CHECK: - pre +// IP: operand #0 +// IP: - pre +// LOCAL: operand #0 +// LOCAL: - +// LC_AW: operand #0 +// LC_AW: - + // CHECK: test_tag: exit_callee: // CHECK: operand #0 // CHECK: - callee + // CHECK: test_tag: before_call: // CHECK: operand #0 // CHECK: - pre -// CHECK: test_tag: after_call: -// CHECK: operand #0 -// CHECK: - call + +// CHECK: test_tag: after_call: +// IP: operand #0 +// IP: - call +// LOCAL: operand #0 +// LOCAL: - +// LC_AW: operand #0 +// LC_AW: - call + // CHECK: test_tag: return: // CHECK: operand #0 // CHECK: - post @@ -162,3 +226,20 @@ func.func @call_and_store_after(%arg0: memref) -> memref { memref.store %1, %arg0[] {tag_name = "post"} : memref return {tag = "return"} %arg0 : memref } + +// ----- + +func.func private @void_return(%ptr: memref) + +// CHECK-LABEL: test_tag: after_opaque_call: +// CHECK: operand #0 +// IP_ONLY: - +// IP_AW: - func.call +func.func @test_opaque_call_return() { + %ptr = memref.alloc() : memref + %c0 = arith.constant 0 : i32 + memref.store %c0, %ptr[] {tag_name = "write0"} : memref + func.call @void_return(%ptr) : (memref) -> () + memref.load %ptr[] {tag = "after_opaque_call"} : memref + return +} diff --git a/mlir/test/Analysis/DataFlow/test-next-access.mlir b/mlir/test/Analysis/DataFlow/test-next-access.mlir index 313a75c171d01..de0788fb6a176 100644 --- a/mlir/test/Analysis/DataFlow/test-next-access.mlir +++ b/mlir/test/Analysis/DataFlow/test-next-access.mlir @@ -1,4 +1,22 @@ -// RUN: mlir-opt %s --test-next-access --split-input-file | FileCheck %s +// RUN: mlir-opt %s --test-next-access --split-input-file |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP +// RUN: mlir-opt %s --test-next-access='interprocedural=false' \ +// RUN: --split-input-file |\ +// RUN: FileCheck %s --check-prefixes=CHECK,LOCAL +// RUN: mlir-opt %s --test-next-access='assume-func-reads=true' \ +// RUN: --split-input-file |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP_AR +// RUN: mlir-opt %s \ +// RUN: --test-next-access='interprocedural=false assume-func-reads=true' \ +// RUN: --split-input-file | FileCheck %s --check-prefixes=CHECK,LC_AR + +// Check prefixes are as follows: +// 'check': common for all runs; +// 'ip_ar': interpocedural runs assuming calls to external functions read +// all arguments; +// 'ip': interprocedural runs not assuming function calls reading; +// 'local': local (non-interprocedural) analysis not assuming calls reading; +// 'lc_ar': local analysis assuming external calls reading all arguments. // CHECK-LABEL: @trivial func.func @trivial(%arg0: memref, %arg1: f32) -> f32 { @@ -252,8 +270,10 @@ func.func @known_conditional_cf(%arg0: memref) { // ----- func.func private @callee1(%arg0: memref) { - // CHECK: name = "callee1" - // CHECK-SAME: next_access = {{\[}}["post"]] + // IP: name = "callee1" + // IP-SAME: next_access = {{\[}}["post"]] + // LOCAL: name = "callee1" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee1"} : memref return } @@ -267,10 +287,14 @@ func.func private @callee2(%arg0: memref) { // CHECK-LABEL: @simple_call func.func @simple_call(%arg0: memref) { - // CHECK: name = "caller" - // CHECK-SAME: next_access = {{\[}}["callee1"]] + // IP: name = "caller" + // IP-SAME: next_access = {{\[}}["callee1"]] + // LOCAL: name = "caller" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "caller" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "caller"} : memref - func.call @callee1(%arg0) : (memref) -> () + func.call @callee1(%arg0) {name = "call"} : (memref) -> () memref.load %arg0[] {name = "post"} : memref return } @@ -279,10 +303,14 @@ func.func @simple_call(%arg0: memref) { // CHECK-LABEL: @infinite_recursive_call func.func @infinite_recursive_call(%arg0: memref) { - // CHECK: name = "pre" - // CHECK-SAME: next_access = {{\[}}["pre"]] + // IP: name = "pre" + // IP-SAME: next_access = {{\[}}["pre"]] + // LOCAL: name = "pre" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "pre" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "pre"} : memref - func.call @infinite_recursive_call(%arg0) : (memref) -> () + func.call @infinite_recursive_call(%arg0) {name = "call"} : (memref) -> () memref.load %arg0[] {name = "post"} : memref return } @@ -291,11 +319,15 @@ func.func @infinite_recursive_call(%arg0: memref) { // CHECK-LABEL: @recursive_call func.func @recursive_call(%arg0: memref, %cond: i1) { - // CHECK: name = "pre" - // CHECK-SAME: next_access = {{\[}}["post", "pre"]] + // IP: name = "pre" + // IP-SAME: next_access = {{\[}}["post", "pre"]] + // LOCAL: name = "pre" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "pre" + // LC_AR-SAME: next_access = {{\[}}["post", "call"]] memref.load %arg0[] {name = "pre"} : memref scf.if %cond { - func.call @recursive_call(%arg0, %cond) : (memref, i1) -> () + func.call @recursive_call(%arg0, %cond) {name = "call"} : (memref, i1) -> () } memref.load %arg0[] {name = "post"} : memref return @@ -305,12 +337,16 @@ func.func @recursive_call(%arg0: memref, %cond: i1) { // CHECK-LABEL: @recursive_call_cf func.func @recursive_call_cf(%arg0: memref, %cond: i1) { - // CHECK: name = "pre" - // CHECK-SAME: next_access = {{\[}}["pre", "post"]] + // IP: name = "pre" + // IP-SAME: next_access = {{\[}}["pre", "post"]] + // LOCAL: name = "pre" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "pre" + // LC_AR-SAME: next_access = {{\[}}["call", "post"]] %0 = memref.load %arg0[] {name = "pre"} : memref cf.cond_br %cond, ^bb1, ^bb2 ^bb1: - call @recursive_call_cf(%arg0, %cond) : (memref, i1) -> () + call @recursive_call_cf(%arg0, %cond) {name = "call"} : (memref, i1) -> () cf.br ^bb2 ^bb2: %2 = memref.load %arg0[] {name = "post"} : memref @@ -320,27 +356,35 @@ func.func @recursive_call_cf(%arg0: memref, %cond: i1) { // ----- func.func private @callee1(%arg0: memref) { - // CHECK: name = "callee1" - // CHECK-SAME: next_access = {{\[}}["post"]] + // IP: name = "callee1" + // IP-SAME: next_access = {{\[}}["post"]] + // LOCAL: name = "callee1" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee1"} : memref return } func.func private @callee2(%arg0: memref) { - // CHECK: name = "callee2" - // CHECK-SAME: next_access = {{\[}}["post"]] + // IP: name = "callee2" + // IP-SAME: next_access = {{\[}}["post"]] + // LOCAL: name = "callee2" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee2"} : memref return } func.func @conditonal_call(%arg0: memref, %cond: i1) { - // CHECK: name = "pre" - // CHECK-SAME: next_access = {{\[}}["callee1", "callee2"]] + // IP: name = "pre" + // IP-SAME: next_access = {{\[}}["callee1", "callee2"]] + // LOCAL: name = "pre" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "pre" + // LC_AR-SAME: next_access = {{\[}}["call1", "call2"]] memref.load %arg0[] {name = "pre"} : memref scf.if %cond { - func.call @callee1(%arg0) : (memref) -> () + func.call @callee1(%arg0) {name = "call1"} : (memref) -> () } else { - func.call @callee2(%arg0) : (memref) -> () + func.call @callee2(%arg0) {name = "call2"} : (memref) -> () } memref.load %arg0[] {name = "post"} : memref return @@ -354,16 +398,22 @@ func.func @conditonal_call(%arg0: memref, %cond: i1) { // "caller" -> "call" -> "callee" -> "post" func.func private @callee(%arg0: memref) { - // CHECK: name = "callee" - // CHECK-SAME-LITERAL: next_access = [["post"]] + // IP: name = "callee" + // IP-SAME-LITERAL: next_access = [["post"]] + // LOCAL: name = "callee" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee"} : memref return } // CHECK-LABEL: @call_and_store_before func.func @call_and_store_before(%arg0: memref) { - // CHECK: name = "caller" - // CHECK-SAME-LITERAL: next_access = [["call"]] + // IP: name = "caller" + // IP-SAME-LITERAL: next_access = [["call"]] + // LOCAL: name = "caller" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "caller" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "caller"} : memref // Note that the access after the entire call is "post". // CHECK: name = "call" @@ -382,20 +432,26 @@ func.func @call_and_store_before(%arg0: memref) { // "caller" -> "callee" -> "call" -> "post" func.func private @callee(%arg0: memref) { - // CHECK: name = "callee" - // CHECK-SAME-LITERAL: next_access = [["call"]] + // IP: name = "callee" + // IP-SAME-LITERAL: next_access = [["call"]] + // LOCAL: name = "callee" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee"} : memref return } // CHECK-LABEL: @call_and_store_after func.func @call_and_store_after(%arg0: memref) { - // CHECK: name = "caller" - // CHECK-SAME-LITERAL: next_access = [["callee"]] + // IP: name = "caller" + // IP-SAME-LITERAL: next_access = [["callee"]] + // LOCAL: name = "caller" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "caller" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "caller"} : memref // CHECK: name = "call" // CHECK-SAME-LITERAL: next_access = [["post"], ["post"]] - test.call_and_store @callee(%arg0), %arg0 {name = "call", store_before_call = true} : (memref, memref) -> () + test.call_and_store @callee(%arg0), %arg0 {name = "call", store_before_call = false} : (memref, memref) -> () // CHECK: name = "post" // CHECK-SAME-LITERAL: next_access = ["unknown"] memref.load %arg0[] {name = "post"} : memref @@ -499,3 +555,23 @@ func.func @store_with_a_region_after_containing_a_load(%arg0: memref) { memref.load %arg0[] {name = "post"} : memref return } + +// ----- + +func.func private @opaque_callee(%arg0: memref) + +// CHECK-LABEL: @call_opaque_callee +func.func @call_opaque_callee(%arg0: memref) { + // IP: name = "pre" + // IP-SAME: next_access = ["unknown"] + // IP_AR: name = "pre" + // IP_AR-SAME: next_access = {{\[}}["call"]] + // LOCAL: name = "pre" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "pre" + // LC_AR-SAME: next_access = {{\[}}["call"]] + memref.load %arg0[] {name = "pre"} : memref + func.call @opaque_callee(%arg0) {name = "call"} : (memref) -> () + memref.load %arg0[] {name = "post"} : memref + return +} diff --git a/mlir/test/Analysis/DataFlow/test-written-to.mlir b/mlir/test/Analysis/DataFlow/test-written-to.mlir index 82fe755aaf5d4..4fc9af164d48e 100644 --- a/mlir/test/Analysis/DataFlow/test-written-to.mlir +++ b/mlir/test/Analysis/DataFlow/test-written-to.mlir @@ -1,4 +1,28 @@ -// RUN: mlir-opt -split-input-file -test-written-to %s 2>&1 | FileCheck %s +// RUN: mlir-opt -split-input-file -test-written-to %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP +// RUN: mlir-opt -split-input-file -test-written-to='interprocedural=false' %s \ +// RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK,LOCAL +// RUN: mlir-opt -split-input-file \ +// RUN: -test-written-to='assume-func-writes=true' %s 2>&1 |\ +// RUN: FileCheck %s --check-prefixes=CHECK,IP_AW +// RUN: mlir-opt -split-input-file \ +// RUN: -test-written-to='interprocedural=false assume-func-writes=true' \ +// RUN: %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LC_AW + +// Check prefixes are as follows: +// 'check': common for all runs; +// 'ip': interprocedural runs; +// 'ip_aw': interpocedural runs assuming calls to external functions write to +// all arguments; +// 'local': local (non-interprocedural) analysis not assuming calls writing; +// 'lc_aw': local analysis assuming external calls writing to all arguments. + +// Note that despite the name of the test analysis being "written to", it is set +// up in a peculiar way where passing a value through a block or region argument +// (via visitCall/BranchOperand) is considered as "writing" that value to the +// corresponding operand, which is itself a value and not necessarily "memory". +// This is arguably okay for testing purposes, but may be surprising for readers +// trying to interpret this test using their intuition. // CHECK-LABEL: test_tag: constant0 // CHECK: result #0: [a] @@ -105,7 +129,9 @@ func.func @test_switch(%flag: i32, %m0: memref) { // ----- // CHECK-LABEL: test_tag: add -// CHECK: result #0: [a] +// IP: result #0: [a] +// LOCAL: result #0: [callarg0] +// LC_AW: result #0: [func.call] func.func @test_caller(%m0: memref, %arg: f32) { %0 = arith.addf %arg, %arg {tag = "add"} : f32 %1 = func.call @callee(%0) : (f32) -> f32 @@ -130,7 +156,9 @@ func.func private @callee(%0 : f32) -> f32 { } // CHECK-LABEL: test_tag: sub -// CHECK: result #0: [a] +// IP: result #0: [a] +// LOCAL: result #0: [callarg0] +// LC_AW: result #0: [func.call] func.func @test_caller_below_callee(%m0: memref, %arg: f32) { %0 = arith.subf %arg, %arg {tag = "sub"} : f32 %1 = func.call @callee(%0) : (f32) -> f32 @@ -155,7 +183,9 @@ func.func private @callee3(%0 : f32) -> f32 { } // CHECK-LABEL: test_tag: mul -// CHECK: result #0: [a] +// IP: result #0: [a] +// LOCAL: result #0: [callarg0] +// LC_AW: result #0: [func.call] func.func @test_callchain(%m0: memref, %arg: f32) { %0 = arith.mulf %arg, %arg {tag = "mul"} : f32 %1 = func.call @callee1(%0) : (f32) -> f32 @@ -239,19 +269,19 @@ func.func @test_for(%m0: memref) { // ----- // CHECK-LABEL: test_tag: default_a -// CHECK-LABEL: result #0: [a] +// CHECK: result #0: [a] // CHECK-LABEL: test_tag: default_b -// CHECK-LABEL: result #0: [b] +// CHECK: result #0: [b] // CHECK-LABEL: test_tag: 1a -// CHECK-LABEL: result #0: [a] +// CHECK: result #0: [a] // CHECK-LABEL: test_tag: 1b -// CHECK-LABEL: result #0: [b] +// CHECK: result #0: [b] // CHECK-LABEL: test_tag: 2a -// CHECK-LABEL: result #0: [a] +// CHECK: result #0: [a] // CHECK-LABEL: test_tag: 2b -// CHECK-LABEL: result #0: [b] +// CHECK: result #0: [b] // CHECK-LABEL: test_tag: switch -// CHECK-LABEL: operand #0: [brancharg0] +// CHECK: operand #0: [brancharg0] func.func @test_switch(%arg0 : index, %m0: memref) { %0, %1 = scf.index_switch %arg0 {tag="switch"} -> i32, i32 case 1 { @@ -276,6 +306,9 @@ func.func @test_switch(%arg0 : index, %m0: memref) { // ----- +// The point of this test is to ensure the analysis doesn't crash in presence of +// external functions. + // CHECK-LABEL: llvm.func @decl(i64) // CHECK-LABEL: llvm.func @func(%arg0: i64) { // CHECK-NEXT: llvm.call @decl(%arg0) : (i64) -> () @@ -295,12 +328,39 @@ func.func private @callee(%arg0 : i32, %arg1 : i32) -> i32 { } // CHECK-LABEL: test_tag: a -// CHECK-LABEL: operand #0: [b] -// CHECK-LABEL: operand #1: [] -// CHECK-LABEL: operand #2: [callarg2] -// CHECK-LABEL: result #0: [b] + +// IP: operand #0: [b] +// LOCAL: operand #0: [callarg0] +// LC_AW: operand #0: [test.call_on_device] + +// IP: operand #1: [] +// LOCAL: operand #1: [callarg1] +// LC_AW: operand #1: [test.call_on_device] + +// IP: operand #2: [callarg2] +// LOCAL: operand #2: [callarg2] +// LC_AW: operand #2: [test.call_on_device] + +// CHECK: result #0: [b] func.func @test_call_on_device(%arg0: i32, %arg1: i32, %device: i32, %m0: memref) { %0 = test.call_on_device @callee(%arg0, %arg1), %device {tag = "a"} : (i32, i32, i32) -> (i32) memref.store %0, %m0[] {tag_name = "b"} : memref return } + +// ----- + +func.func private @external_callee(%arg0: i32) -> i32 + +// CHECK-LABEL: test_tag: add_external +// IP: operand #0: [callarg0] +// LOCAL: operand #0: [callarg0] +// LC_AW: operand #0: [func.call] +// IP_AW: operand #0: [func.call] + +func.func @test_external_callee(%arg0: i32, %m0: memref) { + %0 = arith.addi %arg0, %arg0 { tag = "add_external"}: i32 + %1 = func.call @external_callee(%arg0) : (i32) -> i32 + memref.store %1, %m0[] {tag_name = "a"} : memref + return +} diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp index 8bfd01d828060..ca052392f2f5f 100644 --- a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp +++ b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp @@ -49,7 +49,10 @@ class NextAccess : public AbstractDenseLattice, public AccessLatticeBase { class NextAccessAnalysis : public DenseBackwardDataFlowAnalysis { public: - using DenseBackwardDataFlowAnalysis::DenseBackwardDataFlowAnalysis; + NextAccessAnalysis(DataFlowSolver &solver, SymbolTableCollection &symbolTable, + bool assumeFuncReads = false) + : DenseBackwardDataFlowAnalysis(solver, symbolTable), + assumeFuncReads(assumeFuncReads) {} void visitOperation(Operation *op, const NextAccess &after, NextAccess *before) override; @@ -69,8 +72,10 @@ class NextAccessAnalysis : public DenseBackwardDataFlowAnalysis { // means "we don't know what the next access is" rather than "there is no next // access". But it's unclear how to differentiate the two cases... void setToExitState(NextAccess *lattice) override { - propagateIfChanged(lattice, lattice->reset()); + propagateIfChanged(lattice, lattice->setKnownToUnknown()); } + + const bool assumeFuncReads; }; } // namespace @@ -84,7 +89,13 @@ void NextAccessAnalysis::visitOperation(Operation *op, const NextAccess &after, SmallVector effects; memory.getEffects(effects); - ChangeResult result = before->meet(after); + + // First, check if all underlying values are already known. Otherwise, avoid + // propagating and stay in the "undefined" state to avoid incorrectly + // propagating values that may be overwritten later on as that could be + // problematic for convergence based on monotonicity of lattice updates. + SmallVector underlyingValues; + underlyingValues.reserve(effects.size()); for (const MemoryEffects::EffectInstance &effect : effects) { Value value = effect.getValue(); @@ -95,10 +106,23 @@ void NextAccessAnalysis::visitOperation(Operation *op, const NextAccess &after, // If cannot find the most underlying value, we cannot assume anything about // the next accesses. - value = UnderlyingValueAnalysis::getMostUnderlyingValue( - value, [&](Value value) { - return getOrCreateFor(op, value); - }); + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + value, [&](Value value) { + return getOrCreateFor(op, value); + }); + + // If the underlying value is not known yet, don't propagate. + if (!underlyingValue) + return; + + underlyingValues.push_back(*underlyingValue); + } + + // Update the state if all underlying values are known. + ChangeResult result = before->meet(after); + for (const auto &[effect, value] : llvm::zip(effects, underlyingValues)) { + // If the underlying value is known to be unknown, set to fixpoint. if (!value) return setToExitState(before); @@ -110,6 +134,27 @@ void NextAccessAnalysis::visitOperation(Operation *op, const NextAccess &after, void NextAccessAnalysis::visitCallControlFlowTransfer( CallOpInterface call, CallControlFlowAction action, const NextAccess &after, NextAccess *before) { + if (action == CallControlFlowAction::ExternalCallee && assumeFuncReads) { + SmallVector underlyingValues; + underlyingValues.reserve(call->getNumOperands()); + for (Value operand : call.getArgOperands()) { + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + operand, [&](Value value) { + return getOrCreateFor( + call.getOperation(), value); + }); + if (!underlyingValue) + return; + underlyingValues.push_back(*underlyingValue); + } + + ChangeResult result = before->meet(after); + for (Value operand : underlyingValues) { + result |= before->set(operand, call); + } + return propagateIfChanged(before, result); + } auto testCallAndStore = dyn_cast<::test::TestCallAndStoreOp>(call.getOperation()); if (testCallAndStore && ((action == CallControlFlowAction::EnterCallee && @@ -143,10 +188,24 @@ void NextAccessAnalysis::visitRegionBranchControlFlowTransfer( namespace { struct TestNextAccessPass : public PassWrapper> { + TestNextAccessPass() = default; + TestNextAccessPass(const TestNextAccessPass &other) : PassWrapper(other) { + interprocedural = other.interprocedural; + assumeFuncReads = other.assumeFuncReads; + } + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestNextAccessPass) StringRef getArgument() const override { return "test-next-access"; } + Option interprocedural{ + *this, "interprocedural", llvm::cl::init(true), + llvm::cl::desc("perform interprocedural analysis")}; + Option assumeFuncReads{ + *this, "assume-func-reads", llvm::cl::init(false), + llvm::cl::desc( + "assume external functions have read effect on all arguments")}; + static constexpr llvm::StringLiteral kTagAttrName = "name"; static constexpr llvm::StringLiteral kNextAccessAttrName = "next_access"; static constexpr llvm::StringLiteral kAtEntryPointAttrName = @@ -158,22 +217,29 @@ struct TestNextAccessPass if (!nextAccess) return StringAttr::get(op->getContext(), "not computed"); + // Note that if the underlying value could not be computed or is unknown, we + // conservatively treat the result also unknown. SmallVector attrs; for (Value operand : op->getOperands()) { - Value value = UnderlyingValueAnalysis::getMostUnderlyingValue( - operand, [&](Value value) { - return solver.lookupState(value); - }); - std::optional> nextAcc = - nextAccess->getAdjacentAccess(value); - if (!nextAcc) { + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + operand, [&](Value value) { + return solver.lookupState(value); + }); + if (!underlyingValue) { + attrs.push_back(StringAttr::get(op->getContext(), "unknown")); + continue; + } + Value value = *underlyingValue; + const AdjacentAccess *nextAcc = nextAccess->getAdjacentAccess(value); + if (!nextAcc || !nextAcc->isKnown()) { attrs.push_back(StringAttr::get(op->getContext(), "unknown")); continue; } SmallVector innerAttrs; - innerAttrs.reserve(nextAcc->size()); - for (Operation *nextAccOp : *nextAcc) { + innerAttrs.reserve(nextAcc->get().size()); + for (Operation *nextAccOp : nextAcc->get()) { if (auto nextAccTag = nextAccOp->getAttrOfType(kTagAttrName)) { innerAttrs.push_back(nextAccTag); @@ -193,9 +259,10 @@ struct TestNextAccessPass Operation *op = getOperation(); SymbolTableCollection symbolTable; - DataFlowSolver solver; + auto config = DataFlowConfig().setInterprocedural(interprocedural); + DataFlowSolver solver(config); solver.load(); - solver.load(symbolTable); + solver.load(symbolTable, assumeFuncReads); solver.load(); solver.load(); if (failed(solver.initializeAndRun(op))) { diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseDataFlowAnalysis.h b/mlir/test/lib/Analysis/DataFlow/TestDenseDataFlowAnalysis.h index eab54fbcfbf4a..61ddc13f8a3d4 100644 --- a/mlir/test/lib/Analysis/DataFlow/TestDenseDataFlowAnalysis.h +++ b/mlir/test/lib/Analysis/DataFlow/TestDenseDataFlowAnalysis.h @@ -57,6 +57,62 @@ class UnderlyingValue { std::optional underlyingValue; }; +class AdjacentAccess { +public: + using DeterministicSetVector = + SetVector, + SmallPtrSet>; + + ArrayRef get() const { return accesses.getArrayRef(); } + bool isKnown() const { return !unknown; } + + ChangeResult merge(const AdjacentAccess &other) { + if (unknown) + return ChangeResult::NoChange; + if (other.unknown) { + unknown = true; + accesses.clear(); + return ChangeResult::Change; + } + + size_t sizeBefore = accesses.size(); + accesses.insert(other.accesses.begin(), other.accesses.end()); + return accesses.size() == sizeBefore ? ChangeResult::NoChange + : ChangeResult::Change; + } + + ChangeResult set(Operation *op) { + if (!unknown && accesses.size() == 1 && *accesses.begin() == op) + return ChangeResult::NoChange; + + unknown = false; + accesses.clear(); + accesses.insert(op); + return ChangeResult::Change; + } + + ChangeResult setUnknown() { + if (unknown) + return ChangeResult::NoChange; + + accesses.clear(); + unknown = true; + return ChangeResult::Change; + } + + bool operator==(const AdjacentAccess &other) const { + return unknown == other.unknown && accesses == other.accesses; + } + + bool operator!=(const AdjacentAccess &other) const { + return !operator==(other); + } + +private: + bool unknown = false; + DeterministicSetVector accesses; +}; + /// This lattice represents, for a given memory resource, the potential last /// operations that modified the resource. class AccessLatticeBase { @@ -73,40 +129,42 @@ class AccessLatticeBase { ChangeResult merge(const AccessLatticeBase &rhs) { ChangeResult result = ChangeResult::NoChange; for (const auto &mod : rhs.adjAccesses) { - auto &lhsMod = adjAccesses[mod.first]; - if (lhsMod != mod.second) { - lhsMod.insert(mod.second.begin(), mod.second.end()); - result |= ChangeResult::Change; - } + AdjacentAccess &lhsMod = adjAccesses[mod.first]; + result |= lhsMod.merge(mod.second); } return result; } /// Set the last modification of a value. ChangeResult set(Value value, Operation *op) { - auto &lastMod = adjAccesses[value]; + AdjacentAccess &lastMod = adjAccesses[value]; + return lastMod.set(op); + } + + ChangeResult setKnownToUnknown() { ChangeResult result = ChangeResult::NoChange; - if (lastMod.size() != 1 || *lastMod.begin() != op) { - result = ChangeResult::Change; - lastMod.clear(); - lastMod.insert(op); - } + for (auto &[value, adjacent] : adjAccesses) + result |= adjacent.setUnknown(); return result; } /// Get the adjacent accesses to a value. Returns std::nullopt if they /// are not known. - std::optional> getAdjacentAccess(Value value) const { + const AdjacentAccess *getAdjacentAccess(Value value) const { auto it = adjAccesses.find(value); if (it == adjAccesses.end()) - return {}; - return it->second.getArrayRef(); + return nullptr; + return &it->getSecond(); } void print(raw_ostream &os) const { for (const auto &lastMod : adjAccesses) { os << lastMod.first << ":\n"; - for (Operation *op : lastMod.second) + if (!lastMod.second.isKnown()) { + os << " \n"; + return; + } + for (Operation *op : lastMod.second.get()) os << " " << *op << "\n"; } } @@ -114,9 +172,7 @@ class AccessLatticeBase { private: /// The potential adjacent accesses to a memory resource. Use a set vector to /// keep the results deterministic. - DenseMap, - SmallPtrSet>> - adjAccesses; + DenseMap adjAccesses; }; /// Define the lattice class explicitly to provide a type ID. @@ -148,7 +204,7 @@ class UnderlyingValueAnalysis } /// Look for the most underlying value of a value. - static Value + static std::optional getMostUnderlyingValue(Value value, function_ref getUnderlyingValueFn) { @@ -156,7 +212,7 @@ class UnderlyingValueAnalysis do { underlying = getUnderlyingValueFn(value); if (!underlying || underlying->getValue().isUninitialized()) - return {}; + return std::nullopt; Value underlyingValue = underlying->getValue().getUnderlyingValue(); if (underlyingValue == value) break; diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp index 2520ed3d83b9e..29480f5ad63ee 100644 --- a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp +++ b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp @@ -49,7 +49,9 @@ class LastModification : public AbstractDenseLattice, public AccessLatticeBase { class LastModifiedAnalysis : public DenseForwardDataFlowAnalysis { public: - using DenseForwardDataFlowAnalysis::DenseForwardDataFlowAnalysis; + explicit LastModifiedAnalysis(DataFlowSolver &solver, bool assumeFuncWrites) + : DenseForwardDataFlowAnalysis(solver), + assumeFuncWrites(assumeFuncWrites) {} /// Visit an operation. If the operation has no memory effects, then the state /// is propagated with no change. If the operation allocates a resource, then @@ -74,6 +76,9 @@ class LastModifiedAnalysis void setToEntryState(LastModification *lattice) override { propagateIfChanged(lattice, lattice->reset()); } + +private: + const bool assumeFuncWrites; }; } // end anonymous namespace @@ -89,7 +94,12 @@ void LastModifiedAnalysis::visitOperation(Operation *op, SmallVector effects; memory.getEffects(effects); - ChangeResult result = after->join(before); + // First, check if all underlying values are already known. Otherwise, avoid + // propagating and stay in the "undefined" state to avoid incorrectly + // propagating values that may be overwritten later on as that could be + // problematic for convergence based on monotonicity of lattice updates. + SmallVector underlyingValues; + underlyingValues.reserve(effects.size()); for (const auto &effect : effects) { Value value = effect.getValue(); @@ -100,10 +110,23 @@ void LastModifiedAnalysis::visitOperation(Operation *op, // If we cannot find the underlying value, we shouldn't just propagate the // effects through, return the pessimistic state. - value = UnderlyingValueAnalysis::getMostUnderlyingValue( - value, [&](Value value) { - return getOrCreateFor(op, value); - }); + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + value, [&](Value value) { + return getOrCreateFor(op, value); + }); + + // If the underlying value is not yet known, don't propagate yet. + if (!underlyingValue) + return; + + underlyingValues.push_back(*underlyingValue); + } + + // Update the state when all underlying values are known. + ChangeResult result = after->join(before); + for (const auto &[effect, value] : llvm::zip(effects, underlyingValues)) { + // If the underlying value is known to be unknown, set to fixpoint state. if (!value) return setToEntryState(after); @@ -119,6 +142,26 @@ void LastModifiedAnalysis::visitOperation(Operation *op, void LastModifiedAnalysis::visitCallControlFlowTransfer( CallOpInterface call, CallControlFlowAction action, const LastModification &before, LastModification *after) { + if (action == CallControlFlowAction::ExternalCallee && assumeFuncWrites) { + SmallVector underlyingValues; + underlyingValues.reserve(call->getNumOperands()); + for (Value operand : call.getArgOperands()) { + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + operand, [&](Value value) { + return getOrCreateFor( + call.getOperation(), value); + }); + if (!underlyingValue) + return; + underlyingValues.push_back(*underlyingValue); + } + + ChangeResult result = after->join(before); + for (Value operand : underlyingValues) + result |= after->set(operand, call); + return propagateIfChanged(after, result); + } auto testCallAndStore = dyn_cast<::test::TestCallAndStoreOp>(call.getOperation()); if (testCallAndStore && ((action == CallControlFlowAction::EnterCallee && @@ -155,21 +198,37 @@ struct TestLastModifiedPass : public PassWrapper> { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLastModifiedPass) + TestLastModifiedPass() = default; + TestLastModifiedPass(const TestLastModifiedPass &other) : PassWrapper(other) { + interprocedural = other.interprocedural; + assumeFuncWrites = other.assumeFuncWrites; + } + StringRef getArgument() const override { return "test-last-modified"; } + Option interprocedural{ + *this, "interprocedural", llvm::cl::init(true), + llvm::cl::desc("perform interprocedural analysis")}; + Option assumeFuncWrites{ + *this, "assume-func-writes", llvm::cl::init(false), + llvm::cl::desc( + "assume external functions have write effect on all arguments")}; + void runOnOperation() override { Operation *op = getOperation(); - DataFlowSolver solver; + DataFlowSolver solver(DataFlowConfig().setInterprocedural(interprocedural)); solver.load(); solver.load(); - solver.load(); + solver.load(assumeFuncWrites); solver.load(); if (failed(solver.initializeAndRun(op))) return signalPassFailure(); raw_ostream &os = llvm::errs(); + // Note that if the underlying value could not be computed or is unknown, we + // conservatively treat the result also unknown. op->walk([&](Operation *op) { auto tag = op->getAttrOfType("tag"); if (!tag) @@ -180,19 +239,29 @@ struct TestLastModifiedPass assert(lastMods && "expected a dense lattice"); for (auto [index, operand] : llvm::enumerate(op->getOperands())) { os << " operand #" << index << "\n"; - Value value = UnderlyingValueAnalysis::getMostUnderlyingValue( - operand, [&](Value value) { - return solver.lookupState(value); - }); + std::optional underlyingValue = + UnderlyingValueAnalysis::getMostUnderlyingValue( + operand, [&](Value value) { + return solver.lookupState(value); + }); + if (!underlyingValue) { + os << " - \n"; + continue; + } + Value value = *underlyingValue; assert(value && "expected an underlying value"); - if (std::optional> lastMod = + if (const AdjacentAccess *lastMod = lastMods->getAdjacentAccess(value)) { - for (Operation *lastModifier : *lastMod) { - if (auto tagName = - lastModifier->getAttrOfType("tag_name")) { - os << " - " << tagName.getValue() << "\n"; - } else { - os << " - " << lastModifier->getName() << "\n"; + if (!lastMod->isKnown()) { + os << " - \n"; + } else { + for (Operation *lastModifier : lastMod->get()) { + if (auto tagName = + lastModifier->getAttrOfType("tag_name")) { + os << " - " << tagName.getValue() << "\n"; + } else { + os << " - " << lastModifier->getName() << "\n"; + } } } } else { diff --git a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp index f97a4c8bc5eb3..e1c60f06a6b5e 100644 --- a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp +++ b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp @@ -50,7 +50,10 @@ struct WrittenTo : public AbstractSparseLattice { /// is eventually written to. class WrittenToAnalysis : public SparseBackwardDataFlowAnalysis { public: - using SparseBackwardDataFlowAnalysis::SparseBackwardDataFlowAnalysis; + WrittenToAnalysis(DataFlowSolver &solver, SymbolTableCollection &symbolTable, + bool assumeFuncWrites) + : SparseBackwardDataFlowAnalysis(solver, symbolTable), + assumeFuncWrites(assumeFuncWrites) {} void visitOperation(Operation *op, ArrayRef operands, ArrayRef results) override; @@ -59,7 +62,13 @@ class WrittenToAnalysis : public SparseBackwardDataFlowAnalysis { void visitCallOperand(OpOperand &operand) override; + void visitExternalCall(CallOpInterface call, ArrayRef operands, + ArrayRef results) override; + void setToExitState(WrittenTo *lattice) override { lattice->writes.clear(); } + +private: + bool assumeFuncWrites; }; void WrittenToAnalysis::visitOperation(Operation *op, @@ -99,6 +108,26 @@ void WrittenToAnalysis::visitCallOperand(OpOperand &operand) { propagateIfChanged(lattice, lattice->addWrites(newWrites)); } +void WrittenToAnalysis::visitExternalCall(CallOpInterface call, + ArrayRef operands, + ArrayRef results) { + if (!assumeFuncWrites) { + return SparseBackwardDataFlowAnalysis::visitExternalCall(call, operands, + results); + } + + for (WrittenTo *lattice : operands) { + SetVector newWrites; + StringAttr name = call->getAttrOfType("tag_name"); + if (!name) { + name = StringAttr::get(call->getContext(), + call.getOperation()->getName().getStringRef()); + } + newWrites.insert(name); + propagateIfChanged(lattice, lattice->addWrites(newWrites)); + } +} + } // end anonymous namespace namespace { @@ -106,17 +135,31 @@ struct TestWrittenToPass : public PassWrapper> { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestWrittenToPass) + TestWrittenToPass() = default; + TestWrittenToPass(const TestWrittenToPass &other) : PassWrapper(other) { + interprocedural = other.interprocedural; + assumeFuncWrites = other.assumeFuncWrites; + } + StringRef getArgument() const override { return "test-written-to"; } + Option interprocedural{ + *this, "interprocedural", llvm::cl::init(true), + llvm::cl::desc("perform interprocedural analysis")}; + Option assumeFuncWrites{ + *this, "assume-func-writes", llvm::cl::init(false), + llvm::cl::desc( + "assume external functions have write effect on all arguments")}; + void runOnOperation() override { Operation *op = getOperation(); SymbolTableCollection symbolTable; - DataFlowSolver solver; + DataFlowSolver solver(DataFlowConfig().setInterprocedural(interprocedural)); solver.load(); solver.load(); - solver.load(symbolTable); + solver.load(symbolTable, assumeFuncWrites); if (failed(solver.initializeAndRun(op))) return signalPassFailure(); From fda3a134bcd96bd08ab32d55f4731a06808880d2 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 15:00:43 +0100 Subject: [PATCH 113/884] [InstCombine] Regenerate test checks (NFC) --- .../Transforms/InstCombine/X86/x86-vpermil.ll | 72 +++++++++---------- .../obfuscated_splat-inseltpoison.ll | 11 --- .../InstCombine/obfuscated_splat.ll | 27 ++++++- 3 files changed, 61 insertions(+), 49 deletions(-) delete mode 100644 llvm/test/Transforms/InstCombine/obfuscated_splat-inseltpoison.ll diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll index 1060dc27b0910..9e72b5ad0e1a2 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll @@ -57,8 +57,8 @@ define <8 x double> @identity_test_vpermilvar_pd_512(<8 x double> %v) { define <4 x float> @zero_test_vpermilvar_ps_zero(<4 x float> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_ps_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: ret <4 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: ret <4 x float> [[A]] ; %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> zeroinitializer) ret <4 x float> %a @@ -66,8 +66,8 @@ define <4 x float> @zero_test_vpermilvar_ps_zero(<4 x float> %v) { define <8 x float> @zero_test_vpermilvar_ps_256_zero(<8 x float> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_ps_256_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> zeroinitializer) ret <8 x float> %a @@ -75,8 +75,8 @@ define <8 x float> @zero_test_vpermilvar_ps_256_zero(<8 x float> %v) { define <16 x float> @zero_test_vpermilvar_ps_512_zero(<16 x float> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_ps_512_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> -; CHECK-NEXT: ret <16 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> +; CHECK-NEXT: ret <16 x float> [[A]] ; %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> zeroinitializer) ret <16 x float> %a @@ -84,8 +84,8 @@ define <16 x float> @zero_test_vpermilvar_ps_512_zero(<16 x float> %v) { define <2 x double> @zero_test_vpermilvar_pd_zero(<2 x double> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_pd_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: ret <2 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: ret <2 x double> [[A]] ; %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> zeroinitializer) ret <2 x double> %a @@ -93,8 +93,8 @@ define <2 x double> @zero_test_vpermilvar_pd_zero(<2 x double> %v) { define <4 x double> @zero_test_vpermilvar_pd_256_zero(<4 x double> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_pd_256_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> -; CHECK-NEXT: ret <4 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: ret <4 x double> [[A]] ; %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> zeroinitializer) ret <4 x double> %a @@ -102,8 +102,8 @@ define <4 x double> @zero_test_vpermilvar_pd_256_zero(<4 x double> %v) { define <8 x double> @zero_test_vpermilvar_pd_512_zero(<8 x double> %v) { ; CHECK-LABEL: @zero_test_vpermilvar_pd_512_zero( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> -; CHECK-NEXT: ret <8 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> +; CHECK-NEXT: ret <8 x double> [[A]] ; %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> zeroinitializer) ret <8 x double> %a @@ -113,8 +113,8 @@ define <8 x double> @zero_test_vpermilvar_pd_512_zero(<8 x double> %v) { define <4 x float> @test_vpermilvar_ps(<4 x float> %v) { ; CHECK-LABEL: @test_vpermilvar_ps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> -; CHECK-NEXT: ret <4 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> +; CHECK-NEXT: ret <4 x float> [[A]] ; %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> ) ret <4 x float> %a @@ -122,8 +122,8 @@ define <4 x float> @test_vpermilvar_ps(<4 x float> %v) { define <8 x float> @test_vpermilvar_ps_256(<8 x float> %v) { ; CHECK-LABEL: @test_vpermilvar_ps_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> ) ret <8 x float> %a @@ -131,8 +131,8 @@ define <8 x float> @test_vpermilvar_ps_256(<8 x float> %v) { define <16 x float> @test_vpermilvar_ps_512(<16 x float> %v) { ; CHECK-LABEL: @test_vpermilvar_ps_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> -; CHECK-NEXT: ret <16 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> +; CHECK-NEXT: ret <16 x float> [[A]] ; %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> ) ret <16 x float> %a @@ -140,8 +140,8 @@ define <16 x float> @test_vpermilvar_ps_512(<16 x float> %v) { define <2 x double> @test_vpermilvar_pd(<2 x double> %v) { ; CHECK-LABEL: @test_vpermilvar_pd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> -; CHECK-NEXT: ret <2 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> +; CHECK-NEXT: ret <2 x double> [[A]] ; %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> ) ret <2 x double> %a @@ -149,8 +149,8 @@ define <2 x double> @test_vpermilvar_pd(<2 x double> %v) { define <4 x double> @test_vpermilvar_pd_256(<4 x double> %v) { ; CHECK-LABEL: @test_vpermilvar_pd_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> -; CHECK-NEXT: ret <4 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: ret <4 x double> [[A]] ; %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> ) ret <4 x double> %a @@ -158,8 +158,8 @@ define <4 x double> @test_vpermilvar_pd_256(<4 x double> %v) { define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) { ; CHECK-LABEL: @test_vpermilvar_pd_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> -; CHECK-NEXT: ret <8 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> +; CHECK-NEXT: ret <8 x double> [[A]] ; %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> ) ret <8 x double> %a @@ -169,8 +169,8 @@ define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) { define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_ps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> -; CHECK-NEXT: ret <4 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> +; CHECK-NEXT: ret <4 x float> [[A]] ; %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> ) ret <4 x float> %a @@ -178,8 +178,8 @@ define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) { define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_ps_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> ) ret <8 x float> %a @@ -187,8 +187,8 @@ define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) { define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_ps_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> -; CHECK-NEXT: ret <16 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> +; CHECK-NEXT: ret <16 x float> [[A]] ; %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> ) ret <16 x float> %a @@ -196,8 +196,8 @@ define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) { define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_pd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> -; CHECK-NEXT: ret <2 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> +; CHECK-NEXT: ret <2 x double> [[A]] ; %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> ) ret <2 x double> %a @@ -205,8 +205,8 @@ define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) { define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_pd_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> -; CHECK-NEXT: ret <4 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: ret <4 x double> [[A]] ; %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> ) ret <4 x double> %a @@ -214,8 +214,8 @@ define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) { define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) { ; CHECK-LABEL: @undef_test_vpermilvar_pd_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> -; CHECK-NEXT: ret <8 x double> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> +; CHECK-NEXT: ret <8 x double> [[A]] ; %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> ) ret <8 x double> %a diff --git a/llvm/test/Transforms/InstCombine/obfuscated_splat-inseltpoison.ll b/llvm/test/Transforms/InstCombine/obfuscated_splat-inseltpoison.ll deleted file mode 100644 index bd2a8d02fed2b..0000000000000 --- a/llvm/test/Transforms/InstCombine/obfuscated_splat-inseltpoison.ll +++ /dev/null @@ -1,11 +0,0 @@ -; RUN: opt -passes=instcombine -S < %s | FileCheck %s - -define void @test(ptr %in_ptr, ptr %out_ptr) { - %A = load <4 x float>, ptr %in_ptr, align 16 - %B = shufflevector <4 x float> %A, <4 x float> poison, <4 x i32> - %C = shufflevector <4 x float> %B, <4 x float> %A, <4 x i32> - %D = shufflevector <4 x float> %C, <4 x float> %A, <4 x i32> -; CHECK: %D = shufflevector <4 x float> %A, <4 x float> poison, <4 x i32> zeroinitializer - store <4 x float> %D, ptr %out_ptr - ret void -} diff --git a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll index 181e75b6eb108..949c41900c8dd 100644 --- a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll +++ b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll @@ -1,11 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -passes=instcombine -S < %s | FileCheck %s -define void @test(ptr %in_ptr, ptr %out_ptr) { +define void @test_undef(ptr %in_ptr, ptr %out_ptr) { +; CHECK-LABEL: define void @test_undef( +; CHECK-SAME: ptr [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) { +; CHECK-NEXT: [[A:%.*]] = load <4 x float>, ptr [[IN_PTR]], align 16 +; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: store <4 x float> [[D]], ptr [[OUT_PTR]], align 16 +; CHECK-NEXT: ret void +; %A = load <4 x float>, ptr %in_ptr, align 16 %B = shufflevector <4 x float> %A, <4 x float> undef, <4 x i32> %C = shufflevector <4 x float> %B, <4 x float> %A, <4 x i32> %D = shufflevector <4 x float> %C, <4 x float> %A, <4 x i32> -; CHECK: %D = shufflevector <4 x float> %A, <4 x float> poison, <4 x i32> zeroinitializer + store <4 x float> %D, ptr %out_ptr + ret void +} + +define void @test_poison(ptr %in_ptr, ptr %out_ptr) { +; CHECK-LABEL: define void @test_poison( +; CHECK-SAME: ptr [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) { +; CHECK-NEXT: [[A:%.*]] = load <4 x float>, ptr [[IN_PTR]], align 16 +; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: store <4 x float> [[D]], ptr [[OUT_PTR]], align 16 +; CHECK-NEXT: ret void +; + %A = load <4 x float>, ptr %in_ptr, align 16 + %B = shufflevector <4 x float> %A, <4 x float> poison, <4 x i32> + %C = shufflevector <4 x float> %B, <4 x float> %A, <4 x i32> + %D = shufflevector <4 x float> %C, <4 x float> %A, <4 x i32> store <4 x float> %D, ptr %out_ptr ret void } From 7e5019e82bb3d7c7a573aa9b3bd7c16a920fd453 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Mon, 18 Dec 2023 14:16:02 +0000 Subject: [PATCH 114/884] [AMDGPU] Simplify WaitcntBrackets::getRegInterval with getPhysRegBaseClass (#74087) This means that getRegInterval no longer depends on the MCInstrDesc, so it could be simplified further to take just a MachineOperand or just a physical register. NFCI. --- llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp index 8415a3d77d3bc..55ddb540c51e5 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp @@ -238,7 +238,7 @@ class WaitcntBrackets { bool merge(const WaitcntBrackets &Other); - RegInterval getRegInterval(const MachineInstr *MI, const SIInstrInfo *TII, + RegInterval getRegInterval(const MachineInstr *MI, const MachineRegisterInfo *MRI, const SIRegisterInfo *TRI, unsigned OpNo) const; @@ -500,7 +500,6 @@ class SIInsertWaitcnts : public MachineFunctionPass { } // end anonymous namespace RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI, - const SIInstrInfo *TII, const MachineRegisterInfo *MRI, const SIRegisterInfo *TRI, unsigned OpNo) const { @@ -534,7 +533,7 @@ RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI, else return {-1, -1}; - const TargetRegisterClass *RC = TII->getOpRegClass(*MI, OpNo); + const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Op.getReg()); unsigned Size = TRI->getRegSizeInBits(*RC); Result.second = Result.first + ((Size + 16) / 32); @@ -546,7 +545,7 @@ void WaitcntBrackets::setExpScore(const MachineInstr *MI, const SIRegisterInfo *TRI, const MachineRegisterInfo *MRI, unsigned OpNo, unsigned Val) { - RegInterval Interval = getRegInterval(MI, TII, MRI, TRI, OpNo); + RegInterval Interval = getRegInterval(MI, MRI, TRI, OpNo); assert(TRI->isVectorRegister(*MRI, MI->getOperand(OpNo).getReg())); for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) { setRegScore(RegNo, EXP_CNT, Val); @@ -674,7 +673,7 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII, Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) { MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data); unsigned OpNo;//TODO: find the OpNo for this operand; - RegInterval Interval = getRegInterval(&Inst, TII, MRI, TRI, OpNo); + RegInterval Interval = getRegInterval(&Inst, MRI, TRI, OpNo); for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) { setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore); @@ -686,7 +685,7 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII, auto &Op = Inst.getOperand(I); if (!Op.isReg() || !Op.isDef()) continue; - RegInterval Interval = getRegInterval(&Inst, TII, MRI, TRI, I); + RegInterval Interval = getRegInterval(&Inst, MRI, TRI, I); if (T == VM_CNT) { if (Interval.first >= NUM_ALL_VGPRS) continue; @@ -1140,7 +1139,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI, if (MI.getOperand(CallAddrOpIdx).isReg()) { RegInterval CallAddrOpInterval = - ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, CallAddrOpIdx); + ScoreBrackets.getRegInterval(&MI, MRI, TRI, CallAddrOpIdx); for (int RegNo = CallAddrOpInterval.first; RegNo < CallAddrOpInterval.second; ++RegNo) @@ -1150,7 +1149,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI, AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); if (RtnAddrOpIdx != -1) { RegInterval RtnAddrOpInterval = - ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, RtnAddrOpIdx); + ScoreBrackets.getRegInterval(&MI, MRI, TRI, RtnAddrOpIdx); for (int RegNo = RtnAddrOpInterval.first; RegNo < RtnAddrOpInterval.second; ++RegNo) @@ -1202,8 +1201,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI, if (Op.isTied() && Op.isUse() && TII->doesNotReadTiedSource(MI)) continue; - RegInterval Interval = - ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, I); + RegInterval Interval = ScoreBrackets.getRegInterval(&MI, MRI, TRI, I); const bool IsVGPR = TRI->isVectorRegister(*MRI, Op.getReg()); for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) { @@ -1782,7 +1780,7 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML, MachineOperand &Op = MI.getOperand(I); if (!Op.isReg() || !TRI->isVectorRegister(*MRI, Op.getReg())) continue; - RegInterval Interval = Brackets.getRegInterval(&MI, TII, MRI, TRI, I); + RegInterval Interval = Brackets.getRegInterval(&MI, MRI, TRI, I); // Vgpr use if (Op.isUse()) { for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) { From 6c9813aa02ed8cbf518045e15787dce352e5e84d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 14:44:32 +0100 Subject: [PATCH 115/884] [InstCombine] Check for poison instead of undef in shuffle combine Otherwise we may replace undef with poison. Note that a lot of tests regressing here already have variants that use poison instead of undef (often in a separate inseltpoison file), which is why I'm not adjusting them to the new pattern. --- .../Transforms/InstCombine/InstCombineVectorOps.cpp | 7 ++++--- .../amdgcn-demanded-vector-elts-inseltpoison.ll | 2 +- .../AMDGPU/amdgcn-demanded-vector-elts.ll | 2 +- llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll | 4 ++-- .../test/Transforms/InstCombine/obfuscated_splat.ll | 4 +++- .../Transforms/InstCombine/trunc-extractelement.ll | 3 ++- llvm/test/Transforms/InstCombine/vec_shuffle.ll | 13 +++++++++---- 7 files changed, 22 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index bd5f608045cf1..a1caa67b361c7 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -3013,10 +3013,11 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { ShuffleVectorInst* LHSShuffle = dyn_cast(LHS); ShuffleVectorInst* RHSShuffle = dyn_cast(RHS); if (LHSShuffle) - if (!match(LHSShuffle->getOperand(1), m_Undef()) && !match(RHS, m_Undef())) + if (!match(LHSShuffle->getOperand(1), m_Poison()) && + !match(RHS, m_Poison())) LHSShuffle = nullptr; if (RHSShuffle) - if (!match(RHSShuffle->getOperand(1), m_Undef())) + if (!match(RHSShuffle->getOperand(1), m_Poison())) RHSShuffle = nullptr; if (!LHSShuffle && !RHSShuffle) return MadeChange ? &SVI : nullptr; @@ -3039,7 +3040,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { Value* newRHS = RHS; if (LHSShuffle) { // case 1 - if (match(RHS, m_Undef())) { + if (match(RHS, m_Poison())) { newLHS = LHSOp0; newRHS = LHSOp1; } diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll index b10ce7dac9585..ae0f4299ca181 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll @@ -238,7 +238,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3 %elt2 = extractelement <4 x float> %data, i32 2 %ins0 = insertelement <2 x float> poison, float %elt0, i32 0 %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1 - %shuf = shufflevector <4 x float> undef, <4 x float> %data, <2 x i32> + %shuf = shufflevector <4 x float> poison, <4 x float> %data, <2 x i32> %ret = fadd <2 x float> %ins1, %shuf ret <2 x float> %ret } diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll index 70482aa7a588f..f2a8e563d2d47 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll @@ -238,7 +238,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3 %elt2 = extractelement <4 x float> %data, i32 2 %ins0 = insertelement <2 x float> undef, float %elt0, i32 0 %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1 - %shuf = shufflevector <4 x float> undef, <4 x float> %data, <2 x i32> + %shuf = shufflevector <4 x float> poison, <4 x float> %data, <2 x i32> %ret = fadd <2 x float> %ins1, %shuf ret <2 x float> %ret } diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll index 9e72b5ad0e1a2..ec71ff1fffe4f 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll @@ -236,7 +236,7 @@ define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) { define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_ps_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> ; CHECK-NEXT: ret <8 x float> [[TMP1]] ; %1 = shufflevector <8 x i32> %a1, <8 x i32> , <8 x i32> @@ -270,7 +270,7 @@ define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) { define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_pd_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> ; CHECK-NEXT: ret <4 x double> [[TMP1]] ; %1 = shufflevector <4 x i64> , <4 x i64> %a1, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll index 949c41900c8dd..5752612b0b6c9 100644 --- a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll +++ b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll @@ -5,7 +5,9 @@ define void @test_undef(ptr %in_ptr, ptr %out_ptr) { ; CHECK-LABEL: define void @test_undef( ; CHECK-SAME: ptr [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) { ; CHECK-NEXT: [[A:%.*]] = load <4 x float>, ptr [[IN_PTR]], align 16 -; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[B:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[A]], <4 x i32> +; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[C]], <4 x float> [[A]], <4 x i32> ; CHECK-NEXT: store <4 x float> [[D]], ptr [[OUT_PTR]], align 16 ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll index 38882de08a47c..b6b5bc4562a3e 100644 --- a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll +++ b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll @@ -182,7 +182,8 @@ define <4 x i64> @PR45314(<4 x i64> %x) { ; ; BE-LABEL: @PR45314( ; BE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> +; BE-NEXT: [[I:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> +; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[I]], <8 x i32> undef, <8 x i32> zeroinitializer ; BE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64> ; BE-NEXT: ret <4 x i64> [[B]] ; diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index 978d90d7df94e..ff281eb54aebe 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -108,7 +108,8 @@ define <4 x float> @test8(<4 x float> %x, <4 x float> %y) { ; different length then the second. define <4 x i8> @test9(<16 x i8> %t6) { ; CHECK-LABEL: @test9( -; CHECK-NEXT: [[T9:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[T9]] ; %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 13, i32 9, i32 4, i32 13 > @@ -135,7 +136,8 @@ define <4 x i8> @test9a(<16 x i8> %t6) { ; different length then the second. define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) { ; CHECK-LABEL: @test9b( -; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <4 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <8 x i32> +; CHECK-NEXT: [[T9:%.*]] = shufflevector <8 x i8> [[T1]], <8 x i8> undef, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[T9]] ; %t1 = shufflevector <4 x i8> %t6, <4 x i8> %t7, <8 x i32> @@ -146,7 +148,8 @@ define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) { ; Redundant vector splats should be removed. Radar 8597790. define <4 x i32> @test10(<4 x i32> %t5) { ; CHECK-LABEL: @test10( -; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T6]], <4 x i32> undef, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i32> [[T7]] ; %t6 = shufflevector <4 x i32> %t5, <4 x i32> undef, <4 x i32> @@ -158,7 +161,9 @@ define <4 x i32> @test10(<4 x i32> %t5) { define <8 x i8> @test11(<16 x i8> %t6) { ; CHECK-LABEL: @test11( -; CHECK-NEXT: [[T3:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <8 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T2:%.*]] = shufflevector <16 x i8> [[T6]], <16 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T3:%.*]] = shufflevector <4 x i8> [[T1]], <4 x i8> [[T2]], <8 x i32> ; CHECK-NEXT: ret <8 x i8> [[T3]] ; %t1 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> From 629412938b01773fe92cb5fd12f1c434fd80884d Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Mon, 18 Dec 2023 21:28:33 +0700 Subject: [PATCH 116/884] Triple: Fix handling of macos with unexpected target arches (#75469) Some tools with a specified target arch, but no full triple default to the host triple. On macos hosts, this would then force using macho on targets that didn't expect it, resulting in assertions. We should also probably emit explicit errors if the object format is specified on targets which don't handle it. --- llvm/lib/TargetParser/Triple.cpp | 15 ++++++++------- llvm/unittests/TargetParser/TripleTest.cpp | 7 +++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index ac04dab048971..d475650c2d18c 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -819,8 +819,6 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) { } static Triple::ObjectFormatType getDefaultFormat(const Triple &T) { - if (T.isOSDarwin()) - return Triple::MachO; switch (T.getArch()) { case Triple::UnknownArch: case Triple::aarch64: @@ -829,12 +827,13 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) { case Triple::thumb: case Triple::x86: case Triple::x86_64: - if (T.isOSWindows()) + switch (T.getOS()) { + case Triple::Win32: + case Triple::UEFI: return Triple::COFF; - else if (T.isUEFI()) - return Triple::COFF; - return Triple::ELF; - + default: + return T.isOSDarwin() ? Triple::MachO : Triple::ELF; + } case Triple::aarch64_be: case Triple::amdgcn: case Triple::amdil64: @@ -887,6 +886,8 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) { case Triple::ppc: if (T.isOSAIX()) return Triple::XCOFF; + if (T.isOSDarwin()) + return Triple::MachO; return Triple::ELF; case Triple::systemz: diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp index 575e2ca381df3..d3bde2986ea2c 100644 --- a/llvm/unittests/TargetParser/TripleTest.cpp +++ b/llvm/unittests/TargetParser/TripleTest.cpp @@ -2037,6 +2037,13 @@ TEST(TripleTest, FileFormat) { T.setObjectFormat(Triple::SPIRV); EXPECT_EQ(Triple::SPIRV, T.getObjectFormat()); EXPECT_EQ("spirv", Triple::getObjectFormatTypeName(T.getObjectFormat())); + + EXPECT_EQ(Triple::ELF, Triple("amdgcn-apple-macosx").getObjectFormat()); + EXPECT_EQ(Triple::ELF, Triple("r600-apple-macosx").getObjectFormat()); + EXPECT_EQ(Triple::SPIRV, Triple("spirv-apple-macosx").getObjectFormat()); + EXPECT_EQ(Triple::SPIRV, Triple("spirv32-apple-macosx").getObjectFormat()); + EXPECT_EQ(Triple::SPIRV, Triple("spirv64-apple-macosx").getObjectFormat()); + EXPECT_EQ(Triple::DXContainer, Triple("dxil-apple-macosx").getObjectFormat()); } TEST(TripleTest, NormalizeWindows) { From e93d324adb4ff80dcbf0e5b678ed3342350eb2de Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 15:33:19 +0100 Subject: [PATCH 117/884] [InstCombine] Preserve poison in evaluateInDifferentElementOrder() Don't unnecessarily replace poison with undef. --- .../InstCombine/InstCombineVectorOps.cpp | 4 ++ .../amdgcn-simplify-image-buffer-stores.ll | 4 +- .../shufflevector-div-rem-inseltpoison.ll | 4 +- .../InstCombine/vec_shuffle-inseltpoison.ll | 6 +-- .../invariant-store-vectorization-2.ll | 40 +++++++++---------- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index a1caa67b361c7..c18fd5d99d097 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -1919,6 +1919,10 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef Mask, assert(V->getType()->isVectorTy() && "can't reorder non-vector elements"); Type *EltTy = V->getType()->getScalarType(); + + if (isa(V)) + return PoisonValue::get(FixedVectorType::get(EltTy, Mask.size())); + if (match(V, m_Undef())) return UndefValue::get(FixedVectorType::get(EltTy, Mask.size())); diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll index 961d812459820..f2d904cce7f00 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll @@ -57,13 +57,13 @@ define amdgpu_ps void @image_store_mip_1d_store_insert_zeros_at_end(<8 x i32> in define amdgpu_ps void @buffer_store_format_insert_zeros_at_end(<4 x i32> inreg %a, float %vdata1, i32 %b) { ; GCN-LABEL: @buffer_store_format_insert_zeros_at_end( -; GCN-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[VDATA1:%.*]], i64 0 +; GCN-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[VDATA1:%.*]], i64 0 ; GCN-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer ; GCN-NEXT: call void @llvm.amdgcn.buffer.store.format.v2f32(<2 x float> [[TMP2]], <4 x i32> [[A:%.*]], i32 [[B:%.*]], i32 0, i1 false, i1 false) ; GCN-NEXT: ret void ; ; GFX12-LABEL: @buffer_store_format_insert_zeros_at_end( -; GFX12-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[VDATA1:%.*]], i64 0 +; GFX12-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[VDATA1:%.*]], i64 0 ; GFX12-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer ; GFX12-NEXT: call void @llvm.amdgcn.buffer.store.format.v2f32(<2 x float> [[TMP2]], <4 x i32> [[A:%.*]], i32 [[B:%.*]], i32 0, i1 false, i1 false) ; GFX12-NEXT: ret void diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll index 253d42e9029df..cc1d23943b09c 100644 --- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll @@ -88,7 +88,7 @@ define <2 x i16> @test_udiv(i16 %a, i1 %cmp) { ; shufflevector is eliminated here. define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) { ; CHECK-LABEL: @test_fdiv( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A:%.*]], i64 1 ; CHECK-NEXT: [[SPLAT_OP:%.*]] = fdiv <2 x float> [[TMP1]], ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> , <2 x float> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x float> [[T2]] @@ -105,7 +105,7 @@ define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) { ; shufflevector is eliminated here. define <2 x float> @test_frem(float %a, float %b, i1 %cmp) { ; CHECK-LABEL: @test_frem( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A:%.*]], i64 1 ; CHECK-NEXT: [[SPLAT_OP:%.*]] = frem <2 x float> [[TMP1]], ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> , <2 x float> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x float> [[T2]] diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll index f664c3e78914c..bedabf4e3456d 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll @@ -229,7 +229,7 @@ define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y define <2 x i8> @test13a(i8 %x1, i8 %x2) { ; CHECK-LABEL: @test13a( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X2:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[X2:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X1:%.*]], i64 1 ; CHECK-NEXT: [[D:%.*]] = add <2 x i8> [[TMP2]], ; CHECK-NEXT: ret <2 x i8> [[D]] @@ -279,7 +279,7 @@ define <3 x i32> @div_wider(i32 %y, i32 %z) { define <3 x i8> @fold_inselts_with_widening_shuffle(i8 %x, i8 %y) { ; CHECK-LABEL: @fold_inselts_with_widening_shuffle( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i8> poison, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[WIDEN:%.*]] = insertelement <3 x i8> [[TMP1]], i8 [[Y:%.*]], i64 1 ; CHECK-NEXT: ret <3 x i8> [[WIDEN]] ; @@ -301,7 +301,7 @@ define <2 x i8> @test13b(i8 %x) { define <2 x i8> @test13c(i8 %x1, i8 %x2) { ; CHECK-LABEL: @test13c( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[X1:%.*]], i64 0 ; CHECK-NEXT: [[C:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i64 1 ; CHECK-NEXT: ret <2 x i8> [[C]] ; diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll index d10b8bf458dfd..1652a8e4ca37d 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll @@ -27,10 +27,10 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64 ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) ; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 -; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] -; CHECK-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[B]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: @@ -43,12 +43,12 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope !0, !noalias !3 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]] ; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT4]], ptr [[TMP1]], align 4, !alias.scope !0, !noalias !3 +; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT4]], ptr [[TMP1]], align 4, !alias.scope [[META0]], !noalias [[META3]] ; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[DOTNOT]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i32> [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3 -; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope !3 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META3]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -128,17 +128,17 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) ; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 -; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] -; CHECK-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[B]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775804 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i64 3 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i1> poison, i1 [[CMP]], i64 3 ; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i64 3 ; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT6]] ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3 @@ -146,8 +146,8 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4, !alias.scope !9, !noalias !12 -; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope !12 +; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]] +; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META12]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -216,12 +216,12 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) ; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 -; CHECK-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[A]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[B]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: @@ -231,9 +231,9 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope !16 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META16:![0-9]+]] ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 -; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope !19, !noalias !16 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META16]] ; CHECK-NEXT: [[TMP3]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] From 6d91905f9786943139150bf9e2b1f10ba92444d5 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 12:27:14 +0100 Subject: [PATCH 118/884] [ValueTracking] Short-circuit on unknown bits in isKnownNonEqual() (NFC) Don't bother computing known bits for the second operand if we know nothing about the first. --- llvm/lib/Analysis/ValueTracking.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 45fdd4eda47d7..9f709a51fd6ba 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -3184,11 +3184,12 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth, // Are any known bits in V1 contradictory to known bits in V2? If V1 // has a known zero where V2 has a known one, they must not be equal. KnownBits Known1 = computeKnownBits(V1, Depth, Q); - KnownBits Known2 = computeKnownBits(V2, Depth, Q); - - if (Known1.Zero.intersects(Known2.One) || - Known2.Zero.intersects(Known1.One)) - return true; + if (!Known1.isUnknown()) { + KnownBits Known2 = computeKnownBits(V2, Depth, Q); + if (Known1.Zero.intersects(Known2.One) || + Known2.Zero.intersects(Known1.One)) + return true; + } } if (isNonEqualSelect(V1, V2, Depth, Q) || isNonEqualSelect(V2, V1, Depth, Q)) From 58a2c4e2f24ffce3966c3988d1a4ca7b04c52244 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Mon, 18 Dec 2023 06:35:16 -0800 Subject: [PATCH 119/884] [SLP][NFC]Check for equal opcode preliminary to meet weak strict order requirement, NFC. This change does not affect functionality, just fixes the assertions in some standard c++ library implementations. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 9d799124074ca..8c3ed0888e463 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15777,6 +15777,8 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, if (NodeI1 != NodeI2) return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn(); } + if (I1->getOpcode() == I2->getOpcode()) + continue; InstructionsState S = getSameOpcode({I1, I2}, TLI); if (S.getOpcode() && (IsCompatibility || !S.isAltShuffle())) continue; From 4ef6587715bec4520332528c4a71fe5a9ac10477 Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Mon, 18 Dec 2023 09:47:59 -0500 Subject: [PATCH 120/884] [Clang][OpenMP] Fix mapping of structs to device (#75642) Fix mapping of structs to device. The following example fails: ``` #include #include struct Descriptor { int *datum; long int x; int xi; long int arr[1][30]; }; int main() { Descriptor dat = Descriptor(); dat.datum = (int *)malloc(sizeof(int)*10); dat.xi = 3; dat.arr[0][0] = 1; #pragma omp target enter data map(to: dat.datum[:10]) map(to: dat) #pragma omp target { dat.xi = 4; dat.datum[dat.arr[0][0]] = dat.xi; } #pragma omp target exit data map(from: dat) return 0; } ``` This is a rework of the previous attempt: https://github.com/llvm/llvm-project/pull/72410 --- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 148 +++++++++++---- clang/test/OpenMP/map_struct_ordering.cpp | 172 ++++++++++++++++++ .../struct_mapping_with_pointers.cpp | 114 ++++++++++++ 3 files changed, 401 insertions(+), 33 deletions(-) create mode 100644 clang/test/OpenMP/map_struct_ordering.cpp create mode 100644 openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 7f7e6f5306664..ea6645a39e832 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -6811,8 +6811,10 @@ class MappableExprsHandler { OpenMPMapClauseKind MapType, ArrayRef MapModifiers, ArrayRef MotionModifiers, OMPClauseMappableExprCommon::MappableExprComponentListRef Components, - MapCombinedInfoTy &CombinedInfo, StructRangeInfoTy &PartialStruct, - bool IsFirstComponentList, bool IsImplicit, + MapCombinedInfoTy &CombinedInfo, + MapCombinedInfoTy &StructBaseCombinedInfo, + StructRangeInfoTy &PartialStruct, bool IsFirstComponentList, + bool IsImplicit, bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr, ArrayRef @@ -7098,6 +7100,25 @@ class MappableExprsHandler { bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous; bool IsPrevMemberReference = false; + // We need to check if we will be encountering any MEs. If we do not + // encounter any ME expression it means we will be mapping the whole struct. + // In that case we need to skip adding an entry for the struct to the + // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo + // list only when generating all info for clauses. + bool IsMappingWholeStruct = true; + if (!GenerateAllInfoForClauses) { + IsMappingWholeStruct = false; + } else { + for (auto TempI = I; TempI != CE; ++TempI) { + const MemberExpr *PossibleME = + dyn_cast(TempI->getAssociatedExpression()); + if (PossibleME) { + IsMappingWholeStruct = false; + break; + } + } + } + for (; I != CE; ++I) { // If the current component is member of a struct (parent struct) mark it. if (!EncounteredME) { @@ -7317,21 +7338,41 @@ class MappableExprsHandler { break; } llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression()); + // Skip adding an entry in the CurInfo of this combined entry if the + // whole struct is currently being mapped. The struct needs to be added + // in the first position before any data internal to the struct is being + // mapped. if (!IsMemberPointerOrAddr || (Next == CE && MapType != OMPC_MAP_unknown)) { - CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); - CombinedInfo.DevicePtrDecls.push_back(nullptr); - CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); - CombinedInfo.Sizes.push_back( - CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); - CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize - : 1); + if (!IsMappingWholeStruct) { + CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); + CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.DevicePtrDecls.push_back(nullptr); + CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); + CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( + Size, CGF.Int64Ty, /*isSigned=*/true)); + CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize + : 1); + } else { + StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); + StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); + StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); + StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); + StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); + StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( + Size, CGF.Int64Ty, /*isSigned=*/true)); + StructBaseCombinedInfo.NonContigInfo.Dims.push_back( + IsNonContiguous ? DimSize : 1); + } // If Mapper is valid, the last component inherits the mapper. bool HasMapper = Mapper && Next == CE; - CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr); + if (!IsMappingWholeStruct) + CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr); + else + StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper + : nullptr); // We need to add a pointer flag for each map that comes from the // same expression except for the first one. We also need to signal @@ -7363,7 +7404,10 @@ class MappableExprsHandler { } } - CombinedInfo.Types.push_back(Flags); + if (!IsMappingWholeStruct) + CombinedInfo.Types.push_back(Flags); + else + StructBaseCombinedInfo.Types.push_back(Flags); } // If we have encountered a member expression so far, keep track of the @@ -7954,8 +7998,10 @@ class MappableExprsHandler { for (const auto &Data : Info) { StructRangeInfoTy PartialStruct; - // Temporary generated information. + // Current struct information: MapCombinedInfoTy CurInfo; + // Current struct base information: + MapCombinedInfoTy StructBaseCurInfo; const Decl *D = Data.first; const ValueDecl *VD = cast_or_null(D); for (const auto &M : Data.second) { @@ -7965,29 +8011,55 @@ class MappableExprsHandler { // Remember the current base pointer index. unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size(); + unsigned StructBasePointersIdx = + StructBaseCurInfo.BasePointers.size(); CurInfo.NonContigInfo.IsNonContiguous = L.Components.back().isNonContiguous(); generateInfoForComponentList( L.MapType, L.MapModifiers, L.MotionModifiers, L.Components, - CurInfo, PartialStruct, /*IsFirstComponentList=*/false, - L.IsImplicit, L.Mapper, L.ForDeviceAddr, VD, L.VarRef); + CurInfo, StructBaseCurInfo, PartialStruct, + /*IsFirstComponentList=*/false, L.IsImplicit, + /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD, + L.VarRef); - // If this entry relates with a device pointer, set the relevant + // If this entry relates to a device pointer, set the relevant // declaration and add the 'return pointer' flag. if (L.ReturnDevicePointer) { - assert(CurInfo.BasePointers.size() > CurrentBasePointersIdx && + // Check whether a value was added to either CurInfo or + // StructBaseCurInfo and error if no value was added to either of + // them: + assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() || + StructBasePointersIdx < + StructBaseCurInfo.BasePointers.size()) && "Unexpected number of mapped base pointers."); + // Choose a base pointer index which is always valid: const ValueDecl *RelevantVD = L.Components.back().getAssociatedDeclaration(); assert(RelevantVD && "No relevant declaration related with device pointer??"); - CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD; - CurInfo.DevicePointers[CurrentBasePointersIdx] = - L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer; - CurInfo.Types[CurrentBasePointersIdx] |= - OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + // If StructBaseCurInfo has been updated this iteration then work on + // the first new entry added to it i.e. make sure that when multiple + // values are added to any of the lists, the first value added is + // being modified by the assignments below (not the last value + // added). + if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) { + StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] = + RelevantVD; + StructBaseCurInfo.DevicePointers[StructBasePointersIdx] = + L.ForDeviceAddr ? DeviceInfoTy::Address + : DeviceInfoTy::Pointer; + StructBaseCurInfo.Types[StructBasePointersIdx] |= + OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + } else { + CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD; + CurInfo.DevicePointers[CurrentBasePointersIdx] = + L.ForDeviceAddr ? DeviceInfoTy::Address + : DeviceInfoTy::Pointer; + CurInfo.Types[CurrentBasePointersIdx] |= + OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; + } } } } @@ -8034,17 +8106,24 @@ class MappableExprsHandler { CurInfo.Mappers.push_back(nullptr); } } + + // Unify entries in one list making sure the struct mapping precedes the + // individual fields: + MapCombinedInfoTy UnionCurInfo; + UnionCurInfo.append(StructBaseCurInfo); + UnionCurInfo.append(CurInfo); + // If there is an entry in PartialStruct it means we have a struct with // individual members mapped. Emit an extra combined entry. if (PartialStruct.Base.isValid()) { - CurInfo.NonContigInfo.Dims.push_back(0); - emitCombinedEntry(CombinedInfo, CurInfo.Types, PartialStruct, + UnionCurInfo.NonContigInfo.Dims.push_back(0); + // Emit a combined entry: + emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct, /*IsMapThis*/ !VD, OMPBuilder, VD); } - // We need to append the results of this capture to what we already - // have. - CombinedInfo.append(CurInfo); + // We need to append the results of this capture to what we already have. + CombinedInfo.append(UnionCurInfo); } // Append data for use_device_ptr clauses. CombinedInfo.append(UseDeviceDataCombinedInfo); @@ -8554,6 +8633,7 @@ class MappableExprsHandler { // Associated with a capture, because the mapping flags depend on it. // Go through all of the elements with the overlapped elements. bool IsFirstComponentList = true; + MapCombinedInfoTy StructBaseCombinedInfo; for (const auto &Pair : OverlappedData) { const MapData &L = *Pair.getFirst(); OMPClauseMappableExprCommon::MappableExprComponentListRef Components; @@ -8568,7 +8648,8 @@ class MappableExprsHandler { OverlappedComponents = Pair.getSecond(); generateInfoForComponentList( MapType, MapModifiers, std::nullopt, Components, CombinedInfo, - PartialStruct, IsFirstComponentList, IsImplicit, Mapper, + StructBaseCombinedInfo, PartialStruct, IsFirstComponentList, + IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents); IsFirstComponentList = false; } @@ -8584,10 +8665,11 @@ class MappableExprsHandler { L; auto It = OverlappedData.find(&L); if (It == OverlappedData.end()) - generateInfoForComponentList(MapType, MapModifiers, std::nullopt, - Components, CombinedInfo, PartialStruct, - IsFirstComponentList, IsImplicit, Mapper, - /*ForDeviceAddr=*/false, VD, VarRef); + generateInfoForComponentList( + MapType, MapModifiers, std::nullopt, Components, CombinedInfo, + StructBaseCombinedInfo, PartialStruct, IsFirstComponentList, + IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, + /*ForDeviceAddr=*/false, VD, VarRef); IsFirstComponentList = false; } } diff --git a/clang/test/OpenMP/map_struct_ordering.cpp b/clang/test/OpenMP/map_struct_ordering.cpp new file mode 100644 index 0000000000000..035b39b5b12ab --- /dev/null +++ b/clang/test/OpenMP/map_struct_ordering.cpp @@ -0,0 +1,172 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" --prefix-filecheck-ir-name _ --version 4 + +// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK + +// expected-no-diagnostics +#ifndef HEADER +#define HEADER + +struct Descriptor { + int *datum; + long int x; + int xi; + long int arr[1][30]; +}; + +int map_struct() { + Descriptor dat = Descriptor(); + dat.xi = 3; + dat.arr[0][0] = 1; + + #pragma omp target enter data map(to: dat.datum[:10]) map(to: dat) + + #pragma omp target + { + dat.xi = 4; + dat.datum[dat.arr[0][0]] = dat.xi; + } + + #pragma omp target exit data map(from: dat) + + return dat.xi; +} + +#endif +// CHECK-LABEL: define dso_local noundef signext i32 @_Z10map_structv( +// CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DAT:%.*]] = alloca [[STRUCT_DESCRIPTOR:%.*]], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_BASEPTRS:%.*]] = alloca [3 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_PTRS:%.*]] = alloca [3 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_MAPPERS:%.*]] = alloca [3 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_SIZES:%.*]] = alloca [3 x i64], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_BASEPTRS4:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_PTRS5:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_MAPPERS6:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: [[KERNEL_ARGS:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_BASEPTRS7:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_PTRS8:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: [[DOTOFFLOAD_MAPPERS9:%.*]] = alloca [1 x ptr], align 8 +// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 [[DAT]], i8 0, i64 264, i1 false) +// CHECK-NEXT: [[XI:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 0, i32 2 +// CHECK-NEXT: store i32 3, ptr [[XI]], align 8 +// CHECK-NEXT: [[ARR:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 0, i32 3 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [1 x [30 x i64]], ptr [[ARR]], i64 0, i64 0 +// CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [30 x i64], ptr [[ARRAYIDX]], i64 0, i64 0 +// CHECK-NEXT: store i64 1, ptr [[ARRAYIDX1]], align 8 +// CHECK-NEXT: [[DATUM:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 0, i32 0 +// CHECK-NEXT: [[DATUM2:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 0, i32 0 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[DATUM2]], align 8 +// CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 1 +// CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 +// CHECK-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[DAT]] to i64 +// CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP2]], [[TMP3]] +// CHECK-NEXT: [[TMP5:%.*]] = sdiv exact i64 [[TMP4]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES]], ptr align 8 @.offload_sizes, i64 24, i1 false) +// CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP6]], align 8 +// CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP7]], align 8 +// CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0 +// CHECK-NEXT: store i64 [[TMP5]], ptr [[TMP8]], align 8 +// CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 0 +// CHECK-NEXT: store ptr null, ptr [[TMP9]], align 8 +// CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP10]], align 8 +// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP11]], align 8 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1 +// CHECK-NEXT: store ptr null, ptr [[TMP12]], align 8 +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2 +// CHECK-NEXT: store ptr [[DATUM]], ptr [[TMP13]], align 8 +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2 +// CHECK-NEXT: store ptr [[ARRAYIDX3]], ptr [[TMP14]], align 8 +// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2 +// CHECK-NEXT: store ptr null, ptr [[TMP15]], align 8 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0 +// CHECK-NEXT: call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 3, ptr [[TMP16]], ptr [[TMP17]], ptr [[TMP18]], ptr @.offload_maptypes, ptr null, ptr null) +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS4]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP19]], align 8 +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS5]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP20]], align 8 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS6]], i64 0, i64 0 +// CHECK-NEXT: store ptr null, ptr [[TMP21]], align 8 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS4]], i32 0, i32 0 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS5]], i32 0, i32 0 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0 +// CHECK-NEXT: store i32 2, ptr [[TMP24]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 1 +// CHECK-NEXT: store i32 1, ptr [[TMP25]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 2 +// CHECK-NEXT: store ptr [[TMP22]], ptr [[TMP26]], align 8 +// CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 3 +// CHECK-NEXT: store ptr [[TMP23]], ptr [[TMP27]], align 8 +// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 4 +// CHECK-NEXT: store ptr @.offload_sizes.1, ptr [[TMP28]], align 8 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 5 +// CHECK-NEXT: store ptr @.offload_maptypes.2, ptr [[TMP29]], align 8 +// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 6 +// CHECK-NEXT: store ptr null, ptr [[TMP30]], align 8 +// CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 7 +// CHECK-NEXT: store ptr null, ptr [[TMP31]], align 8 +// CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 8 +// CHECK-NEXT: store i64 0, ptr [[TMP32]], align 8 +// CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 9 +// CHECK-NEXT: store i64 0, ptr [[TMP33]], align 8 +// CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 10 +// CHECK-NEXT: store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP34]], align 4 +// CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 11 +// CHECK-NEXT: store [3 x i32] zeroinitializer, ptr [[TMP35]], align 4 +// CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 12 +// CHECK-NEXT: store i32 0, ptr [[TMP36]], align 4 +// CHECK-NEXT: [[TMP37:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z10map_structv_l23.region_id, ptr [[KERNEL_ARGS]]) +// CHECK-NEXT: [[TMP38:%.*]] = icmp ne i32 [[TMP37]], 0 +// CHECK-NEXT: br i1 [[TMP38]], label [[OMP_OFFLOAD_FAILED:%.*]], label [[OMP_OFFLOAD_CONT:%.*]] +// CHECK: omp_offload.failed: +// CHECK-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z10map_structv_l23(ptr [[DAT]]) #[[ATTR3:[0-9]+]] +// CHECK-NEXT: br label [[OMP_OFFLOAD_CONT]] +// CHECK: omp_offload.cont: +// CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS7]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP39]], align 8 +// CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS8]], i32 0, i32 0 +// CHECK-NEXT: store ptr [[DAT]], ptr [[TMP40]], align 8 +// CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS9]], i64 0, i64 0 +// CHECK-NEXT: store ptr null, ptr [[TMP41]], align 8 +// CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS7]], i32 0, i32 0 +// CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS8]], i32 0, i32 0 +// CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP42]], ptr [[TMP43]], ptr @.offload_sizes.3, ptr @.offload_maptypes.4, ptr null, ptr null) +// CHECK-NEXT: [[XI10:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[DAT]], i32 0, i32 2 +// CHECK-NEXT: [[TMP44:%.*]] = load i32, ptr [[XI10]], align 8 +// CHECK-NEXT: ret i32 [[TMP44]] +// +// +// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z10map_structv_l23( +// CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(264) [[DAT:%.*]]) #[[ATTR4:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DAT_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[DAT]], ptr [[DAT_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[DAT_ADDR]], align 8 +// CHECK-NEXT: [[XI:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR:%.*]], ptr [[TMP0]], i32 0, i32 2 +// CHECK-NEXT: store i32 4, ptr [[XI]], align 8 +// CHECK-NEXT: [[XI1:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[TMP0]], i32 0, i32 2 +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[XI1]], align 8 +// CHECK-NEXT: [[DATUM:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[TMP0]], i32 0, i32 0 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[DATUM]], align 8 +// CHECK-NEXT: [[ARR:%.*]] = getelementptr inbounds [[STRUCT_DESCRIPTOR]], ptr [[TMP0]], i32 0, i32 3 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [1 x [30 x i64]], ptr [[ARR]], i64 0, i64 0 +// CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [30 x i64], ptr [[ARRAYIDX]], i64 0, i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8 +// CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 [[TMP3]] +// CHECK-NEXT: store i32 [[TMP1]], ptr [[ARRAYIDX3]], align 4 +// CHECK-NEXT: ret void +// +// +// CHECK-LABEL: define internal void @.omp_offloading.requires_reg( +// CHECK-SAME: ) #[[ATTR5:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: call void @__tgt_register_requires(i64 1) +// CHECK-NEXT: ret void +// diff --git a/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp b/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp new file mode 100644 index 0000000000000..cecafe4c58416 --- /dev/null +++ b/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp @@ -0,0 +1,114 @@ +// clang-format off +// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | %fcheck-generic +// clang-format on + +#include +#include + +struct Descriptor { + int *datum; + long int x; + int *more_datum; + int xi; + int val_datum, val_more_datum; + long int arr[1][30]; + int val_arr; +}; + +int main() { + Descriptor dat = Descriptor(); + dat.datum = (int *)malloc(sizeof(int) * 10); + dat.more_datum = (int *)malloc(sizeof(int) * 20); + dat.xi = 3; + dat.arr[0][0] = 1; + + dat.datum[7] = 7; + dat.more_datum[17] = 17; + + /// The struct is mapped with type 0x0 when the pointer fields are mapped. + /// The struct is also map explicitely by the user. The second mapping by + /// the user must not overwrite the mapping set up for the pointer fields + /// when mapping the struct happens after the mapping of the pointers. + + // clang-format off + // CHECK: omptarget --> Entry 0: Base=[[DAT_HST_PTR_BASE:0x.*]], Begin=[[DAT_HST_PTR_BASE]], Size=288, Type=0x0, Name=unknown + // CHECK: omptarget --> Entry 1: Base=[[DAT_HST_PTR_BASE]], Begin=[[DAT_HST_PTR_BASE]], Size=288, Type=0x1000000000001, Name=unknown + // CHECK: omptarget --> Entry 2: Base=[[DAT_HST_PTR_BASE]], Begin=[[DATUM_HST_PTR_BASE:0x.*]], Size=40, Type=0x1000000000011, Name=unknown + // CHECK: omptarget --> Entry 3: Base=[[MORE_DATUM_HST_PTR_BASE:0x.*]], Begin=[[MORE_DATUM_HST_PTR_BEGIN:0x.*]], Size=80, Type=0x1000000000011, Name=unknown + // clang-format on + + /// The struct will be mapped in the same order as the above entries. + + /// First argument is the struct itself and it will be mapped once. + + // clang-format off + // CHECK: omptarget --> Looking up mapping(HstPtrBegin=[[DAT_HST_PTR_BASE]], Size=288)... + // CHECK: PluginInterface --> MemoryManagerTy::allocate: size 288 with host pointer [[DAT_HST_PTR_BASE]]. + // CHECK: omptarget --> Creating new map entry with HstPtrBase=[[DAT_HST_PTR_BASE]], HstPtrBegin=[[DAT_HST_PTR_BASE]], TgtAllocBegin=[[DAT_DEVICE_PTR_BASE:0x.*]], TgtPtrBegin=[[DAT_DEVICE_PTR_BASE]], Size=288, DynRefCount=1, HoldRefCount=0, Name=unknown + // CHECK: omptarget --> Moving 288 bytes (hst:[[DAT_HST_PTR_BASE]]) -> (tgt:[[DAT_DEVICE_PTR_BASE]]) + // clang-format on + + /// Second argument is dat.datum: + // clang-format off + // CHECK: omptarget --> Looking up mapping(HstPtrBegin=[[DATUM_HST_PTR_BASE]], Size=40)... + // CHECK: PluginInterface --> MemoryManagerTy::allocate: size 40 with host pointer [[DATUM_HST_PTR_BASE]]. + // CHECK: omptarget --> Creating new map entry with HstPtrBase=[[DATUM_HST_PTR_BASE]], HstPtrBegin=[[DATUM_HST_PTR_BASE]], TgtAllocBegin=[[DATUM_DEVICE_PTR_BASE:0x.*]], TgtPtrBegin=[[DATUM_DEVICE_PTR_BASE]], Size=40, DynRefCount=1, HoldRefCount=0, Name=unknown + // CHECK: omptarget --> Moving 40 bytes (hst:[[DATUM_HST_PTR_BASE]]) -> (tgt:[[DATUM_DEVICE_PTR_BASE]]) + // clang-format on + + /// Third argument is dat.more_datum: + // clang-format off + // CHECK: omptarget --> Looking up mapping(HstPtrBegin=[[MORE_DATUM_HST_PTR_BEGIN]], Size=80)... + // CHECK: PluginInterface --> MemoryManagerTy::allocate: size 80 with host pointer [[MORE_DATUM_HST_PTR_BEGIN]]. + // CHECK: omptarget --> Creating new map entry with HstPtrBase=[[MORE_DATUM_HST_PTR_BEGIN]], HstPtrBegin=[[MORE_DATUM_HST_PTR_BEGIN]], TgtAllocBegin=[[MORE_DATUM_DEVICE_PTR_BEGIN:0x.*]], TgtPtrBegin=[[MORE_DATUM_DEVICE_PTR_BEGIN]], Size=80, DynRefCount=1, HoldRefCount=0, Name=unknown + // CHECK: omptarget --> Moving 80 bytes (hst:[[MORE_DATUM_HST_PTR_BEGIN]]) -> (tgt:[[MORE_DATUM_DEVICE_PTR_BEGIN]]) + // clang-format on + +#pragma omp target enter data map(to : dat.datum[ : 10]) \ + map(to : dat.more_datum[ : 20]) map(to : dat) + + /// Checks induced by having a target region: + // clang-format off + // CHECK: omptarget --> Entry 0: Base=[[DAT_HST_PTR_BASE]], Begin=[[DAT_HST_PTR_BASE]], Size=288, Type=0x223, Name=unknown + // CHECK: omptarget --> Mapping exists (implicit) with HstPtrBegin=[[DAT_HST_PTR_BASE]], TgtPtrBegin=[[DAT_DEVICE_PTR_BASE]], Size=288, DynRefCount=2 (incremented), HoldRefCount=0, Name=unknown + // CHECK: omptarget --> Obtained target argument [[DAT_DEVICE_PTR_BASE]] from host pointer [[DAT_HST_PTR_BASE]] + // clang-format on + +#pragma omp target + { + dat.xi = 4; + dat.datum[7]++; + dat.more_datum[17]++; + dat.val_datum = dat.datum[7]; + dat.val_more_datum = dat.more_datum[17]; + dat.datum[dat.arr[0][0]] = dat.xi; + dat.val_arr = dat.datum[dat.arr[0][0]]; + } + + /// Post-target region checks: + // clang-format off + // CHECK: omptarget --> Mapping exists with HstPtrBegin=[[DAT_HST_PTR_BASE]], TgtPtrBegin=[[DAT_DEVICE_PTR_BASE]], Size=288, DynRefCount=1 (decremented), HoldRefCount=0 + // clang-format on + +#pragma omp target exit data map(from : dat) + + /// Target data end checks: + // clang-format off + // CHECK: omptarget --> Mapping exists with HstPtrBegin=[[DAT_HST_PTR_BASE]], TgtPtrBegin=[[DAT_DEVICE_PTR_BASE]], Size=288, DynRefCount=0 (decremented, delayed deletion), HoldRefCount=0 + // CHECK: omptarget --> Moving 288 bytes (tgt:[[DAT_DEVICE_PTR_BASE]]) -> (hst:[[DAT_HST_PTR_BASE]]) + // clang-format on + + // CHECK: dat.xi = 4 + // CHECK: dat.val_datum = 8 + // CHECK: dat.val_more_datum = 18 + // CHECK: dat.datum[dat.arr[0][0]] = 0 + // CHECK: dat.val_arr = 4 + + printf("dat.xi = %d\n", dat.xi); + printf("dat.val_datum = %d\n", dat.val_datum); + printf("dat.val_more_datum = %d\n", dat.val_more_datum); + printf("dat.datum[dat.arr[0][0]] = %d\n", dat.datum[dat.arr[0][0]]); + printf("dat.val_arr = %d\n", dat.val_arr); + + return 0; +} From d0605e21af516f60c081df56fc85ef13342b5aa0 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 15:52:39 +0100 Subject: [PATCH 121/884] [InstCombine] Canonicalize splat shuffles to use poison operand If the splat shuffle is represented using an undef RHS, replace it with poison. --- .../InstCombineSimplifyDemanded.cpp | 2 +- .../Transforms/InstCombine/X86/x86-avx2.ll | 46 ++++++++-------- .../Transforms/InstCombine/X86/x86-muldq.ll | 2 +- .../Transforms/InstCombine/X86/x86-pshufb.ll | 2 +- .../Transforms/InstCombine/X86/x86-vpermil.ll | 2 +- .../Transforms/InstCombine/icmp-bc-vec.ll | 2 +- llvm/test/Transforms/InstCombine/icmp-vec.ll | 52 +++++++++---------- .../InstCombine/masked_intrinsics.ll | 2 +- llvm/test/Transforms/InstCombine/shift-add.ll | 6 +-- .../InstCombine/trunc-extractelement.ll | 5 +- .../InstCombine/vec_demanded_elts.ll | 8 +-- .../Transforms/InstCombine/vec_shuffle.ll | 29 +++++------ .../multiply-fused-multiple-blocks.ll | 12 ++--- .../LowerMatrixIntrinsics/multiply-minimal.ll | 4 +- 14 files changed, 86 insertions(+), 88 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index a8ed6fe1432d9..00f97c2f96c6c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1520,7 +1520,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // operand. if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) && DemandedElts.isAllOnes()) { - if (!match(I->getOperand(1), m_Undef())) { + if (!isa(I->getOperand(1))) { I->setOperand(1, PoisonValue::get(I->getOperand(1)->getType())); MadeChange = true; } diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll index 23bcb4816af8b..be6ad78303825 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll @@ -25,8 +25,8 @@ define <8 x float> @identity_test_vpermps(<8 x float> %a0) { define <8 x i32> @zero_test_vpermd(<8 x i32> %a0) { ; CHECK-LABEL: @zero_test_vpermd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: ret <8 x i32> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: ret <8 x i32> [[A]] ; %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer) ret <8 x i32> %a @@ -34,8 +34,8 @@ define <8 x i32> @zero_test_vpermd(<8 x i32> %a0) { define <8 x float> @zero_test_vpermps(<8 x float> %a0) { ; CHECK-LABEL: @zero_test_vpermps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer) ret <8 x float> %a @@ -45,8 +45,8 @@ define <8 x float> @zero_test_vpermps(<8 x float> %a0) { define <8 x i32> @shuffle_test_vpermd(<8 x i32> %a0) { ; CHECK-LABEL: @shuffle_test_vpermd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> -; CHECK-NEXT: ret <8 x i32> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> +; CHECK-NEXT: ret <8 x i32> [[A]] ; %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> ) ret <8 x i32> %a @@ -54,8 +54,8 @@ define <8 x i32> @shuffle_test_vpermd(<8 x i32> %a0) { define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) { ; CHECK-LABEL: @shuffle_test_vpermps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> ) ret <8 x float> %a @@ -65,8 +65,8 @@ define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) { define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) { ; CHECK-LABEL: @undef_test_vpermd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> -; CHECK-NEXT: ret <8 x i32> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> +; CHECK-NEXT: ret <8 x i32> [[A]] ; %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> ) ret <8 x i32> %a @@ -74,8 +74,8 @@ define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) { define <8 x float> @undef_test_vpermps(<8 x float> %a0) { ; CHECK-LABEL: @undef_test_vpermps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP1]] +; CHECK-NEXT: [[A:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[A]] ; %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> ) ret <8 x float> %a @@ -97,7 +97,7 @@ define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) { define <8 x float> @elts_test_vpermps(<8 x float> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpermps( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: ret <8 x float> [[TMP2]] ; %1 = insertelement <8 x i32> %a1, i32 0, i32 7 @@ -109,7 +109,7 @@ define <8 x float> @elts_test_vpermps(<8 x float> %a0, <8 x i32> %a1) { define <2 x i64> @elts_test_vpsllvq(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpsllvq( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i64> [[TMP2]] ; %1 = insertelement <2 x i64> %a1, i64 0, i64 1 @@ -121,7 +121,7 @@ define <2 x i64> @elts_test_vpsllvq(<2 x i64> %a0, <2 x i64> %a1) { define <2 x i64> @elts_test_vpsrlvq(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpsrlvq( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i64> [[TMP2]] ; %1 = insertelement <2 x i64> %a1, i64 0, i64 1 @@ -133,7 +133,7 @@ define <2 x i64> @elts_test_vpsrlvq(<2 x i64> %a0, <2 x i64> %a1) { define <4 x i64> @elts_test_vpsllvq_256(<4 x i64> %a0, <4 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpsllvq_256( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i64> [[TMP2]] ; %1 = insertelement <4 x i64> %a1, i64 0, i64 2 @@ -145,7 +145,7 @@ define <4 x i64> @elts_test_vpsllvq_256(<4 x i64> %a0, <4 x i64> %a1) { define <4 x i64> @elts_test_vpsrlvq_256(<4 x i64> %a0, <4 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpsrlvq_256( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i64> [[TMP2]] ; %1 = insertelement <4 x i64> %a1, i64 0, i64 3 @@ -157,7 +157,7 @@ define <4 x i64> @elts_test_vpsrlvq_256(<4 x i64> %a0, <4 x i64> %a1) { define <4 x i32> @elts_test_vpsllvd(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsllvd( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; %1 = insertelement <4 x i32> %a1, i32 0, i64 3 @@ -169,7 +169,7 @@ define <4 x i32> @elts_test_vpsllvd(<4 x i32> %a0, <4 x i32> %a1) { define <4 x i32> @elts_test_vpsravd(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsravd( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; %1 = insertelement <4 x i32> %a1, i32 0, i64 1 @@ -181,7 +181,7 @@ define <4 x i32> @elts_test_vpsravd(<4 x i32> %a0, <4 x i32> %a1) { define <4 x i32> @elts_test_vpsrlvd(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsrlvd( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x i32> [[TMP2]] ; %1 = insertelement <4 x i32> %a1, i32 0, i64 2 @@ -193,7 +193,7 @@ define <4 x i32> @elts_test_vpsrlvd(<4 x i32> %a0, <4 x i32> %a1) { define <8 x i32> @elts_test_vpsllvd_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsllvd_256( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: ret <8 x i32> [[TMP2]] ; %1 = insertelement <8 x i32> %a1, i32 0, i64 3 @@ -205,7 +205,7 @@ define <8 x i32> @elts_test_vpsllvd_256(<8 x i32> %a0, <8 x i32> %a1) { define <8 x i32> @elts_test_vpsravd_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsravd_256( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: ret <8 x i32> [[TMP2]] ; %1 = insertelement <8 x i32> %a1, i32 0, i64 4 @@ -217,7 +217,7 @@ define <8 x i32> @elts_test_vpsravd_256(<8 x i32> %a0, <8 x i32> %a1) { define <8 x i32> @elts_test_vpsrlvd_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpsrlvd_256( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: ret <8 x i32> [[TMP2]] ; %1 = insertelement <8 x i32> %a1, i32 0, i64 5 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll index d32bce2193c2a..431079648c845 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll @@ -166,7 +166,7 @@ define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i64> [[TMP3]], ; CHECK-NEXT: [[TMP6:%.*]] = and <2 x i64> [[TMP4]], ; CHECK-NEXT: [[TMP7:%.*]] = mul nuw <2 x i64> [[TMP5]], [[TMP6]] -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i64> [[TMP8]] ; %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll index 9e49b09424da4..3af3d7a9449f5 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll @@ -500,7 +500,7 @@ define <64 x i8> @demanded_elts_insertion_avx512(<64 x i8> %InVec, <64 x i8> %Ba ; CHECK-LABEL: @demanded_elts_insertion_avx512( ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <64 x i8> poison, i8 [[M0:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[INVEC:%.*]], <64 x i8> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP2]], <64 x i8> undef, <64 x i32> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP2]], <64 x i8> poison, <64 x i32> zeroinitializer ; CHECK-NEXT: ret <64 x i8> [[TMP3]] ; %1 = insertelement <64 x i8> %BaseMask, i8 %M0, i32 0 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll index ec71ff1fffe4f..931160610b258 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll @@ -283,7 +283,7 @@ define <8 x double> @elts_test_vpermilvar_pd_512(<8 x double> %a0, <8 x i64> %a1 ; CHECK-LABEL: @elts_test_vpermilvar_pd_512( ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i64> poison, i64 [[A2:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[A0:%.*]], <8 x i64> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> undef, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: ret <8 x double> [[TMP3]] ; %1 = insertelement <8 x i64> %a1, i64 %a2, i32 0 diff --git a/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll index 99290f399b2ba..a24766ad8ab61 100644 --- a/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll +++ b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll @@ -89,7 +89,7 @@ define i1 @test_i8_pattern_3(<4 x i8> %invec) { define i1 @test_i8_nopattern(i8 %val) { ; CHECK-LABEL: @test_i8_nopattern( ; CHECK-NEXT: [[INSVEC:%.*]] = insertelement <4 x i8> undef, i8 [[VAL:%.*]], i64 0 -; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INSVEC]], <4 x i8> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INSVEC]], <4 x i8> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32 ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696647 ; CHECK-NEXT: ret i1 [[COND]] diff --git a/llvm/test/Transforms/InstCombine/icmp-vec.ll b/llvm/test/Transforms/InstCombine/icmp-vec.ll index f4db2db2a99b6..30bfb0daf7a3d 100644 --- a/llvm/test/Transforms/InstCombine/icmp-vec.ll +++ b/llvm/test/Transforms/InstCombine/icmp-vec.ll @@ -276,8 +276,8 @@ define <2 x i1> @same_shuffle_inputs_icmp_extra_use2(<4 x i8> %x, <4 x i8> %y) { define <2 x i1> @same_shuffle_inputs_icmp_extra_use3(<4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use3( -; CHECK-NEXT: [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <2 x i32> zeroinitializer -; CHECK-NEXT: [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[SHUFX]], [[SHUFY]] ; CHECK-NEXT: call void @use_v2i8(<2 x i8> [[SHUFX]]) ; CHECK-NEXT: call void @use_v2i8(<2 x i8> [[SHUFY]]) @@ -417,9 +417,9 @@ define i1 @eq_cast_eq-1(<2 x i4> %x, <2 x i4> %y) { define i1 @ne_cast_eq-1(<3 x i7> %x, <3 x i7> %y) { ; CHECK-LABEL: @ne_cast_eq-1( -; CHECK-NEXT: [[IC:%.*]] = icmp eq <3 x i7> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[IC]] to i3 -; CHECK-NEXT: [[R:%.*]] = icmp eq i3 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i7> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <3 x i1> [[TMP1]] to i3 +; CHECK-NEXT: [[R:%.*]] = icmp eq i3 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp ne <3 x i7> %x, %y @@ -430,9 +430,9 @@ define i1 @ne_cast_eq-1(<3 x i7> %x, <3 x i7> %y) { define i1 @eq_cast_ne-1(<2 x i7> %x, <2 x i7> %y) { ; CHECK-LABEL: @eq_cast_ne-1( -; CHECK-NEXT: [[IC:%.*]] = icmp ne <2 x i7> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2 -; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x i7> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i1> [[TMP1]] to i2 +; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp eq <2 x i7> %x, %y @@ -456,9 +456,9 @@ define i1 @eq_cast_ne-1-legal-scalar(<2 x i8> %x, <2 x i8> %y) { define i1 @ne_cast_ne-1(<3 x i5> %x, <3 x i5> %y) { ; CHECK-LABEL: @ne_cast_ne-1( -; CHECK-NEXT: [[IC:%.*]] = icmp eq <3 x i5> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[IC]] to i3 -; CHECK-NEXT: [[R:%.*]] = icmp ne i3 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i5> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <3 x i1> [[TMP1]] to i3 +; CHECK-NEXT: [[R:%.*]] = icmp ne i3 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp ne <3 x i5> %x, %y @@ -469,9 +469,9 @@ define i1 @ne_cast_ne-1(<3 x i5> %x, <3 x i5> %y) { define i1 @ugt_cast_eq-1(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @ugt_cast_eq-1( -; CHECK-NEXT: [[IC:%.*]] = icmp ule <2 x i4> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2 -; CHECK-NEXT: [[R:%.*]] = icmp eq i2 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i4> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i1> [[TMP1]] to i2 +; CHECK-NEXT: [[R:%.*]] = icmp eq i2 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp ugt <2 x i4> %x, %y @@ -482,9 +482,9 @@ define i1 @ugt_cast_eq-1(<2 x i4> %x, <2 x i4> %y) { define i1 @slt_cast_ne-1(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @slt_cast_ne-1( -; CHECK-NEXT: [[IC:%.*]] = icmp sge <2 x i4> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2 -; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp sge <2 x i4> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i1> [[TMP1]] to i2 +; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp slt <2 x i4> %x, %y @@ -495,9 +495,9 @@ define i1 @slt_cast_ne-1(<2 x i4> %x, <2 x i4> %y) { define i1 @ueq_cast_eq-1(<3 x float> %x, <3 x float> %y) { ; CHECK-LABEL: @ueq_cast_eq-1( -; CHECK-NEXT: [[FC:%.*]] = fcmp one <3 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[FC]] to i3 -; CHECK-NEXT: [[R:%.*]] = icmp eq i3 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = fcmp one <3 x float> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <3 x i1> [[TMP1]] to i3 +; CHECK-NEXT: [[R:%.*]] = icmp eq i3 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %fc = fcmp ueq <3 x float> %x, %y @@ -713,9 +713,9 @@ define i1 @eq_cast_zext_use2(<5 x i3> %b) { define i1 @eq_cast_eq_ptr-1(<2 x ptr> %x, <2 x ptr> %y) { ; CHECK-LABEL: @eq_cast_eq_ptr-1( -; CHECK-NEXT: [[IC:%.*]] = icmp ne <2 x ptr> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2 -; CHECK-NEXT: [[R:%.*]] = icmp eq i2 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x ptr> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i1> [[TMP1]] to i2 +; CHECK-NEXT: [[R:%.*]] = icmp eq i2 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp eq <2 x ptr> %x, %y @@ -726,9 +726,9 @@ define i1 @eq_cast_eq_ptr-1(<2 x ptr> %x, <2 x ptr> %y) { define i1 @eq_cast_ne_ptr-1(<2 x ptr> %x, <2 x ptr> %y) { ; CHECK-LABEL: @eq_cast_ne_ptr-1( -; CHECK-NEXT: [[IC:%.*]] = icmp ne <2 x ptr> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2 -; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP1]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x ptr> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i1> [[TMP1]] to i2 +; CHECK-NEXT: [[R:%.*]] = icmp ne i2 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %ic = icmp eq <2 x ptr> %x, %y diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll index 6155e0afa1001..615784d473483 100644 --- a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll @@ -202,7 +202,7 @@ define <4 x double> @gather_lane2(ptr %base, double %pt) { ; CHECK-LABEL: @gather_lane2( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> ; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <4 x double> undef, double [[PT:%.*]], i64 0 -; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> undef, <4 x i32> +; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> poison, <4 x i32> ; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> [[PT_V2]]) ; CHECK-NEXT: ret <4 x double> [[RES]] ; diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 006d303cb56d3..4e9499ca7a586 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -79,7 +79,7 @@ define <4 x i32> @shl_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @shl_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 ; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 -; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = shl <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] ; @@ -95,7 +95,7 @@ define <4 x i32> @ashr_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @ashr_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 ; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 -; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = ashr <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] ; @@ -111,7 +111,7 @@ define <4 x i32> @lshr_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @lshr_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 ; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 -; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = lshr <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] ; diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll index b6b5bc4562a3e..10fd7e5dc0e3d 100644 --- a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll +++ b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll @@ -176,14 +176,13 @@ define i16 @shrinkExtractElt_i64_to_i16_2_extra_use(<3 x i64> %x) { define <4 x i64> @PR45314(<4 x i64> %x) { ; LE-LABEL: @PR45314( ; LE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; LE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer +; LE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer ; LE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64> ; LE-NEXT: ret <4 x i64> [[B]] ; ; BE-LABEL: @PR45314( ; BE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; BE-NEXT: [[I:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> -; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[I]], <8 x i32> undef, <8 x i32> zeroinitializer +; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> ; BE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64> ; BE-NEXT: ret <4 x i64> [[B]] ; diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll index 4aa783881683a..717645c129e31 100644 --- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll +++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll @@ -528,7 +528,7 @@ define ptr @gep_splat_base_w_s_idx(ptr %base) { define ptr @gep_splat_base_w_cv_idx(ptr %base) { ; CHECK-LABEL: @gep_splat_base_w_cv_idx( -; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> undef, ptr [[BASE:%.*]], i64 1 +; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1 ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> ; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1 ; CHECK-NEXT: ret ptr [[EE]] @@ -542,7 +542,7 @@ define ptr @gep_splat_base_w_cv_idx(ptr %base) { define ptr @gep_splat_base_w_vidx(ptr %base, <2 x i64> %idxvec) { ; CHECK-LABEL: @gep_splat_base_w_vidx( -; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> undef, ptr [[BASE:%.*]], i64 1 +; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1 ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]] ; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1 ; CHECK-NEXT: ret ptr [[EE]] @@ -600,8 +600,8 @@ define ptr @gep_sbase_w_splat_idx(ptr %base, i64 %idx) { } define ptr @gep_splat_both(ptr %base, i64 %idx) { ; CHECK-LABEL: @gep_splat_both( -; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> undef, ptr [[BASE:%.*]], i64 1 -; CHECK-NEXT: [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i64 1 +; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1 +; CHECK-NEXT: [[IDXVEC2:%.*]] = insertelement <2 x i64> poison, i64 [[IDX:%.*]], i64 1 ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC2]] ; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1 ; CHECK-NEXT: ret ptr [[EE]] diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index ff281eb54aebe..ab27d75e0b783 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -148,8 +148,7 @@ define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) { ; Redundant vector splats should be removed. Radar 8597790. define <4 x i32> @test10(<4 x i32> %t5) { ; CHECK-LABEL: @test10( -; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> -; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T6]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[T7]] ; %t6 = shufflevector <4 x i32> %t5, <4 x i32> undef, <4 x i32> @@ -582,7 +581,7 @@ define <4 x i32> @lshr_const_half_splat(<4 x i32> %v) { define <2 x float> @fmul_const_invalid_constant(<2 x float> %v) { ; CHECK-LABEL: @fmul_const_invalid_constant( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[R:%.*]] = fmul <2 x float> [[T1]], ; CHECK-NEXT: ret <2 x float> [[R]] ; @@ -777,8 +776,8 @@ define i32 @pr19737(<4 x i32> %in0) { define <4 x i32> @pr20059(<4 x i32> %p1, <4 x i32> %p2) { ; CHECK-LABEL: @pr20059( -; CHECK-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[P1:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer -; CHECK-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[P2:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[P1:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[P2:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[RETVAL:%.*]] = srem <4 x i32> [[SPLAT1]], [[SPLAT2]] ; CHECK-NEXT: ret <4 x i32> [[RETVAL]] ; @@ -934,7 +933,7 @@ define <2 x i32> @lshr_splat_constant1(<2 x i32> %x) { define <2 x i32> @urem_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @urem_splat_constant0( -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[R:%.*]] = urem <2 x i32> , [[SPLAT]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -956,7 +955,7 @@ define <2 x i32> @urem_splat_constant1(<2 x i32> %x) { define <2 x i32> @srem_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @srem_splat_constant0( -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[R:%.*]] = srem <2 x i32> , [[SPLAT]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -978,7 +977,7 @@ define <2 x i32> @srem_splat_constant1(<2 x i32> %x) { define <2 x i32> @udiv_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @udiv_splat_constant0( -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[R:%.*]] = udiv <2 x i32> , [[SPLAT]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -1000,7 +999,7 @@ define <2 x i32> @udiv_splat_constant1(<2 x i32> %x) { define <2 x i32> @sdiv_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @sdiv_splat_constant0( -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[R:%.*]] = sdiv <2 x i32> , [[SPLAT]] ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -1540,7 +1539,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index(<4 x i32> %x, <4 define <4 x i32> @splat_assoc_add_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_undef_constant_elts( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] @@ -1553,7 +1552,7 @@ define <4 x i32> @splat_assoc_add_undef_constant_elts(<4 x i32> %x, <4 x i32> %y define <4 x i32> @splat_assoc_add_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_undef_constant_elt_at_splat_index( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] @@ -1713,7 +1712,7 @@ define <3 x i8> @splat_assoc_or(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) { define <2 x float> @splat_assoc_fdiv(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @splat_assoc_fdiv( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[A:%.*]] = fdiv reassoc nsz <2 x float> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = fdiv reassoc nsz <2 x float> [[A]], [[SPLATX]] ; CHECK-NEXT: ret <2 x float> [[R]] @@ -1745,7 +1744,7 @@ define <2 x float> @splat_assoc_fadd(<2 x float> %x, <2 x float> %y) { define <3 x i32> @splat_assoc_and(<4 x i32> %x, <3 x i32> %y) { ; CHECK-LABEL: @splat_assoc_and( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <3 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <3 x i32> zeroinitializer ; CHECK-NEXT: [[A:%.*]] = and <3 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = and <3 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <3 x i32> [[R]] @@ -1760,7 +1759,7 @@ define <3 x i32> @splat_assoc_and(<4 x i32> %x, <3 x i32> %y) { define <5 x i32> @splat_assoc_xor(<4 x i32> %x, <5 x i32> %y) { ; CHECK-LABEL: @splat_assoc_xor( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <5 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <5 x i32> zeroinitializer ; CHECK-NEXT: [[TMP1:%.*]] = xor <5 x i32> [[SPLATX]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = xor <5 x i32> [[TMP1]], ; CHECK-NEXT: ret <5 x i32> [[R]] @@ -1775,7 +1774,7 @@ define <5 x i32> @splat_assoc_xor(<4 x i32> %x, <5 x i32> %y) { define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_mul( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = mul <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll index 2f4846a2840e5..7671fecef7b64 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll @@ -71,11 +71,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <2 x double>, ptr [[TMP7]], align 8 ; CHECK-NEXT: [[VEC_GEP26:%.*]] = getelementptr double, ptr [[TMP7]], i64 2 ; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <2 x double>, ptr [[VEC_GEP26]], align 8 -; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT29]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> [[TMP13]]) -; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT35]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP16:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> [[TMP15]]) @@ -100,7 +100,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <1 x double>, ptr [[VEC_GEP52]], align 8 ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP7]], i64 4 ; CHECK-NEXT: [[COL_LOAD54:%.*]] = load <2 x double>, ptr [[TMP23]], align 8 -; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD51]], [[SPLAT_SPLATINSERT56]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD53]], <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> [[TMP24]]) @@ -179,11 +179,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[TMP39]], align 8 ; CHECK-NEXT: [[VEC_GEP98:%.*]] = getelementptr double, ptr [[TMP39]], i64 2 ; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[VEC_GEP98]], align 8 -; CHECK-NEXT: [[SPLAT_SPLATINSERT101:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT101:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP45:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT101]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT104:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP46:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT104]], <1 x double> [[TMP45]]) -; CHECK-NEXT: [[SPLAT_SPLATINSERT107:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT107:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP47:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT107]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT110:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP48:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT110]], <1 x double> [[TMP47]]) @@ -208,7 +208,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD125:%.*]] = load <1 x double>, ptr [[VEC_GEP124]], align 8 ; CHECK-NEXT: [[TMP55:%.*]] = getelementptr double, ptr [[TMP39]], i64 4 ; CHECK-NEXT: [[COL_LOAD126:%.*]] = load <2 x double>, ptr [[TMP55]], align 8 -; CHECK-NEXT: [[SPLAT_SPLATINSERT128:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLATINSERT128:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP56:%.*]] = fmul contract <1 x double> [[COL_LOAD123]], [[SPLAT_SPLATINSERT128]] ; CHECK-NEXT: [[SPLAT_SPLATINSERT131:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> ; CHECK-NEXT: [[TMP57:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD125]], <1 x double> [[SPLAT_SPLATINSERT131]], <1 x double> [[TMP56]]) diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll index 2dac1e51cfef3..27eb90520373d 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll @@ -23,7 +23,7 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <4 x double>, ptr [[B:%.*]], align 8 ; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B]], i64 4 ; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <4 x double>, ptr [[VEC_GEP7]], align 8 -; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] ; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT11]], <2 x double> [[TMP0]]) @@ -31,7 +31,7 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[TMP2:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP1]]) ; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT17]], <2 x double> [[TMP2]]) -; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT20]] ; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP4]]) From a35629cd8de18387c6f9d1be9111d517df88554a Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 18 Dec 2023 10:00:47 -0500 Subject: [PATCH 122/884] [libc++] Remove assumptions that std::array::iterator is a raw pointer (#74624) This patch removes assumptions that std::array's iterators are raw pointers in the source code and in our test suite. While this is true right now, this doesn't have to be true and ion the future we might want to enable bounded iterators in std::array, which would require this change. This is a pre-requisite for landing #74482 --- libcxx/include/__format/buffer.h | 12 +- libcxx/include/__format/formatter_integral.h | 37 +- libcxx/include/__format/formatter_output.h | 34 +- .../print.fun/transcoding.pass.cpp | 5 +- ...nst_proxy_iterators_lifetime_bugs.pass.cpp | 18 +- .../containers/sequences/array/types.pass.cpp | 4 - .../span.cons/iterator_sentinel.pass.cpp | 5 +- .../formatter.floating_point.pass.cpp | 594 ++++++++++-------- .../formatter.pointer.pass.cpp | 4 +- 9 files changed, 407 insertions(+), 306 deletions(-) diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h index 8aa58d6464bbf..24608a0b1d200 100644 --- a/libcxx/include/__format/buffer.h +++ b/libcxx/include/__format/buffer.h @@ -130,8 +130,10 @@ class _LIBCPP_TEMPLATE_VIS __output_buffer { /// A std::transform wrapper. /// /// Like @ref __copy it may need to do type conversion. - template <__fmt_char_type _InCharT, class _UnaryOperation> - _LIBCPP_HIDE_FROM_ABI void __transform(const _InCharT* __first, const _InCharT* __last, _UnaryOperation __operation) { + template ::value_type> + _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) { _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "not a valid range"); size_t __n = static_cast(__last - __first); @@ -590,8 +592,10 @@ class _LIBCPP_TEMPLATE_VIS __retarget_buffer { __size_ += __n; } - template <__fmt_char_type _InCharT, class _UnaryOperation> - _LIBCPP_HIDE_FROM_ABI void __transform(const _InCharT* __first, const _InCharT* __last, _UnaryOperation __operation) { + template ::value_type> + _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) { _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "not a valid range"); size_t __n = static_cast(__last - __first); diff --git a/libcxx/include/__format/formatter_integral.h b/libcxx/include/__format/formatter_integral.h index 598decb0a95ea..cbb3505bca2f2 100644 --- a/libcxx/include/__format/formatter_integral.h +++ b/libcxx/include/__format/formatter_integral.h @@ -20,6 +20,9 @@ #include <__format/format_error.h> #include <__format/formatter_output.h> #include <__format/parser_std_format_spec.h> +#include <__iterator/concepts.h> +#include <__iterator/iterator_traits.h> +#include <__memory/pointer_traits.h> #include <__system_error/errc.h> #include <__type_traits/make_unsigned.h> #include <__utility/unreachable.h> @@ -49,7 +52,9 @@ namespace __formatter { // Generic // -_LIBCPP_HIDE_FROM_ABI inline char* __insert_sign(char* __buf, bool __negative, __format_spec::__sign __sign) { +template + requires same_as> +_LIBCPP_HIDE_FROM_ABI inline _Iterator __insert_sign(_Iterator __buf, bool __negative, __format_spec::__sign __sign) { if (__negative) *__buf++ = '-'; else @@ -148,14 +153,16 @@ _LIBCPP_HIDE_FROM_ABI auto __format_char( // Integer // -/** Wrapper around @ref to_chars, returning the output pointer. */ -template -_LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value, int __base) { +/** Wrapper around @ref to_chars, returning the output iterator. */ +template + requires same_as> +_LIBCPP_HIDE_FROM_ABI _Iterator __to_buffer(_Iterator __first, _Iterator __last, _Tp __value, int __base) { // TODO FMT Evaluate code overhead due to not calling the internal function // directly. (Should be zero overhead.) - to_chars_result __r = std::to_chars(__first, __last, __value, __base); + to_chars_result __r = std::to_chars(std::to_address(__first), std::to_address(__last), __value, __base); _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); - return __r.ptr; + auto __diff = __r.ptr - std::to_address(__first); + return __first + __diff; } /** @@ -203,9 +210,10 @@ consteval size_t __buffer_size() noexcept + 1; // Reserve space for the sign. } -template -_LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, const char* __begin, const char* __first, - const char* __last, string&& __grouping, _CharT __sep, +template + requires same_as> +_LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, _Iterator __begin, _Iterator __first, + _Iterator __last, string&& __grouping, _CharT __sep, __format_spec::__parsed_specifications<_CharT> __specs) { int __size = (__first - __begin) + // [sign][prefix] (__last - __first) + // data @@ -269,22 +277,23 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, c -template +template + requires same_as> _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_integer( _Tp __value, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs, bool __negative, - char* __begin, - char* __end, + _Iterator __begin, + _Iterator __end, const char* __prefix, int __base) { - char* __first = __formatter::__insert_sign(__begin, __negative, __specs.__std_.__sign_); + _Iterator __first = __formatter::__insert_sign(__begin, __negative, __specs.__std_.__sign_); if (__specs.__std_.__alternate_form_ && __prefix) while (*__prefix) *__first++ = *__prefix++; - char* __last = __formatter::__to_buffer(__first, __end, __value, __base); + _Iterator __last = __formatter::__to_buffer(__first, __end, __value, __base); # ifndef _LIBCPP_HAS_NO_LOCALIZATION if (__specs.__std_.__locale_specific_form_) { diff --git a/libcxx/include/__format/formatter_output.h b/libcxx/include/__format/formatter_output.h index 2909fcd9baf1e..6c7892d86900d 100644 --- a/libcxx/include/__format/formatter_output.h +++ b/libcxx/include/__format/formatter_output.h @@ -23,8 +23,9 @@ #include <__format/unicode.h> #include <__iterator/back_insert_iterator.h> #include <__iterator/concepts.h> -#include <__iterator/iterator_traits.h> // iter_value_t +#include <__iterator/iterator_traits.h> #include <__memory/addressof.h> +#include <__memory/pointer_traits.h> #include <__utility/move.h> #include <__utility/unreachable.h> #include @@ -110,26 +111,32 @@ _LIBCPP_HIDE_FROM_ABI auto __copy(basic_string_view<_CharT> __str, output_iterat } } -template <__fmt_char_type _CharT, __fmt_char_type _OutCharT = _CharT> -_LIBCPP_HIDE_FROM_ABI auto -__copy(const _CharT* __first, const _CharT* __last, output_iterator auto __out_it) +template ::value_type, + __fmt_char_type _OutCharT = _CharT> +_LIBCPP_HIDE_FROM_ABI auto __copy(_Iterator __first, _Iterator __last, output_iterator auto __out_it) -> decltype(__out_it) { return __formatter::__copy(basic_string_view{__first, __last}, std::move(__out_it)); } -template <__fmt_char_type _CharT, __fmt_char_type _OutCharT = _CharT> -_LIBCPP_HIDE_FROM_ABI auto __copy(const _CharT* __first, size_t __n, output_iterator auto __out_it) +template ::value_type, + __fmt_char_type _OutCharT = _CharT> +_LIBCPP_HIDE_FROM_ABI auto __copy(_Iterator __first, size_t __n, output_iterator auto __out_it) -> decltype(__out_it) { - return __formatter::__copy(basic_string_view{__first, __n}, std::move(__out_it)); + return __formatter::__copy(basic_string_view{std::to_address(__first), __n}, std::move(__out_it)); } /// Transform wrapper. /// /// This uses a "mass output function" of __format::__output_buffer when possible. -template <__fmt_char_type _CharT, __fmt_char_type _OutCharT = _CharT, class _UnaryOperation> +template ::value_type, + __fmt_char_type _OutCharT = _CharT, + class _UnaryOperation> _LIBCPP_HIDE_FROM_ABI auto -__transform(const _CharT* __first, - const _CharT* __last, +__transform(_Iterator __first, + _Iterator __last, output_iterator auto __out_it, _UnaryOperation __operation) -> decltype(__out_it) { if constexpr (std::same_as>>) { @@ -260,8 +267,11 @@ __write(_Iterator __first, return __formatter::__write(__first, __last, std::move(__out_it), __specs, __last - __first); } -template -_LIBCPP_HIDE_FROM_ABI auto __write_transformed(const _CharT* __first, const _CharT* __last, +template ::value_type, + class _ParserCharT, + class _UnaryOperation> +_LIBCPP_HIDE_FROM_ABI auto __write_transformed(_Iterator __first, _Iterator __last, output_iterator auto __out_it, __format_spec::__parsed_specifications<_ParserCharT> __specs, _UnaryOperation __op) -> decltype(__out_it) { diff --git a/libcxx/test/libcxx/input.output/iostream.format/print.fun/transcoding.pass.cpp b/libcxx/test/libcxx/input.output/iostream.format/print.fun/transcoding.pass.cpp index 3936edb8bd083..168fa40a243c5 100644 --- a/libcxx/test/libcxx/input.output/iostream.format/print.fun/transcoding.pass.cpp +++ b/libcxx/test/libcxx/input.output/iostream.format/print.fun/transcoding.pass.cpp @@ -8,6 +8,7 @@ // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 // UNSUPPORTED: no-filesystem // UNSUPPORTED: GCC-ALWAYS_INLINE-FIXME +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=2000000 // @@ -32,9 +33,9 @@ constexpr void test(std::basic_string_view expected, std::string_view inp std::array buffer; std::ranges::fill(buffer, CharT('*')); - CharT* out = std::__unicode::__transcode(input.begin(), input.end(), buffer.data()); + auto out = std::__unicode::__transcode(input.begin(), input.end(), buffer.begin()); - assert(std::basic_string_view(buffer.data(), out) == expected); + assert(std::basic_string_view(buffer.begin(), out) == expected); out = std::find_if(out, buffer.end(), [](CharT c) { return c != CharT('*'); }); assert(out == buffer.end()); diff --git a/libcxx/test/std/algorithms/robust_against_proxy_iterators_lifetime_bugs.pass.cpp b/libcxx/test/std/algorithms/robust_against_proxy_iterators_lifetime_bugs.pass.cpp index 3a335c44ed1e1..0de22022526a6 100644 --- a/libcxx/test/std/algorithms/robust_against_proxy_iterators_lifetime_bugs.pass.cpp +++ b/libcxx/test/std/algorithms/robust_against_proxy_iterators_lifetime_bugs.pass.cpp @@ -547,24 +547,22 @@ class ConstexprIterator { #endif // TEST_STD_VER > 17 -template +template class Input { - using Array = std::array; - std::size_t size_ = 0; - Array values_ = {}; + T values_[StorageSize] = {}; public: - template - TEST_CONSTEXPR_CXX20 Input(std::array from) { - static_assert(N2 <= N, ""); + template + TEST_CONSTEXPR_CXX20 Input(std::array from) { + static_assert(N <= StorageSize, ""); std::copy(from.begin(), from.end(), begin()); - size_ = N2; + size_ = N; } - TEST_CONSTEXPR_CXX20 typename Array::iterator begin() { return values_.begin(); } - TEST_CONSTEXPR_CXX20 typename Array::iterator end() { return values_.begin() + size_; } + TEST_CONSTEXPR_CXX20 T* begin() { return values_; } + TEST_CONSTEXPR_CXX20 T* end() { return values_ + size_; } TEST_CONSTEXPR_CXX20 std::size_t size() const { return size_; } }; diff --git a/libcxx/test/std/containers/sequences/array/types.pass.cpp b/libcxx/test/std/containers/sequences/array/types.pass.cpp index f86e008d2e8de..c509810507962 100644 --- a/libcxx/test/std/containers/sequences/array/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/array/types.pass.cpp @@ -54,8 +54,6 @@ int main(int, char**) typedef std::array C; static_assert((std::is_same::value), ""); static_assert((std::is_same::value), ""); - LIBCPP_STATIC_ASSERT((std::is_same::value), ""); - LIBCPP_STATIC_ASSERT((std::is_same::value), ""); test_iterators(); static_assert((std::is_same::value), ""); static_assert((std::is_same::value), ""); @@ -76,8 +74,6 @@ int main(int, char**) typedef std::array C; static_assert((std::is_same::value), ""); static_assert((std::is_same::value), ""); - LIBCPP_STATIC_ASSERT((std::is_same::value), ""); - LIBCPP_STATIC_ASSERT((std::is_same::value), ""); test_iterators(); static_assert((std::is_same::value), ""); static_assert((std::is_same::value), ""); diff --git a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.pass.cpp index 73b13ccc34cf8..e893b5ae62874 100644 --- a/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.pass.cpp +++ b/libcxx/test/std/containers/views/views.span/span.cons/iterator_sentinel.pass.cpp @@ -114,7 +114,10 @@ class throw_operator_minus { friend difference_type operator-(throw_operator_minus, throw_operator_minus) { throw 42; }; friend bool operator==(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ == y.it_; } - friend auto operator<=>(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ <=> y.it_; } + friend bool operator<(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ < y.it_; } + friend bool operator>(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ > y.it_; } + friend bool operator<=(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ <= y.it_; } + friend bool operator>=(const throw_operator_minus& x, const throw_operator_minus& y) { return x.it_ >= y.it_; } }; template diff --git a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.floating_point.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.floating_point.pass.cpp index 87e2f9628757b..2df7834477291 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.floating_point.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.floating_point.pass.cpp @@ -61,7 +61,7 @@ void test(std::basic_string_view fmt, ArithmeticT arg, std::basic_string< if (expected.empty()) { std::array buffer; - expected.append(buffer.begin(), std::to_chars(buffer.begin(), buffer.end(), arg).ptr); + expected.append(buffer.data(), std::to_chars(buffer.data(), buffer.data() + buffer.size(), arg).ptr); } assert(result == expected); @@ -84,321 +84,401 @@ void test_termination_condition(StringT f, ArithmeticT arg, StringT expected = { template void test_hex_lower_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::hex, 20'000).ptr; - test_termination_condition(STR(".20000a}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::hex, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000a}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000a}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::hex, 20'000).ptr; - test_termination_condition(STR(".20000La}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::hex, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000La}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000La}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_hex_upper_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::hex, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000A}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::hex, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000A}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000A}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::hex, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000LA}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::hex, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000LA}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000LA}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_scientific_lower_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::scientific, 20'000).ptr; - test_termination_condition(STR(".20000e}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::scientific, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000e}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000e}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::scientific, 20'000).ptr; - test_termination_condition(STR(".20000Le}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::scientific, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000Le}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000Le}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_scientific_upper_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::scientific, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000E}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::scientific, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000E}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000E}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::scientific, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000LE}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::scientific, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000LE}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000LE}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_fixed_lower_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::fixed, 20'000).ptr; - test_termination_condition(STR(".20000f}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::fixed, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000f}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000f}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::fixed, 20'000).ptr; - test_termination_condition(STR(".20000Lf}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::fixed, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000Lf}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000Lf}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_fixed_upper_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::fixed, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000F}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::fixed, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000F}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000F}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::fixed, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000LF}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::fixed, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000LF}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000LF}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_general_lower_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::general, 20'000).ptr; - test_termination_condition(STR(".20000g}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::general, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000g}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000g}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::general, 20'000).ptr; - test_termination_condition(STR(".20000Lg}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::general, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + test_termination_condition(STR(".20000Lg}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000Lg}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } template void test_general_upper_case_precision(ArithmeticT value) { - std::array buffer; - char* end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::general, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000G}"), value, std::basic_string{buffer.begin(), end}); - - std::size_t size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::general, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000G}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000G}"), value, std::basic_string{buffer.begin(), buffer.end()}); + #ifndef TEST_HAS_NO_LOCALIZATION - end = std::to_chars(buffer.begin(), buffer.end(), value, std::chars_format::general, 20'000).ptr; - std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); - test_termination_condition(STR(".20000LG}"), value, std::basic_string{buffer.begin(), end}); - - size = buffer.end() - end; - std::fill_n(end, size, '#'); - test_termination_condition(STR("#<25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - (size / 2), buffer.end()); - test_termination_condition(STR("#^25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::rotate(buffer.begin(), buffer.end() - ((size + 1) / 2), buffer.end()); - test_termination_condition(STR("#>25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); - std::fill_n(buffer.begin(), size, '0'); - if (std::signbit(value)) { - buffer[0] = '-'; - buffer[size] = '0'; + { + std::array buffer; + char* end_ptr = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::general, 20'000).ptr; + std::size_t size = end_ptr - buffer.data(); + auto end = buffer.begin() + size; + std::transform(buffer.begin(), end, buffer.begin(), [](char c) { return std::toupper(c); }); + test_termination_condition(STR(".20000LG}"), value, std::basic_string{buffer.begin(), end}); + + std::size_t unused = buffer.end() - end; + std::fill_n(end, unused, '#'); + test_termination_condition(STR("#<25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - (unused / 2), buffer.end()); + test_termination_condition(STR("#^25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::rotate(buffer.begin(), buffer.end() - ((unused + 1) / 2), buffer.end()); + test_termination_condition(STR("#>25000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); + std::fill_n(buffer.begin(), unused, '0'); + if (std::signbit(value)) { + buffer[0] = '-'; + buffer[unused] = '0'; + } + test_termination_condition(STR("025000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); } - test_termination_condition(STR("025000.20000LG}"), value, std::basic_string{buffer.begin(), buffer.end()}); #endif } diff --git a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.pointer.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.pointer.pass.cpp index 347458d7d7562..ff5bfe0fb472a 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.pointer.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.pointer.pass.cpp @@ -58,8 +58,8 @@ void test(StringT expected, StringViewT fmt, PointerT arg, std::size_t offset) { std::array buffer; buffer[0] = CharT('0'); buffer[1] = CharT('x'); - expected.append(buffer.begin(), - std::to_chars(buffer.begin() + 2, buffer.end(), reinterpret_cast(arg), 16).ptr); + expected.append(buffer.data(), + std::to_chars(buffer.data() + 2, buffer.data() + buffer.size(), reinterpret_cast(arg), 16).ptr); } assert(result == expected); } From cd1038a46a337042da7685973332481eb9c37707 Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Mon, 18 Dec 2023 10:07:52 -0500 Subject: [PATCH 123/884] [OpenMP][libomptarget][Fix]Require presence of libomptarget-debug for newly added test (#75807) Require presence of libomptarget-debug fixes https://github.com/llvm/llvm-project/pull/75642 --- .../test/offloading/struct_mapping_with_pointers.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp b/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp index cecafe4c58416..ae8fed97c4642 100644 --- a/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp +++ b/openmp/libomptarget/test/offloading/struct_mapping_with_pointers.cpp @@ -2,6 +2,8 @@ // RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | %fcheck-generic // clang-format on +// REQUIRES: libomptarget-debug + #include #include From 5cda366221236a43fdd89bca59e153b4384eaba8 Mon Sep 17 00:00:00 2001 From: "Podchishchaeva, Mariya" Date: Mon, 18 Dec 2023 06:25:22 -0800 Subject: [PATCH 124/884] Revert "[clang] Fix false positive -Wmissing-field-initializer for anonymous unions (#70829)" This reverts commit a01307a6ee788fc6ac2e09e58f0f52e5666def86 and its follow-up fix 32d5221ec4810dd723ccebaabbda1df5d3b4cfcf. It caused unexpected warnings emitted for nested designators in C. --- clang/lib/Sema/SemaInit.cpp | 153 +++++++----------- clang/test/Sema/missing-field-initializers.c | 25 +-- .../SemaCXX/cxx2a-initializer-aggregates.cpp | 88 +--------- 3 files changed, 65 insertions(+), 201 deletions(-) diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 035eaae58965a..d6459fd9d7875 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -465,8 +465,7 @@ class InitListChecker { void FillInEmptyInitForField(unsigned Init, FieldDecl *Field, const InitializedEntity &ParentEntity, InitListExpr *ILE, bool &RequiresSecondPass, - bool FillWithNoInit = false, - bool WarnIfMissing = false); + bool FillWithNoInit = false); void FillInEmptyInitializations(const InitializedEntity &Entity, InitListExpr *ILE, bool &RequiresSecondPass, InitListExpr *OuterILE, unsigned OuterIndex, @@ -655,16 +654,11 @@ void InitListChecker::FillInEmptyInitForBase( } } -static bool hasAnyDesignatedInits(const InitListExpr *IL) { - return llvm::any_of(*IL, [=](const Stmt *Init) { - return isa_and_nonnull(Init); - }); -} - -void InitListChecker::FillInEmptyInitForField( - unsigned Init, FieldDecl *Field, const InitializedEntity &ParentEntity, - InitListExpr *ILE, bool &RequiresSecondPass, bool FillWithNoInit, - bool WarnIfMissing) { +void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field, + const InitializedEntity &ParentEntity, + InitListExpr *ILE, + bool &RequiresSecondPass, + bool FillWithNoInit) { SourceLocation Loc = ILE->getEndLoc(); unsigned NumInits = ILE->getNumInits(); InitializedEntity MemberEntity @@ -732,52 +726,15 @@ void InitListChecker::FillInEmptyInitForField( if (hadError || VerifyOnly) { // Do nothing - } else { - if (WarnIfMissing) { - auto CheckAnonMember = [&](const FieldDecl *FD, - auto &&CheckAnonMember) -> FieldDecl * { - FieldDecl *Uninitialized = nullptr; - RecordDecl *RD = FD->getType()->getAsRecordDecl(); - assert(RD && "Not anonymous member checked?"); - for (auto *F : RD->fields()) { - FieldDecl *UninitializedFieldInF = nullptr; - if (F->isAnonymousStructOrUnion()) - UninitializedFieldInF = CheckAnonMember(F, CheckAnonMember); - else if (!F->isUnnamedBitfield() && - !F->getType()->isIncompleteArrayType() && - !F->hasInClassInitializer()) - UninitializedFieldInF = F; - - if (RD->isUnion() && !UninitializedFieldInF) - return nullptr; - if (!Uninitialized) - Uninitialized = UninitializedFieldInF; - } - return Uninitialized; - }; - - FieldDecl *FieldToDiagnose = nullptr; - if (Field->isAnonymousStructOrUnion()) - FieldToDiagnose = CheckAnonMember(Field, CheckAnonMember); - else if (!Field->isUnnamedBitfield() && - !Field->getType()->isIncompleteArrayType()) - FieldToDiagnose = Field; - - if (FieldToDiagnose) - SemaRef.Diag(Loc, diag::warn_missing_field_initializers) - << FieldToDiagnose; - } - - if (Init < NumInits) { - ILE->setInit(Init, MemberInit.getAs()); - } else if (!isa(MemberInit.get())) { - // Empty initialization requires a constructor call, so - // extend the initializer list to include the constructor - // call and make a note that we'll need to take another pass - // through the initializer list. - ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs()); - RequiresSecondPass = true; - } + } else if (Init < NumInits) { + ILE->setInit(Init, MemberInit.getAs()); + } else if (!isa(MemberInit.get())) { + // Empty initialization requires a constructor call, so + // extend the initializer list to include the constructor + // call and make a note that we'll need to take another pass + // through the initializer list. + ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs()); + RequiresSecondPass = true; } } else if (InitListExpr *InnerILE = dyn_cast(ILE->getInit(Init))) { @@ -845,36 +802,9 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, } } } else { - InitListExpr *SForm = - ILE->isSyntacticForm() ? ILE : ILE->getSyntacticForm(); // The fields beyond ILE->getNumInits() are default initialized, so in // order to leave them uninitialized, the ILE is expanded and the extra // fields are then filled with NoInitExpr. - - // Some checks that are required for missing fields warning are bound to - // how many elements the initializer list originally was provided; perform - // them before the list is expanded. - bool WarnIfMissingField = - !SForm->isIdiomaticZeroInitializer(SemaRef.getLangOpts()) && - ILE->getNumInits(); - - // Disable check for missing fields when designators are used in C to - // match gcc behaviour. - // FIXME: Should we emulate possible gcc warning bug? - WarnIfMissingField &= - SemaRef.getLangOpts().CPlusPlus || !hasAnyDesignatedInits(SForm); - - if (OuterILE) { - // When nested designators are present, there might be two nested init - // lists created and only outer will contain designated initializer - // expression, so check outer list as well. - InitListExpr *OuterSForm = OuterILE->isSyntacticForm() - ? OuterILE - : OuterILE->getSyntacticForm(); - WarnIfMissingField &= SemaRef.getLangOpts().CPlusPlus || - !hasAnyDesignatedInits(OuterSForm); - } - unsigned NumElems = numStructUnionElements(ILE->getType()); if (!RDecl->isUnion() && RDecl->hasFlexibleArrayMember()) ++NumElems; @@ -902,7 +832,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, return; FillInEmptyInitForField(Init, Field, Entity, ILE, RequiresSecondPass, - FillWithNoInit, WarnIfMissingField); + FillWithNoInit); if (hadError) return; @@ -1017,6 +947,13 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity, } } +static bool hasAnyDesignatedInits(const InitListExpr *IL) { + for (const Stmt *Init : *IL) + if (isa_and_nonnull(Init)) + return true; + return false; +} + InitListChecker::InitListChecker( Sema &S, const InitializedEntity &Entity, InitListExpr *IL, QualType &T, bool VerifyOnly, bool TreatUnavailableAsInvalid, bool InOverloadResolution, @@ -2288,8 +2225,12 @@ void InitListChecker::CheckStructUnionTypes( size_t NumRecordDecls = llvm::count_if(RD->decls(), [&](const Decl *D) { return isa(D) || isa(D); }); + bool CheckForMissingFields = + !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts()); bool HasDesignatedInit = false; + llvm::SmallPtrSet InitializedFields; + while (Index < IList->getNumInits()) { Expr *Init = IList->getInit(Index); SourceLocation InitLoc = Init->getBeginLoc(); @@ -2313,17 +2254,24 @@ void InitListChecker::CheckStructUnionTypes( // Find the field named by the designated initializer. DesignatedInitExpr::Designator *D = DIE->getDesignator(0); - if (!VerifyOnly && D->isFieldDesignator() && !DesignatedInitFailed) { + if (!VerifyOnly && D->isFieldDesignator()) { FieldDecl *F = D->getFieldDecl(); - QualType ET = SemaRef.Context.getBaseElementType(F->getType()); - if (checkDestructorReference(ET, InitLoc, SemaRef)) { - hadError = true; - return; + InitializedFields.insert(F); + if (!DesignatedInitFailed) { + QualType ET = SemaRef.Context.getBaseElementType(F->getType()); + if (checkDestructorReference(ET, InitLoc, SemaRef)) { + hadError = true; + return; + } } } InitializedSomething = true; + // Disable check for missing fields when designators are used. + // This matches gcc behaviour. + if (!SemaRef.getLangOpts().CPlusPlus) + CheckForMissingFields = false; continue; } @@ -2402,6 +2350,7 @@ void InitListChecker::CheckStructUnionTypes( CheckSubElementType(MemberEntity, IList, Field->getType(), Index, StructuredList, StructuredIndex); InitializedSomething = true; + InitializedFields.insert(*Field); if (RD->isUnion() && StructuredList) { // Initialize the first field within the union. @@ -2411,6 +2360,28 @@ void InitListChecker::CheckStructUnionTypes( ++Field; } + // Emit warnings for missing struct field initializers. + if (!VerifyOnly && InitializedSomething && CheckForMissingFields && + !RD->isUnion()) { + // It is possible we have one or more unnamed bitfields remaining. + // Find first (if any) named field and emit warning. + for (RecordDecl::field_iterator it = HasDesignatedInit ? RD->field_begin() + : Field, + end = RD->field_end(); + it != end; ++it) { + if (HasDesignatedInit && InitializedFields.count(*it)) + continue; + + if (!it->isUnnamedBitfield() && !it->hasInClassInitializer() && + !it->getType()->isIncompleteArrayType()) { + SemaRef.Diag(IList->getSourceRange().getEnd(), + diag::warn_missing_field_initializers) + << *it; + break; + } + } + } + // Check that any remaining fields can be value-initialized if we're not // building a structured list. (If we are, we'll check this later.) if (!StructuredList && Field != FieldEnd && !RD->isUnion() && diff --git a/clang/test/Sema/missing-field-initializers.c b/clang/test/Sema/missing-field-initializers.c index 8dc8288ad92e6..1e65b2d62e1ab 100644 --- a/clang/test/Sema/missing-field-initializers.c +++ b/clang/test/Sema/missing-field-initializers.c @@ -18,7 +18,7 @@ struct Foo bar1[] = { 1, 2, 1, 2, 1 -}; // expected-warning@-1 {{missing field 'b' initializer}} +}; // expected-warning {{missing field 'b' initializer}} struct Foo bar2[] = { {}, {}, {} }; @@ -61,26 +61,3 @@ struct S { // f1, now we no longer issue that warning (note, this code is still unsafe // because of the buffer overrun). struct S s = {1, {1, 2}}; - -struct S1 { - long int l; - struct { int a, b; } d1; -}; - -struct S1 s01 = { 1, {1} }; // expected-warning {{missing field 'b' initializer}} -struct S1 s02 = { .d1.a = 1 }; // designator avoids MFI warning - -union U1 { - long int l; - struct { int a, b; } d1; -}; - -union U1 u01 = { 1 }; -union U1 u02 = { .d1.a = 1 }; // designator avoids MFI warning - -struct S2 { - long int l; - struct { int a, b; struct {int c; } d2; } d1; -}; - -struct S2 s22 = { .d1.d2.c = 1 }; // designator avoids MFI warning diff --git a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp index 0d977e07ed034..510ace58c35a6 100644 --- a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp +++ b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic,override,reorder -pedantic-errors // RUN: %clang_cc1 -std=c++17 %s -verify=expected,pedantic,override,reorder -Wno-c++20-designator -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic -Werror=c99-designator -Wno-reorder-init-list -Wno-initializer-overrides -Werror=nested-anon-types -Werror=gnu-anonymous-struct +// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic -Werror=c99-designator -Wno-reorder-init-list -Wno-initializer-overrides // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,reorder -Wno-c99-designator -Werror=reorder-init-list -Wno-initializer-overrides // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,override -Wno-c99-designator -Wno-reorder-init-list -Werror=initializer-overrides // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides @@ -39,7 +39,6 @@ A a1 = { }; int arr[3] = {[1] = 5}; // pedantic-error {{array designators are a C99 extension}} B b = {.a.x = 0}; // pedantic-error {{nested designators are a C99 extension}} - // wmissing-warning@-1 {{missing field 'y' initializer}} A a2 = { .x = 1, // pedantic-error {{mixture of designated and non-designated initializers in the same initializer list is a C99 extension}} 2 // pedantic-note {{first non-designated initializer is here}} @@ -61,6 +60,7 @@ B b2 = {.a = 1}; // pedantic-error {{brace elision for designated initializer is B b3 = {.a = 1, 2}; // pedantic-error {{mixture of designated and non-designated}} pedantic-note {{first non-designated}} pedantic-error {{brace elision}} B b4 = {.a = 1, 2, 3}; // pedantic-error {{mixture of designated and non-designated}} pedantic-note {{first non-designated}} pedantic-error {{brace elision}} expected-error {{excess elements}} B b5 = {.a = nullptr}; // expected-error {{cannot initialize}} + // wmissing-warning@-1 {{missing field 'y' initializer}} struct C { int :0, x, :0, y, :0; }; C c = { .x = 1, // override-note {{previous}} @@ -247,87 +247,3 @@ void foo() { // } } - -namespace GH70384 { - -struct A { - int m; - union { int a; float n = 0; }; -}; - -struct B { - int m; - int b; - union { int a ; }; -}; - -union CU { - int a = 1; - double b; -}; - -struct C { - int a; - union { int b; CU c;}; -}; - -struct CC { - int a; - CU c; -}; - -void foo() { - A a = A{.m = 0}; - A aa = {0}; - A aaa = {.a = 7}; // wmissing-warning {{missing field 'm' initializer}} - B b = {.m = 1, .b = 3 }; //wmissing-warning {{missing field 'a' initializer}} - B bb = {1}; // wmissing-warning {{missing field 'b' initializer}} - // wmissing-warning@-1 {{missing field 'a' initializer}} - C c = {.a = 1}; // wmissing-warning {{missing field 'b' initializer}} - CC cc = {.a = 1}; // wmissing-warning {{missing field 'c' initializer}} -} - -struct C1 { - int m; - union { float b; union {int n = 1; }; }; - // pedantic-error@-1 {{anonymous types declared in an anonymous union are an extension}} -}; - -struct C2 { - int m; - struct { float b; int n = 1; }; // pedantic-error {{anonymous structs are a GNU extension}} -}; - -struct C3 { - int m; - struct { float b = 1; union {int a;}; int n = 1; }; - // pedantic-error@-1 {{anonymous structs are a GNU extension}} - // pedantic-error@-2 {{anonymous types declared in an anonymous struct are an extension}} -}; - -C1 c = C1{.m = 1}; -C1 cc = C1{.b = 1}; // wmissing-warning {{missing field 'm' initializer}} -C2 c1 = C2{.m = 1}; // wmissing-warning {{missing field 'b' initializer}} -C2 c22 = C2{.m = 1, .b = 1}; -C3 c2 = C3{.b = 1}; // wmissing-warning {{missing field 'a' initializer}} - // wmissing-warning@-1 {{missing field 'm' initializer}} - -struct C4 { - union { - struct { int n; }; // pedantic-error {{anonymous structs are a GNU extension}} - // pedantic-error@-1 {{anonymous types declared in an anonymous union are an extension}} - int m = 0; }; - int z; -}; -C4 a = {.z = 1}; - -struct C5 { - int a; - struct { // pedantic-error {{anonymous structs are a GNU extension}} - int x; - struct { int y = 0; }; // pedantic-error {{anonymous types declared in an anonymous struct are an extension}} - // pedantic-error@-1 {{anonymous structs are a GNU extension}} - }; -}; -C5 c5 = C5{.a = 0}; //wmissing-warning {{missing field 'x' initializer}} -} From a5f34155339b4c01357462da95aac62291ed7ec8 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 15:59:33 +0100 Subject: [PATCH 125/884] [InstCombine] Replace non-demanded undef vector with poison If an operand (esp to shufflevector or insertelement) is not demanded, canonicalize it from undef to poison. --- .../builtins-systemz-zvector-constrained.c | 2 +- .../SystemZ/builtins-systemz-zvector.c | 26 +- .../builtins-systemz-zvector2-constrained.c | 4 +- .../SystemZ/builtins-systemz-zvector2.c | 4 +- clang/test/CodeGen/aarch64-neon-vcmla.c | 32 +-- clang/test/CodeGen/nofpclass.c | 8 +- clang/test/Headers/wasm.c | 224 +++++++++--------- .../InstCombineSimplifyDemanded.cpp | 10 +- .../InstCombine/AArch64/demandelts.ll | 22 +- ...mdgcn-demanded-vector-elts-inseltpoison.ll | 10 +- .../AMDGPU/amdgcn-demanded-vector-elts.ll | 10 +- .../Transforms/InstCombine/X86/x86-addsub.ll | 6 +- .../Transforms/InstCombine/X86/x86-avx2.ll | 2 +- .../Transforms/InstCombine/X86/x86-muldq.ll | 28 +-- .../InstCombine/X86/x86-pack-inseltpoison.ll | 6 +- .../Transforms/InstCombine/X86/x86-pack.ll | 22 +- .../Transforms/InstCombine/X86/x86-pshufb.ll | 4 +- .../InstCombine/X86/x86-vector-shifts.ll | 2 +- .../Transforms/InstCombine/X86/x86-vpermil.ll | 10 +- llvm/test/Transforms/InstCombine/assume.ll | 2 +- .../binop-select-cast-of-select-cond.ll | 2 +- .../bitcast-vec-canon-inseltpoison.ll | 2 +- .../InstCombine/bitcast-vec-canon.ll | 2 +- .../test/Transforms/InstCombine/bitreverse.ll | 4 +- llvm/test/Transforms/InstCombine/broadcast.ll | 22 +- llvm/test/Transforms/InstCombine/bswap.ll | 6 +- .../Transforms/InstCombine/extractelement.ll | 2 +- .../Transforms/InstCombine/icmp-bc-vec.ll | 4 +- llvm/test/Transforms/InstCombine/icmp-vec.ll | 10 +- .../insert-extract-shuffle-inseltpoison.ll | 2 +- .../InstCombine/insert-extract-shuffle.ll | 34 +-- .../Transforms/InstCombine/logical-select.ll | 8 +- .../masked_intrinsics-inseltpoison.ll | 2 +- .../InstCombine/masked_intrinsics.ll | 24 +- .../matrix-multiplication-negation.ll | 2 +- llvm/test/Transforms/InstCombine/nsw.ll | 4 +- .../InstCombine/obfuscated_splat.ll | 4 +- .../InstCombine/select-binop-cmp.ll | 2 +- .../select-extractelement-inseltpoison.ll | 8 +- .../InstCombine/select-extractelement.ll | 10 +- llvm/test/Transforms/InstCombine/shift-add.ll | 8 +- .../Transforms/InstCombine/shuffle-cast.ll | 16 +- .../InstCombine/shuffle-select-narrow.ll | 16 +- .../Transforms/InstCombine/shuffle_select.ll | 2 +- .../InstCombine/shufflevec-bitcast.ll | 24 +- .../InstCombine/shufflevector-div-rem.ll | 12 +- .../InstCombine/sub-of-negatible.ll | 6 +- .../trunc-extractelement-inseltpoison.ll | 2 +- .../InstCombine/trunc-extractelement.ll | 2 +- llvm/test/Transforms/InstCombine/trunc.ll | 2 +- llvm/test/Transforms/InstCombine/type_pun.ll | 2 +- .../InstCombine/vec_demanded_elts.ll | 68 +++--- .../vec_phi_extract-inseltpoison.ll | 2 +- .../Transforms/InstCombine/vec_phi_extract.ll | 2 +- .../Transforms/InstCombine/vec_shuffle.ll | 114 +++++---- .../AArch64/deterministic-type-shrinkage.ll | 16 +- .../multiply-fused-dominance.ll | 16 +- .../multiply-fused-loops.ll | 16 +- .../multiply-fused-multiple-blocks.ll | 24 +- .../LowerMatrixIntrinsics/multiply-fused.ll | 64 ++--- .../LowerMatrixIntrinsics/multiply-minimal.ll | 12 +- .../Transforms/PhaseOrdering/X86/pr61061.ll | 2 +- .../PhaseOrdering/X86/scalarization.ll | 2 +- .../Transforms/PhaseOrdering/X86/shuffle.ll | 26 +- .../SLPVectorizer/AMDGPU/add_sub_sat.ll | 20 +- .../SLPVectorizer/X86/alternate-calls.ll | 8 +- .../SLPVectorizer/X86/alternate-int.ll | 4 +- .../test/Transforms/SLPVectorizer/X86/hadd.ll | 10 +- .../Transforms/SLPVectorizer/X86/pr47629.ll | 10 +- .../Transforms/SLPVectorizer/X86/pr47642.ll | 6 +- 70 files changed, 548 insertions(+), 554 deletions(-) diff --git a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector-constrained.c b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector-constrained.c index 07d6469d18664..54a3365e03e32 100644 --- a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector-constrained.c +++ b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector-constrained.c @@ -82,7 +82,7 @@ void test_core(void) { // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vd = vec_splat(vd, 1); - // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> undef, <2 x i32> + // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> // CHECK-ASM: vrepg vd = vec_splats(d); diff --git a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector.c b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector.c index 5b6973787d1d7..48d775b888786 100644 --- a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector.c +++ b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector.c @@ -732,79 +732,79 @@ void test_core(void) { // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-ASM: vrepb vsc = vec_splat(vsc, 15); - // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <16 x i32> + // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> // CHECK-ASM: vrepb vuc = vec_splat(vuc, 0); // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-ASM: vrepb vuc = vec_splat(vuc, 15); - // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <16 x i32> + // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> // CHECK-ASM: vrepb vbc = vec_splat(vbc, 0); // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-ASM: vrepb vbc = vec_splat(vbc, 15); - // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <16 x i32> + // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> poison, <16 x i32> // CHECK-ASM: vrepb vss = vec_splat(vss, 0); // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-ASM: vreph vss = vec_splat(vss, 7); - // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> undef, <8 x i32> + // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> // CHECK-ASM: vreph vus = vec_splat(vus, 0); // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-ASM: vreph vus = vec_splat(vus, 7); - // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> undef, <8 x i32> + // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> // CHECK-ASM: vreph vbs = vec_splat(vbs, 0); // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-ASM: vreph vbs = vec_splat(vbs, 7); - // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> undef, <8 x i32> + // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> // CHECK-ASM: vreph vsi = vec_splat(vsi, 0); // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-ASM: vrepf vsi = vec_splat(vsi, 3); - // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> + // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> // CHECK-ASM: vrepf vui = vec_splat(vui, 0); // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-ASM: vrepf vui = vec_splat(vui, 3); - // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> + // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> // CHECK-ASM: vrepf vbi = vec_splat(vbi, 0); // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-ASM: vrepf vbi = vec_splat(vbi, 3); - // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> + // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> // CHECK-ASM: vrepf vsl = vec_splat(vsl, 0); // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vsl = vec_splat(vsl, 1); - // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> undef, <2 x i32> + // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> // CHECK-ASM: vrepg vul = vec_splat(vul, 0); // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vul = vec_splat(vul, 1); - // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> undef, <2 x i32> + // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> // CHECK-ASM: vrepg vbl = vec_splat(vbl, 0); // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vbl = vec_splat(vbl, 1); - // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> undef, <2 x i32> + // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> poison, <2 x i32> // CHECK-ASM: vrepg vd = vec_splat(vd, 0); // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vd = vec_splat(vd, 1); - // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> undef, <2 x i32> + // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> // CHECK-ASM: vrepg vsc = vec_splat_s8(-128); diff --git a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2-constrained.c b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2-constrained.c index 112bbac3394e9..750f5011a2679 100644 --- a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2-constrained.c +++ b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2-constrained.c @@ -133,13 +133,13 @@ void test_core(void) { // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> zeroinitializer // CHECK-ASM: vrepf vf = vec_splat(vf, 1); - // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> undef, <4 x i32> + // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> // CHECK-ASM: vrepf vd = vec_splat(vd, 0); // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vd = vec_splat(vd, 1); - // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> undef, <2 x i32> + // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> // CHECK-ASM: vrepg vf = vec_splats(f); diff --git a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2.c b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2.c index 874840b3e6bce..6c26b51c542e5 100644 --- a/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2.c +++ b/clang/test/CodeGen/SystemZ/builtins-systemz-zvector2.c @@ -238,13 +238,13 @@ void test_core(void) { // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> zeroinitializer // CHECK-ASM: vrepf vf = vec_splat(vf, 1); - // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> undef, <4 x i32> + // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> // CHECK-ASM: vrepf vd = vec_splat(vd, 0); // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> zeroinitializer // CHECK-ASM: vrepg vd = vec_splat(vd, 1); - // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> undef, <2 x i32> + // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> // CHECK-ASM: vrepg vf = vec_splats(f); diff --git a/clang/test/CodeGen/aarch64-neon-vcmla.c b/clang/test/CodeGen/aarch64-neon-vcmla.c index 9cd046d63d1b5..02171527cc6a3 100644 --- a/clang/test/CodeGen/aarch64-neon-vcmla.c +++ b/clang/test/CodeGen/aarch64-neon-vcmla.c @@ -158,7 +158,7 @@ float16x4_t test_vcmla_lane_f16(float16x4_t acc, float16x4_t lhs, float16x4_t rh // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <2 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <2 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <2 x i32> [[DUP]] to <4 x half> // CHECK: [[RES:%.*]] = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> [[DUP_FLT]]) // CHECK: ret <4 x half> [[RES]] @@ -176,7 +176,7 @@ float16x8_t test_vcmlaq_lane_f16(float16x8_t acc, float16x8_t lhs, float16x4_t r // CHECK-LABEL: @test_vcmlaq_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <4 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <4 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <4 x i32> [[DUP]] to <8 x half> // CHECK: [[RES:%.*]] = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> [[DUP_FLT]]) // CHECK: ret <8 x half> [[RES]] @@ -194,7 +194,7 @@ float32x2_t test_vcmla_lane_f32(float32x2_t acc, float32x2_t lhs, float32x2_t rh // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_laneq_f32( // CHECK: [[CPLX:%.*]] = bitcast <4 x float> %rhs to <2 x i64> -// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> undef, <1 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> poison, <1 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <1 x i64> [[DUP]] to <2 x float> // CHECK: [[RES:%.*]] = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot0.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> [[DUP_FLT]]) // CHECK: ret <2 x float> [[RES]] @@ -204,7 +204,7 @@ float32x2_t test_vcmla_laneq_f32(float32x2_t acc, float32x2_t lhs, float32x4_t r // CHECK-LABEL: @test_vcmlaq_lane_f32( // CHECK: [[CPLX:%.*]] = bitcast <2 x float> %rhs to i64 -// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> undef, i64 [[CPLX]], i64 0 +// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> poison, i64 [[CPLX]], i64 0 // CHECK: [[CPLX2:%.*]] = bitcast <2 x i64> [[CPLX_VEC]] to <4 x float> // CHECK: [[DUP:%.*]] = shufflevector <4 x float> [[CPLX2]], <4 x float> poison, <4 x i32> // CHECK: [[RES:%.*]] = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> [[DUP]]) @@ -232,7 +232,7 @@ float16x4_t test_vcmla_rot90_lane_f16(float16x4_t acc, float16x4_t lhs, float16x // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot90_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <2 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <2 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <2 x i32> [[DUP]] to <4 x half> // CHECK: [[RES:%.*]] = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> [[DUP_FLT]]) // CHECK: ret <4 x half> [[RES]] @@ -250,7 +250,7 @@ float16x8_t test_vcmlaq_rot90_lane_f16(float16x8_t acc, float16x8_t lhs, float16 // CHECK-LABEL: @test_vcmlaq_rot90_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <4 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <4 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <4 x i32> [[DUP]] to <8 x half> // CHECK: [[RES:%.*]] = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> [[DUP_FLT]]) // CHECK: ret <8 x half> [[RES]] @@ -268,7 +268,7 @@ float32x2_t test_vcmla_rot90_lane_f32(float32x2_t acc, float32x2_t lhs, float32x // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot90_laneq_f32( // CHECK: [[CPLX:%.*]] = bitcast <4 x float> %rhs to <2 x i64> -// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> undef, <1 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> poison, <1 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <1 x i64> [[DUP]] to <2 x float> // CHECK: [[RES:%.*]] = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot90.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> [[DUP_FLT]]) // CHECK: ret <2 x float> [[RES]] @@ -278,7 +278,7 @@ float32x2_t test_vcmla_rot90_laneq_f32(float32x2_t acc, float32x2_t lhs, float32 // CHECK-LABEL: @test_vcmlaq_rot90_lane_f32( // CHECK: [[CPLX:%.*]] = bitcast <2 x float> %rhs to i64 -// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> undef, i64 [[CPLX]], i64 0 +// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> poison, i64 [[CPLX]], i64 0 // CHECK: [[CPLX2:%.*]] = bitcast <2 x i64> [[CPLX_VEC]] to <4 x float> // CHECK: [[DUP:%.*]] = shufflevector <4 x float> [[CPLX2]], <4 x float> poison, <4 x i32> // CHECK: [[RES:%.*]] = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> [[DUP]]) @@ -306,7 +306,7 @@ float16x4_t test_vcmla_rot180_lane_f16(float16x4_t acc, float16x4_t lhs, float16 // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot180_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <2 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <2 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <2 x i32> [[DUP]] to <4 x half> // CHECK: [[RES:%.*]] = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> [[DUP_FLT]]) // CHECK: ret <4 x half> [[RES]] @@ -324,7 +324,7 @@ float16x8_t test_vcmlaq_rot180_lane_f16(float16x8_t acc, float16x8_t lhs, float1 // CHECK-LABEL: @test_vcmlaq_rot180_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <4 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <4 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <4 x i32> [[DUP]] to <8 x half> // CHECK: [[RES:%.*]] = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> [[DUP_FLT]]) // CHECK: ret <8 x half> [[RES]] @@ -342,7 +342,7 @@ float32x2_t test_vcmla_rot180_lane_f32(float32x2_t acc, float32x2_t lhs, float32 // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot180_laneq_f32( // CHECK: [[CPLX:%.*]] = bitcast <4 x float> %rhs to <2 x i64> -// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> undef, <1 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> poison, <1 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <1 x i64> [[DUP]] to <2 x float> // CHECK: [[RES:%.*]] = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot180.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> [[DUP_FLT]]) // CHECK: ret <2 x float> [[RES]] @@ -352,7 +352,7 @@ float32x2_t test_vcmla_rot180_laneq_f32(float32x2_t acc, float32x2_t lhs, float3 // CHECK-LABEL: @test_vcmlaq_rot180_lane_f32( // CHECK: [[CPLX:%.*]] = bitcast <2 x float> %rhs to i64 -// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> undef, i64 [[CPLX]], i64 0 +// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> poison, i64 [[CPLX]], i64 0 // CHECK: [[CPLX2:%.*]] = bitcast <2 x i64> [[CPLX_VEC]] to <4 x float> // CHECK: [[DUP:%.*]] = shufflevector <4 x float> [[CPLX2]], <4 x float> poison, <4 x i32> // CHECK: [[RES:%.*]] = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot180.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> [[DUP]]) @@ -380,7 +380,7 @@ float16x4_t test_vcmla_rot270_lane_f16(float16x4_t acc, float16x4_t lhs, float16 // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot270_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <2 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <2 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <2 x i32> [[DUP]] to <4 x half> // CHECK: [[RES:%.*]] = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> [[DUP_FLT]]) // CHECK: ret <4 x half> [[RES]] @@ -398,7 +398,7 @@ float16x8_t test_vcmlaq_rot270_lane_f16(float16x8_t acc, float16x8_t lhs, float1 // CHECK-LABEL: @test_vcmlaq_rot270_laneq_f16( // CHECK: [[CPLX:%.*]] = bitcast <8 x half> %rhs to <4 x i32> -// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> undef, <4 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <4 x i32> [[CPLX]], <4 x i32> poison, <4 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <4 x i32> [[DUP]] to <8 x half> // CHECK: [[RES:%.*]] = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> [[DUP_FLT]]) // CHECK: ret <8 x half> [[RES]] @@ -416,7 +416,7 @@ float32x2_t test_vcmla_rot270_lane_f32(float32x2_t acc, float32x2_t lhs, float32 // ACLE says this exists, but it won't map to a single instruction if lane > 1. // CHECK-LABEL: @test_vcmla_rot270_laneq_f32( // CHECK: [[CPLX:%.*]] = bitcast <4 x float> %rhs to <2 x i64> -// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> undef, <1 x i32> +// CHECK: [[DUP:%.*]] = shufflevector <2 x i64> [[CPLX]], <2 x i64> poison, <1 x i32> // CHECK: [[DUP_FLT:%.*]] = bitcast <1 x i64> [[DUP]] to <2 x float> // CHECK: [[RES:%.*]] = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot270.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> [[DUP_FLT]]) // CHECK: ret <2 x float> [[RES]] @@ -426,7 +426,7 @@ float32x2_t test_vcmla_rot270_laneq_f32(float32x2_t acc, float32x2_t lhs, float3 // CHECK-LABEL: @test_vcmlaq_rot270_lane_f32( // CHECK: [[CPLX:%.*]] = bitcast <2 x float> %rhs to i64 -// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> undef, i64 [[CPLX]], i64 0 +// CHECK: [[CPLX_VEC:%.*]] = insertelement <2 x i64> poison, i64 [[CPLX]], i64 0 // CHECK: [[CPLX2:%.*]] = bitcast <2 x i64> [[DUP]] to <4 x float> // CHECK: [[DUP:%.*]] = shufflevector <4 x float> [[CPLX2]], <4 x float> poison, <4 x i32> // CHECK: [[RES:%.*]] = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> [[DUP]]) diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c index 7fea8192cda75..daaf24e77b09d 100644 --- a/clang/test/CodeGen/nofpclass.c +++ b/clang/test/CodeGen/nofpclass.c @@ -1,4 +1,4 @@ -// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-attributes --version 2 +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-attributes --version 2 // REQUIRES: x86-registered-target // RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-feature +avx -fenable-matrix -ffinite-math-only -emit-llvm -o - %s | FileCheck -check-prefixes=CFINITEONLY %s // RUN: %clang_cc1 -x cl -triple x86_64-unknown-unknown -target-feature +avx -fenable-matrix -cl-finite-math-only -emit-llvm -o - %s | FileCheck -check-prefixes=CLFINITEONLY %s @@ -733,7 +733,7 @@ _Complex double defined_complex_func_f64_ret(_Complex double c) { // CLFINITEONLY-NEXT: [[MUL_R:%.*]] = fsub nnan ninf float [[MUL_AC]], [[MUL_BD]] // CLFINITEONLY-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half // CLFINITEONLY-NEXT: [[UNPROMOTION9:%.*]] = fptrunc float [[MUL_I]] to half -// CLFINITEONLY-NEXT: [[RETVAL_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> undef, half [[UNPROMOTION]], i64 0 +// CLFINITEONLY-NEXT: [[RETVAL_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[UNPROMOTION]], i64 0 // CLFINITEONLY-NEXT: [[RETVAL_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[RETVAL_SROA_0_0_VEC_INSERT]], half [[UNPROMOTION9]], i64 1 // CLFINITEONLY-NEXT: ret <2 x half> [[RETVAL_SROA_0_2_VEC_INSERT]] // @@ -930,7 +930,7 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) { // CLFINITEONLY-NEXT: [[INDIRECT_ARG_TEMP_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[INDIRECT_ARG_TEMP]], i64 0, i32 1 // CLFINITEONLY-NEXT: store double [[CF64_COERCE0]], ptr [[INDIRECT_ARG_TEMP]], align 8 // CLFINITEONLY-NEXT: store double [[CF64_COERCE1]], ptr [[INDIRECT_ARG_TEMP_IMAGP]], align 8 -// CLFINITEONLY-NEXT: [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> undef, half [[CF16_REAL]], i64 0 +// CLFINITEONLY-NEXT: [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0 // CLFINITEONLY-NEXT: [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1 // CLFINITEONLY-NEXT: [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[INDIRECT_ARG_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR10]] // CLFINITEONLY-NEXT: ret float [[CALL]] @@ -1181,7 +1181,7 @@ float call_variadic(float f32, double f64, _Float16 f16, // CLFINITEONLY-NEXT: [[INDIRECT_ARG_TEMP_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[INDIRECT_ARG_TEMP]], i64 0, i32 1 // CLFINITEONLY-NEXT: store double [[CF64_COERCE0]], ptr [[INDIRECT_ARG_TEMP]], align 8 // CLFINITEONLY-NEXT: store double [[CF64_COERCE1]], ptr [[INDIRECT_ARG_TEMP_IMAGP]], align 8 -// CLFINITEONLY-NEXT: [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> undef, half [[CF16_REAL]], i64 0 +// CLFINITEONLY-NEXT: [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0 // CLFINITEONLY-NEXT: [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1 // CLFINITEONLY-NEXT: [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) [[FPTR]](float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[INDIRECT_ARG_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR10]] // CLFINITEONLY-NEXT: ret float [[CALL]] diff --git a/clang/test/Headers/wasm.c b/clang/test/Headers/wasm.c index 9643cafc1ce6c..f77b95a52ee74 100644 --- a/clang/test/Headers/wasm.c +++ b/clang/test/Headers/wasm.c @@ -19,7 +19,7 @@ v128_t test_v128_load(const void *mem) { // CHECK-LABEL: @test_v128_load8_splat( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[MEM:%.*]], align 1, !tbaa [[TBAA2]] -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> undef, i8 [[TMP0]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0 // CHECK-NEXT: [[VECINIT16_I:%.*]] = shufflevector <16 x i8> [[VECINIT_I]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[VECINIT16_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -31,7 +31,7 @@ v128_t test_v128_load8_splat(const void *mem) { // CHECK-LABEL: @test_v128_load16_splat( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[MEM:%.*]], align 1, !tbaa [[TBAA2]] -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[TMP0]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> poison, i16 [[TMP0]], i64 0 // CHECK-NEXT: [[VECINIT8_I:%.*]] = shufflevector <8 x i16> [[VECINIT_I]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[VECINIT8_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -43,7 +43,7 @@ v128_t test_v128_load16_splat(const void *mem) { // CHECK-LABEL: @test_v128_load32_splat( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[MEM:%.*]], align 1, !tbaa [[TBAA2]] -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0 // CHECK-NEXT: [[VECINIT4_I:%.*]] = shufflevector <4 x i32> [[VECINIT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: ret <4 x i32> [[VECINIT4_I]] // @@ -54,7 +54,7 @@ v128_t test_v128_load32_splat(const void *mem) { // CHECK-LABEL: @test_v128_load64_splat( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[MEM:%.*]], align 1, !tbaa [[TBAA2]] -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> undef, i64 [[TMP0]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> poison, i64 [[TMP0]], i64 0 // CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <2 x i64> [[VECINIT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[VECINIT2_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -298,7 +298,7 @@ v128_t test_u8x16_make(uint8_t c0, uint8_t c1, uint8_t c2, uint8_t c3, uint8_t c // CHECK-LABEL: @test_i16x8_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> poison, i16 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <8 x i16> [[VECINIT_I]], i16 [[C1:%.*]], i64 1 // CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <8 x i16> [[VECINIT1_I]], i16 [[C2:%.*]], i64 2 // CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <8 x i16> [[VECINIT2_I]], i16 [[C3:%.*]], i64 3 @@ -315,7 +315,7 @@ v128_t test_i16x8_make(int16_t c0, int16_t c1, int16_t c2, int16_t c3, int16_t c // CHECK-LABEL: @test_u16x8_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> poison, i16 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <8 x i16> [[VECINIT_I]], i16 [[C1:%.*]], i64 1 // CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <8 x i16> [[VECINIT1_I]], i16 [[C2:%.*]], i64 2 // CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <8 x i16> [[VECINIT2_I]], i16 [[C3:%.*]], i64 3 @@ -332,7 +332,7 @@ v128_t test_u16x8_make(uint16_t c0, uint16_t c1, uint16_t c2, uint16_t c3, uint1 // CHECK-LABEL: @test_i32x4_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> poison, i32 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <4 x i32> [[VECINIT_I]], i32 [[C1:%.*]], i64 1 // CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <4 x i32> [[VECINIT1_I]], i32 [[C2:%.*]], i64 2 // CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <4 x i32> [[VECINIT2_I]], i32 [[C3:%.*]], i64 3 @@ -344,7 +344,7 @@ v128_t test_i32x4_make(int32_t c0, int32_t c1, int32_t c2, int32_t c3) { // CHECK-LABEL: @test_u32x4_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> poison, i32 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <4 x i32> [[VECINIT_I]], i32 [[C1:%.*]], i64 1 // CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <4 x i32> [[VECINIT1_I]], i32 [[C2:%.*]], i64 2 // CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <4 x i32> [[VECINIT2_I]], i32 [[C3:%.*]], i64 3 @@ -356,7 +356,7 @@ v128_t test_u32x4_make(uint32_t c0, uint32_t c1, uint32_t c2, uint32_t c3) { // CHECK-LABEL: @test_i64x2_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> undef, i64 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> poison, i64 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <2 x i64> [[VECINIT_I]], i64 [[C1:%.*]], i64 1 // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -367,7 +367,7 @@ v128_t test_i64x2_make(int64_t c0, int64_t c1) { // CHECK-LABEL: @test_u64x2_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> undef, i64 [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> poison, i64 [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <2 x i64> [[VECINIT_I]], i64 [[C1:%.*]], i64 1 // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -378,7 +378,7 @@ v128_t test_u64x2_make(uint64_t c0, uint64_t c1) { // CHECK-LABEL: @test_f32x4_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x float> undef, float [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[C1:%.*]], i64 1 // CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT1_I]], float [[C2:%.*]], i64 2 // CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[C3:%.*]], i64 3 @@ -391,7 +391,7 @@ v128_t test_f32x4_make(float c0, float c1, float c2, float c3) { // CHECK-LABEL: @test_f64x2_make( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x double> undef, double [[C0:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[C0:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <2 x double> [[VECINIT_I]], double [[C1:%.*]], i64 1 // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -562,7 +562,7 @@ v128_t test_f64x2_const_splat(void) { // CHECK-LABEL: @test_i8x16_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> undef, i8 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> poison, i8 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT15_I:%.*]] = shufflevector <16 x i8> [[VECINIT_I]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[VECINIT15_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -573,7 +573,7 @@ v128_t test_i8x16_splat(int8_t a) { // CHECK-LABEL: @test_u8x16_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> undef, i8 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <16 x i8> poison, i8 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT15_I:%.*]] = shufflevector <16 x i8> [[VECINIT_I]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[VECINIT15_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -626,7 +626,7 @@ v128_t test_u8x16_replace_lane(v128_t a, uint8_t b) { // CHECK-LABEL: @test_i16x8_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> poison, i16 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT7_I:%.*]] = shufflevector <8 x i16> [[VECINIT_I]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x i16> [[VECINIT7_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -637,7 +637,7 @@ v128_t test_i16x8_splat(int16_t a) { // CHECK-LABEL: @test_u16x8_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> undef, i16 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x i16> poison, i16 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT7_I:%.*]] = shufflevector <8 x i16> [[VECINIT_I]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x i16> [[VECINIT7_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -690,7 +690,7 @@ v128_t test_u16x8_replace_lane(v128_t a, uint16_t b) { // CHECK-LABEL: @test_i32x4_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT3_I:%.*]] = shufflevector <4 x i32> [[VECINIT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: ret <4 x i32> [[VECINIT3_I]] // @@ -700,7 +700,7 @@ v128_t test_i32x4_splat(int32_t a) { // CHECK-LABEL: @test_u32x4_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT3_I:%.*]] = shufflevector <4 x i32> [[VECINIT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: ret <4 x i32> [[VECINIT3_I]] // @@ -746,7 +746,7 @@ v128_t test_u32x4_replace_lane(v128_t a, uint32_t b) { // CHECK-LABEL: @test_i64x2_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> undef, i64 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = shufflevector <2 x i64> [[VECINIT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -757,7 +757,7 @@ v128_t test_i64x2_splat(int64_t a) { // CHECK-LABEL: @test_u64x2_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> undef, i64 [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = shufflevector <2 x i64> [[VECINIT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -810,7 +810,7 @@ v128_t test_u64x2_replace_lane(v128_t a, uint64_t b) { // CHECK-LABEL: @test_f32x4_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i64 0 // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[VECINIT_I]] to <4 x i32> // CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -842,7 +842,7 @@ v128_t test_f32x4_replace_lane(v128_t a, float b) { // CHECK-LABEL: @test_f64x2_splat( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i64 0 +// CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <2 x double> poison, double [[A:%.*]], i64 0 // CHECK-NEXT: [[VECINIT1_I:%.*]] = shufflevector <2 x double> [[VECINIT_I]], <2 x double> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[VECINIT1_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -1572,7 +1572,7 @@ uint32_t test_i8x16_bitmask(v128_t a) { // CHECK-LABEL: @test_i8x16_popcnt( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP0]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP0]]), !range [[RNG5:![0-9]+]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP2]] // @@ -1585,7 +1585,7 @@ v128_t test_i8x16_popcnt(v128_t a) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8 // CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP1]], 7 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> undef, i8 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> poison, i8 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[SHL_I:%.*]] = shl <16 x i8> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[SHL_I]] to <4 x i32> @@ -1600,7 +1600,7 @@ v128_t test_i8x16_shl(v128_t a, uint32_t b) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8 // CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP1]], 7 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> undef, i8 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> poison, i8 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = ashr <16 x i8> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[SHR_I]] to <4 x i32> @@ -1615,7 +1615,7 @@ v128_t test_i8x16_shr(v128_t a, uint32_t b) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8 // CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP1]], 7 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> undef, i8 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> poison, i8 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> poison, <16 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = lshr <16 x i8> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[SHR_I]] to <4 x i32> @@ -1805,7 +1805,7 @@ uint32_t test_i16x8_bitmask(v128_t a) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i16 // CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> undef, i16 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> poison, i16 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[SHL_I:%.*]] = shl <8 x i16> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[SHL_I]] to <4 x i32> @@ -1820,7 +1820,7 @@ v128_t test_i16x8_shl(v128_t a, uint32_t b) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i16 // CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> undef, i16 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> poison, i16 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = ashr <8 x i16> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[SHR_I]] to <4 x i32> @@ -1835,7 +1835,7 @@ v128_t test_i16x8_shr(v128_t a, uint32_t b) { // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i16 // CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15 -// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> undef, i16 [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> poison, i16 [[TMP2]], i64 0 // CHECK-NEXT: [[SH_PROM_I:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <8 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = lshr <8 x i16> [[TMP0]], [[SH_PROM_I]] // CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[SHR_I]] to <4 x i32> @@ -2028,8 +2028,8 @@ uint32_t test_i32x4_bitmask(v128_t a) { // CHECK-LABEL: @test_i32x4_shl( // CHECK-NEXT: entry: -// CHECK-NEXT: [[REM_I:%.*]] = and i32 [[B:%.*]], 31 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = and i32 [[B:%.*]], 31 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: [[SHL_I:%.*]] = shl <4 x i32> [[A:%.*]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: ret <4 x i32> [[SHL_I]] @@ -2040,8 +2040,8 @@ v128_t test_i32x4_shl(v128_t a, uint32_t b) { // CHECK-LABEL: @test_i32x4_shr( // CHECK-NEXT: entry: -// CHECK-NEXT: [[REM_I:%.*]] = and i32 [[B:%.*]], 31 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = and i32 [[B:%.*]], 31 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = ashr <4 x i32> [[A:%.*]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: ret <4 x i32> [[SHR_I]] @@ -2052,8 +2052,8 @@ v128_t test_i32x4_shr(v128_t a, uint32_t b) { // CHECK-LABEL: @test_u32x4_shr( // CHECK-NEXT: entry: -// CHECK-NEXT: [[REM_I:%.*]] = and i32 [[B:%.*]], 31 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = and i32 [[B:%.*]], 31 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <4 x i32> poison, i32 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT_I]], <4 x i32> poison, <4 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = lshr <4 x i32> [[A:%.*]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: ret <4 x i32> [[SHR_I]] @@ -2183,8 +2183,8 @@ uint32_t test_i64x2_bitmask(v128_t a) { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <2 x i64> // CHECK-NEXT: [[TMP1:%.*]] = and i32 [[B:%.*]], 63 -// CHECK-NEXT: [[REM_I:%.*]] = zext nneg i32 [[TMP1]] to i64 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = zext nneg i32 [[TMP1]] to i64 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <2 x i64> [[SPLAT_SPLATINSERT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[SHL_I:%.*]] = shl <2 x i64> [[TMP0]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[SHL_I]] to <4 x i32> @@ -2198,8 +2198,8 @@ v128_t test_i64x2_shl(v128_t a, uint32_t b) { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <2 x i64> // CHECK-NEXT: [[TMP1:%.*]] = and i32 [[B:%.*]], 63 -// CHECK-NEXT: [[REM_I:%.*]] = zext nneg i32 [[TMP1]] to i64 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = zext nneg i32 [[TMP1]] to i64 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <2 x i64> [[SPLAT_SPLATINSERT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = ashr <2 x i64> [[TMP0]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[SHR_I]] to <4 x i32> @@ -2213,8 +2213,8 @@ v128_t test_i64x2_shr(v128_t a, uint32_t b) { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <2 x i64> // CHECK-NEXT: [[TMP1:%.*]] = and i32 [[B:%.*]], 63 -// CHECK-NEXT: [[REM_I:%.*]] = zext nneg i32 [[TMP1]] to i64 -// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[REM_I]], i64 0 +// CHECK-NEXT: [[AND_I:%.*]] = zext nneg i32 [[TMP1]] to i64 +// CHECK-NEXT: [[SPLAT_SPLATINSERT_I:%.*]] = insertelement <2 x i64> poison, i64 [[AND_I]], i64 0 // CHECK-NEXT: [[SPLAT_SPLAT_I:%.*]] = shufflevector <2 x i64> [[SPLAT_SPLATINSERT_I]], <2 x i64> poison, <2 x i32> zeroinitializer // CHECK-NEXT: [[SHR_I:%.*]] = lshr <2 x i64> [[TMP0]], [[SPLAT_SPLAT_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[SHR_I]] to <4 x i32> @@ -2648,7 +2648,7 @@ v128_t test_f32x4_convert_u32x4(v128_t a) { // CHECK-LABEL: @test_f64x2_convert_low_i32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sitofp <2 x i32> [[VECINIT2_I]] to <2 x double> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2659,7 +2659,7 @@ v128_t test_f64x2_convert_low_i32x4(v128_t a) { // CHECK-LABEL: @test_f64x2_convert_low_u32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = uitofp <2 x i32> [[VECINIT2_I]] to <2 x double> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2705,7 +2705,7 @@ v128_t test_f32x4_demote_f64x2_zero(v128_t a) { // CHECK-LABEL: @test_f64x2_promote_low_f32x4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <4 x float> -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = fpext <2 x float> [[VECINIT2_I]] to <2 x double> // CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x double> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -2821,7 +2821,7 @@ v128_t test_u16x8_narrow_i32x4(v128_t a, v128_t b) { // CHECK-LABEL: @test_i16x8_extend_low_i8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <8 x i8> [[VECINIT14_I]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -2833,7 +2833,7 @@ v128_t test_i16x8_extend_low_i8x16(v128_t a) { // CHECK-LABEL: @test_i16x8_extend_high_i8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <8 x i8> [[VECINIT14_I]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -2845,7 +2845,7 @@ v128_t test_i16x8_extend_high_i8x16(v128_t a) { // CHECK-LABEL: @test_u16x8_extend_low_u8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <8 x i8> [[VECINIT14_I]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -2857,7 +2857,7 @@ v128_t test_u16x8_extend_low_u8x16(v128_t a) { // CHECK-LABEL: @test_u16x8_extend_high_u8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <8 x i8> [[VECINIT14_I]] to <8 x i16> // CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] @@ -2869,7 +2869,7 @@ v128_t test_u16x8_extend_high_u8x16(v128_t a) { // CHECK-LABEL: @test_i32x4_extend_low_i16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <4 x i16> [[VECINIT6_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[CONV_I]] // @@ -2880,7 +2880,7 @@ v128_t test_i32x4_extend_low_i16x8(v128_t a) { // CHECK-LABEL: @test_i32x4_extend_high_i16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <4 x i16> [[VECINIT6_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[CONV_I]] // @@ -2891,7 +2891,7 @@ v128_t test_i32x4_extend_high_i16x8(v128_t a) { // CHECK-LABEL: @test_u32x4_extend_low_u16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <4 x i16> [[VECINIT6_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[CONV_I]] // @@ -2902,7 +2902,7 @@ v128_t test_u32x4_extend_low_u16x8(v128_t a) { // CHECK-LABEL: @test_u32x4_extend_high_u16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <4 x i16> [[VECINIT6_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[CONV_I]] // @@ -2912,7 +2912,7 @@ v128_t test_u32x4_extend_high_u16x8(v128_t a) { // CHECK-LABEL: @test_i64x2_extend_low_i32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <2 x i32> [[VECINIT2_I]] to <2 x i64> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2923,7 +2923,7 @@ v128_t test_i64x2_extend_low_i32x4(v128_t a) { // CHECK-LABEL: @test_i64x2_extend_high_i32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = sext <2 x i32> [[VECINIT2_I]] to <2 x i64> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2934,7 +2934,7 @@ v128_t test_i64x2_extend_high_i32x4(v128_t a) { // CHECK-LABEL: @test_u64x2_extend_low_u32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <2 x i32> [[VECINIT2_I]] to <2 x i64> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2945,7 +2945,7 @@ v128_t test_u64x2_extend_low_u32x4(v128_t a) { // CHECK-LABEL: @test_u64x2_extend_high_u32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I:%.*]] = zext <2 x i32> [[VECINIT2_I]] to <2 x i64> // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[CONV_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] @@ -2999,12 +2999,12 @@ v128_t test_u32x4_extadd_pairwise_u16x8(v128_t a) { // CHECK-LABEL: @test_i16x8_extmul_low_i8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <8 x i8> [[VECINIT14_I2_I]] to <8 x i16> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <8 x i16> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> +// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <8 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <8 x i16> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP2]] // @@ -3015,12 +3015,12 @@ v128_t test_i16x8_extmul_low_i8x16(v128_t a, v128_t b) { // CHECK-LABEL: @test_i16x8_extmul_high_i8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <8 x i8> [[VECINIT14_I2_I]] to <8 x i16> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <8 x i16> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> +// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <8 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <8 x i16> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP2]] // @@ -3031,12 +3031,12 @@ v128_t test_i16x8_extmul_high_i8x16(v128_t a, v128_t b) { // CHECK-LABEL: @test_u16x8_extmul_low_u8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <8 x i8> [[VECINIT14_I2_I]] to <8 x i16> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <8 x i16> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> +// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <8 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <8 x i16> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP2]] // @@ -3047,12 +3047,12 @@ v128_t test_u16x8_extmul_low_u8x16(v128_t a, v128_t b) { // CHECK-LABEL: @test_u16x8_extmul_high_u8x16( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> undef, <8 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> -// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <8 x i32> +// CHECK-NEXT: [[VECINIT14_I2_I:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <8 x i8> [[VECINIT14_I2_I]] to <8 x i16> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <8 x i16> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <16 x i8> +// CHECK-NEXT: [[VECINIT14_I_I:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <8 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <8 x i8> [[VECINIT14_I_I]] to <8 x i16> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <8 x i16> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP2]] // @@ -3063,12 +3063,12 @@ v128_t test_u16x8_extmul_high_u8x16(v128_t a, v128_t b) { // CHECK-LABEL: @test_i32x4_extmul_low_i16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <4 x i16> [[VECINIT6_I2_I]] to <4 x i32> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <4 x i32> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> +// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <4 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <4 x i32> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: ret <4 x i32> [[MUL_I]] // v128_t test_i32x4_extmul_low_i16x8(v128_t a, v128_t b) { @@ -3078,12 +3078,12 @@ v128_t test_i32x4_extmul_low_i16x8(v128_t a, v128_t b) { // CHECK-LABEL: @test_i32x4_extmul_high_i16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <4 x i16> [[VECINIT6_I2_I]] to <4 x i32> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <4 x i32> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> +// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <4 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <4 x i32> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: ret <4 x i32> [[MUL_I]] // v128_t test_i32x4_extmul_high_i16x8(v128_t a, v128_t b) { @@ -3093,12 +3093,12 @@ v128_t test_i32x4_extmul_high_i16x8(v128_t a, v128_t b) { // CHECK-LABEL: @test_u32x4_extmul_low_u16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <4 x i16> [[VECINIT6_I2_I]] to <4 x i32> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <4 x i32> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> +// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <4 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <4 x i32> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: ret <4 x i32> [[MUL_I]] // v128_t test_u32x4_extmul_low_u16x8(v128_t a, v128_t b) { @@ -3108,12 +3108,12 @@ v128_t test_u32x4_extmul_low_u16x8(v128_t a, v128_t b) { // CHECK-LABEL: @test_u32x4_extmul_high_u16x8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[A:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> undef, <4 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> -// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> +// CHECK-NEXT: [[VECINIT6_I2_I:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <4 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <4 x i16> [[VECINIT6_I2_I]] to <4 x i32> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <4 x i32> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B:%.*]] to <8 x i16> +// CHECK-NEXT: [[VECINIT6_I_I:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <4 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <4 x i16> [[VECINIT6_I_I]] to <4 x i32> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <4 x i32> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: ret <4 x i32> [[MUL_I]] // v128_t test_u32x4_extmul_high_u16x8(v128_t a, v128_t b) { @@ -3122,11 +3122,11 @@ v128_t test_u32x4_extmul_high_u16x8(v128_t a, v128_t b) { // CHECK-LABEL: @test_i64x2_extmul_low_i32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> -// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <2 x i32> [[VECINIT2_I2_I]] to <2 x i64> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <2 x i64> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> poison, <2 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <2 x i64> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] // @@ -3136,11 +3136,11 @@ v128_t test_i64x2_extmul_low_i32x4(v128_t a, v128_t b) { // CHECK-LABEL: @test_i64x2_extmul_high_i32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> -// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = sext <2 x i32> [[VECINIT2_I2_I]] to <2 x i64> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <2 x i64> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> poison, <2 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = sext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nsw <2 x i64> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] // @@ -3150,11 +3150,11 @@ v128_t test_i64x2_extmul_high_i32x4(v128_t a, v128_t b) { // CHECK-LABEL: @test_u64x2_extmul_low_u32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> -// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <2 x i32> [[VECINIT2_I2_I]] to <2 x i64> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <2 x i64> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> poison, <2 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <2 x i64> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] // @@ -3164,11 +3164,11 @@ v128_t test_u64x2_extmul_low_u32x4(v128_t a, v128_t b) { // CHECK-LABEL: @test_u64x2_extmul_high_u32x4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> -// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> -// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> +// CHECK-NEXT: [[VECINIT2_I2_I:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> poison, <2 x i32> // CHECK-NEXT: [[CONV_I3_I:%.*]] = zext <2 x i32> [[VECINIT2_I2_I]] to <2 x i64> -// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <2 x i64> [[CONV_I3_I]], [[CONV_I_I]] +// CHECK-NEXT: [[VECINIT2_I_I:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> poison, <2 x i32> +// CHECK-NEXT: [[CONV_I_I:%.*]] = zext <2 x i32> [[VECINIT2_I_I]] to <2 x i64> +// CHECK-NEXT: [[MUL_I:%.*]] = mul nuw <2 x i64> [[CONV_I_I]], [[CONV_I3_I]] // CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[MUL_I]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP0]] // diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 00f97c2f96c6c..5dcd7598c2a50 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1345,15 +1345,15 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, APInt EltMask(APInt::getAllOnes(VWidth)); assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!"); - if (match(V, m_Undef())) { - // If the entire vector is undef or poison, just return this info. + if (DemandedElts.isZero()) { // If nothing is demanded, provide poison. PoisonElts = EltMask; - return nullptr; + return !isa(V) ? PoisonValue::get(V->getType()) : nullptr; } - if (DemandedElts.isZero()) { // If nothing is demanded, provide poison. + if (match(V, m_Undef())) { + // If the entire vector is undef or poison, just return this info. PoisonElts = EltMask; - return PoisonValue::get(V->getType()); + return nullptr; } PoisonElts = 0; diff --git a/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll b/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll index 39e0956db22a1..e3c81cb14d40a 100644 --- a/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll +++ b/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll @@ -4,7 +4,7 @@ define <2 x float> @fcvtxn(<2 x double> %d1) { ; CHECK-LABEL: @fcvtxn( ; CHECK-NEXT: [[I:%.*]] = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> [[D1:%.*]]) -; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x float> [[I]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x float> [[I]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: ret <2 x float> [[S]] ; %a = shufflevector <2 x double> %d1, <2 x double> undef, <2 x i32> @@ -17,7 +17,7 @@ define <4 x i16> @rshrn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @rshrn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.rshrn.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -30,7 +30,7 @@ define <4 x i16> @sqrshrn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqrshrn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrshrn.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -43,7 +43,7 @@ define <4 x i16> @sqrshrun(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqrshrun( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrshrun.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -56,7 +56,7 @@ define <4 x i16> @sqshrn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqshrn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqshrn.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -69,7 +69,7 @@ define <4 x i16> @sqshrun(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqshrun( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqshrun.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -82,7 +82,7 @@ define <4 x i16> @sqxtn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqxtn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[A]]) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -95,7 +95,7 @@ define <4 x i16> @sqxtun(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @sqxtun( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[A]]) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -108,7 +108,7 @@ define <4 x i16> @uqrshrn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @uqrshrn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqrshrn.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -121,7 +121,7 @@ define <4 x i16> @uqshrn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @uqshrn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqshrn.v4i16(<4 x i32> [[A]], i32 9) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> @@ -134,7 +134,7 @@ define <4 x i16> @uqxtn(<2 x i32> %d1, <2 x i32> %d2) { ; CHECK-LABEL: @uqxtn( ; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[A]]) -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[S]] ; %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll index ae0f4299ca181..53300e0c9771a 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll @@ -210,7 +210,7 @@ define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -228,7 +228,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i3 define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -246,7 +246,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3 define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -4872,7 +4872,7 @@ define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { ; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32( ; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0) -; CHECK-NEXT: [[SHUF:%.*]] = insertelement <2 x float> undef, float [[DATA]], i64 0 +; CHECK-NEXT: [[SHUF:%.*]] = insertelement <2 x float> poison, float [[DATA]], i64 0 ; CHECK-NEXT: ret <2 x float> [[SHUF]] ; %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0) @@ -4913,7 +4913,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32( ; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0) -; CHECK-NEXT: [[SHUF:%.*]] = insertelement <3 x float> undef, float [[DATA]], i64 0 +; CHECK-NEXT: [[SHUF:%.*]] = insertelement <3 x float> poison, float [[DATA]], i64 0 ; CHECK-NEXT: ret <3 x float> [[SHUF]] ; %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0) diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll index f2a8e563d2d47..ce809a5676b82 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll @@ -210,7 +210,7 @@ define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -228,7 +228,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i3 define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -246,7 +246,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3 define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5( ; CHECK-NEXT: [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false) -; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> +; CHECK-NEXT: [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> ; CHECK-NEXT: [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]] ; CHECK-NEXT: ret <2 x float> [[RET]] @@ -4871,7 +4871,7 @@ define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { ; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32( ; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0) -; CHECK-NEXT: [[SHUF:%.*]] = insertelement <2 x float> undef, float [[DATA]], i64 0 +; CHECK-NEXT: [[SHUF:%.*]] = insertelement <2 x float> poison, float [[DATA]], i64 0 ; CHECK-NEXT: ret <2 x float> [[SHUF]] ; %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0) @@ -4912,7 +4912,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32( ; CHECK-NEXT: [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0) -; CHECK-NEXT: [[SHUF:%.*]] = insertelement <3 x float> undef, float [[DATA]], i64 0 +; CHECK-NEXT: [[SHUF:%.*]] = insertelement <3 x float> poison, float [[DATA]], i64 0 ; CHECK-NEXT: ret <3 x float> [[SHUF]] ; %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0) diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll index 3ade1ba3cf78b..c788e4ea81efc 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll @@ -13,7 +13,7 @@ declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8 immarg define double @elts_addsub_v2f64(<2 x double> %0, <2 x double> %1) { ; CHECK-LABEL: @elts_addsub_v2f64( -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP3]] ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0 ; CHECK-NEXT: ret double [[TMP5]] @@ -142,7 +142,7 @@ define float @elts_addsub_v8f32_sub(<8 x float> %0, <8 x float> %1) { define void @PR46277(float %0, float %1, float %2, float %3, <4 x float> %4, ptr %5) { ; CHECK-LABEL: @PR46277( -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> undef, float [[TMP0:%.*]], i64 0 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[TMP0:%.*]], i64 0 ; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1:%.*]], i64 1 ; CHECK-NEXT: [[TMP9:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP8]], <4 x float> [[TMP4:%.*]]) ; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i64 0 @@ -181,7 +181,7 @@ define double @PR48476_fsub(<2 x double> %x) { define double @PR48476_fadd_fsub(<2 x double> %x) { ; CHECK-LABEL: @PR48476_fadd_fsub( ; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], -; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = fsub <2 x double> [[S]], [[X]] ; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <2 x double> [[TMP2]], i64 0 ; CHECK-NEXT: ret double [[VECEXT]] diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll index be6ad78303825..04e095d7055c3 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll @@ -85,7 +85,7 @@ define <8 x float> @undef_test_vpermps(<8 x float> %a0) { define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) { ; CHECK-LABEL: @elts_test_vpermd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i32> [[TMP1]] ; %1 = insertelement <8 x i32> , i32 %a1, i32 0 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll index 431079648c845..5504fa15dc302 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll @@ -159,8 +159,8 @@ define <8 x i64> @fold_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) { define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuludq_128( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i64> [[TMP3]], @@ -178,8 +178,8 @@ define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) { define <4 x i64> @test_demanded_elts_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuludq_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i64> [[TMP3]], @@ -195,8 +195,8 @@ define <4 x i64> @test_demanded_elts_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) { define <8 x i64> @test_demanded_elts_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuludq_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> poison, <16 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = and <8 x i64> [[TMP3]], @@ -212,8 +212,8 @@ define <8 x i64> @test_demanded_elts_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) define <2 x i64> @test_demanded_elts_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuldq_128( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = shl <2 x i64> [[TMP3]], @@ -231,8 +231,8 @@ define <2 x i64> @test_demanded_elts_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) { define <4 x i64> @test_demanded_elts_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuldq_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = shl <4 x i64> [[TMP3]], @@ -240,7 +240,7 @@ define <4 x i64> @test_demanded_elts_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-NEXT: [[TMP7:%.*]] = shl <4 x i64> [[TMP4]], ; CHECK-NEXT: [[TMP8:%.*]] = ashr exact <4 x i64> [[TMP7]], ; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i64> [[TMP6]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[TMP10]] ; %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> @@ -252,8 +252,8 @@ define <4 x i64> @test_demanded_elts_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) { define <8 x i64> @test_demanded_elts_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) { ; CHECK-LABEL: @test_demanded_elts_pmuldq_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> poison, <16 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64> ; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64> ; CHECK-NEXT: [[TMP5:%.*]] = shl <8 x i64> [[TMP3]], @@ -261,7 +261,7 @@ define <8 x i64> @test_demanded_elts_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) ; CHECK-NEXT: [[TMP7:%.*]] = shl <8 x i64> [[TMP4]], ; CHECK-NEXT: [[TMP8:%.*]] = ashr exact <8 x i64> [[TMP7]], ; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <8 x i64> [[TMP6]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> undef, <8 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i64> [[TMP10]] ; %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll index c84839b7ee9b8..9c85ca504beca 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll @@ -207,7 +207,7 @@ define <64 x i8> @fold_packuswb_512() { define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_128( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef) +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> poison) ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[TMP2]] ; @@ -255,7 +255,7 @@ define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) { define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_256( -; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef) +; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> poison) ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i16> [[TMP2]] ; @@ -304,7 +304,7 @@ define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) { define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_512( -; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef) +; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> poison) ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> poison, <32 x i32> ; CHECK-NEXT: ret <32 x i16> [[TMP2]] ; diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll index cea7974e29f6e..6ddcb856692b0 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll @@ -207,8 +207,8 @@ define <64 x i8> @fold_packuswb_512() { define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_128( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> poison) +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[TMP2]] ; %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> @@ -221,7 +221,7 @@ define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) { define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: @elts_packusdw_128( ; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[TMP2]] ; %1 = insertelement <4 x i32> %a0, i32 0, i32 0 @@ -255,8 +255,8 @@ define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) { define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_256( -; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> undef, <16 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> poison) +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i16> [[TMP2]] ; %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> @@ -268,9 +268,9 @@ define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) { define <16 x i16> @elts_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_packusdw_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> poison, <8 x i32> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> undef, <16 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i16> [[TMP3]] ; %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> @@ -304,8 +304,8 @@ define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) { define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) { ; CHECK-LABEL: @elts_packssdw_512( -; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> undef, <32 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> poison) +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> poison, <32 x i32> ; CHECK-NEXT: ret <32 x i16> [[TMP2]] ; %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> @@ -317,9 +317,9 @@ define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) { define <32 x i16> @elts_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) { ; CHECK-LABEL: @elts_packusdw_512( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> poison, <16 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> poison, <16 x i32> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> undef, <32 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> poison, <32 x i32> ; CHECK-NEXT: ret <32 x i16> [[TMP3]] ; %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll index 3af3d7a9449f5..9fde3237737ec 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll @@ -473,7 +473,7 @@ define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) { define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, i8 %M0, i8 %M15) { ; CHECK-LABEL: @demanded_elts_insertion( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> [[INVEC:%.*]], <16 x i8> [[BASEMASK:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i8> [[TMP2]] ; %1 = insertelement <16 x i8> %BaseMask, i8 %M0, i32 0 @@ -486,7 +486,7 @@ define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %BaseMask, i8 %M0, i8 %M22) { ; CHECK-LABEL: @demanded_elts_insertion_avx2( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[INVEC:%.*]], <32 x i8> [[BASEMASK:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> undef, <32 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <32 x i32> ; CHECK-NEXT: ret <32 x i8> [[TMP2]] ; %1 = insertelement <32 x i8> %BaseMask, i8 %M0, i32 0 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll index a3b14ef2b1c1b..4600a6654a362 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll @@ -2963,7 +2963,7 @@ define <4 x i32> @avx2_psrav_d_128_masked(<4 x i32> %v, <4 x i32> %a) { define <4 x i32> @avx2_psrav_d_128_masked_shuffle(<4 x i32> %v, <4 x i32> %a) { ; CHECK-LABEL: @avx2_psrav_d_128_masked_shuffle( ; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[V:%.*]], [[TMP2]] ; CHECK-NEXT: ret <4 x i32> [[TMP3]] ; diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll index 931160610b258..103d739a56e7c 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll @@ -225,7 +225,7 @@ define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) { define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_ps( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x i32> , i32 %a1, i32 3 @@ -236,7 +236,7 @@ define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) { define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_ps_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> ; CHECK-NEXT: ret <8 x float> [[TMP1]] ; %1 = shufflevector <8 x i32> %a1, <8 x i32> , <8 x i32> @@ -248,7 +248,7 @@ define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a1, i32 %a2) { ; CHECK-LABEL: @elts_test_vpermilvar_ps_512( ; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[A0:%.*]], <16 x i32> [[A1:%.*]]) -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> poison, <16 x i32> ; CHECK-NEXT: ret <16 x float> [[TMP2]] ; %1 = insertelement <16 x i32> %a1, i32 %a2, i32 0 @@ -259,7 +259,7 @@ define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_pd( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x i64> , i64 %a1, i32 1 @@ -270,7 +270,7 @@ define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) { define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) { ; CHECK-LABEL: @elts_test_vpermilvar_pd_256( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> ; CHECK-NEXT: ret <4 x double> [[TMP1]] ; %1 = shufflevector <4 x i64> , <4 x i64> %a1, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll index 86d45bc706409..c7609564685fe 100644 --- a/llvm/test/Transforms/InstCombine/assume.ll +++ b/llvm/test/Transforms/InstCombine/assume.ll @@ -426,7 +426,7 @@ define void @debug_interference(i8 %x) { define i32 @PR40940(<4 x i8> %x) { ; CHECK-LABEL: @PR40940( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[T2:%.*]] = bitcast <4 x i8> [[SHUF]] to i32 ; CHECK-NEXT: [[T3:%.*]] = icmp ult i32 [[T2]], 65536 ; CHECK-NEXT: call void @llvm.assume(i1 [[T3]]) diff --git a/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll b/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll index 6cf8bb73c9ff7..7dc2fe1cb88e1 100644 --- a/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll +++ b/llvm/test/Transforms/InstCombine/binop-select-cast-of-select-cond.ll @@ -132,7 +132,7 @@ define <2 x i64> @vector_test(i1 %c) { ; CHECK-SAME: (i1 [[C:%.*]]) { ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], <2 x i64> , <2 x i64> ; CHECK-NEXT: [[EXT:%.*]] = zext i1 [[C]] to i64 -; CHECK-NEXT: [[VEC0:%.*]] = insertelement <2 x i64> undef, i64 [[EXT]], i64 0 +; CHECK-NEXT: [[VEC0:%.*]] = insertelement <2 x i64> poison, i64 [[EXT]], i64 0 ; CHECK-NEXT: [[VEC1:%.*]] = shufflevector <2 x i64> [[VEC0]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw <2 x i64> [[SEL]], [[VEC1]] ; CHECK-NEXT: ret <2 x i64> [[ADD]] diff --git a/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll b/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll index 94e08d537575e..89f0c3f96349d 100644 --- a/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll @@ -154,7 +154,7 @@ define <3 x i64> @bitcast_inselt_undef_from_mmx(x86_mmx %x, i32 %idx) { define <2 x i64> @PR45748(double %x, double %y) { ; CHECK-LABEL: @PR45748( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[Y:%.*]], i64 1 ; CHECK-NEXT: [[I1:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[I1]] diff --git a/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll index fb5c91c7460a1..8b8325b147263 100644 --- a/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll +++ b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll @@ -154,7 +154,7 @@ define <3 x i64> @bitcast_inselt_undef_from_mmx(x86_mmx %x, i32 %idx) { define <2 x i64> @PR45748(double %x, double %y) { ; CHECK-LABEL: @PR45748( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[Y:%.*]], i64 1 ; CHECK-NEXT: [[I1:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[I1]] diff --git a/llvm/test/Transforms/InstCombine/bitreverse.ll b/llvm/test/Transforms/InstCombine/bitreverse.ll index bf09ffe141012..7d122297c11b2 100644 --- a/llvm/test/Transforms/InstCombine/bitreverse.ll +++ b/llvm/test/Transforms/InstCombine/bitreverse.ll @@ -298,7 +298,7 @@ define i4 @shuf_bitcast_twice_4bits(i4 %x) { ; Negative tests - not reverse define i4 @shuf_4bits_not_reverse(<4 x i1> %x) { ; CHECK-LABEL: @shuf_4bits_not_reverse( -; CHECK-NEXT: [[BITREVERSE:%.*]] = shufflevector <4 x i1> [[X:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[BITREVERSE:%.*]] = shufflevector <4 x i1> [[X:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i1> [[BITREVERSE]] to i4 ; CHECK-NEXT: ret i4 [[CAST]] ; @@ -312,7 +312,7 @@ declare void @use(<4 x i1>) define i4 @shuf_4bits_extra_use(<4 x i1> %x) { ; CHECK-LABEL: @shuf_4bits_extra_use( -; CHECK-NEXT: [[BITREVERSE:%.*]] = shufflevector <4 x i1> [[X:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[BITREVERSE:%.*]] = shufflevector <4 x i1> [[X:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x i1> [[BITREVERSE]]) ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i1> [[BITREVERSE]] to i4 ; CHECK-NEXT: ret i4 [[CAST]] diff --git a/llvm/test/Transforms/InstCombine/broadcast.ll b/llvm/test/Transforms/InstCombine/broadcast.ll index 5e5c3d6a03380..c70b975a312ad 100644 --- a/llvm/test/Transforms/InstCombine/broadcast.ll +++ b/llvm/test/Transforms/InstCombine/broadcast.ll @@ -3,7 +3,7 @@ define <4 x float> @good1(float %arg) { ; CHECK-LABEL: @good1( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x float> [[T6]] ; @@ -16,7 +16,7 @@ define <4 x float> @good1(float %arg) { define <4 x float> @good2(float %arg) { ; CHECK-LABEL: @good2( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: ret <4 x float> [[T6]] ; @@ -77,8 +77,8 @@ define <4 x float> @good5(float %v) { define <4 x float> @splat_undef1(float %arg) { ; CHECK-LABEL: @splat_undef1( ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 1 -; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG:%.*]], i64 2 -; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG:%.*]], i64 3 +; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 +; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] ; %t = insertelement <4 x float> undef, float %arg, i32 1 @@ -93,8 +93,8 @@ define <4 x float> @splat_undef1(float %arg) { define <4 x float> @splat_undef2(float %arg) { ; CHECK-LABEL: @splat_undef2( ; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 -; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T]], float [[ARG:%.*]], i64 2 -; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG:%.*]], i64 3 +; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 2 +; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] ; %t = insertelement <4 x float> undef, float %arg, i32 0 @@ -105,7 +105,7 @@ define <4 x float> @splat_undef2(float %arg) { define <4 x float> @bad3(float %arg, float %arg2) { ; CHECK-LABEL: @bad3( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG2:%.*]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 @@ -120,7 +120,7 @@ define <4 x float> @bad3(float %arg, float %arg2) { define <1 x float> @bad4(float %arg) { ; CHECK-LABEL: @bad4( -; CHECK-NEXT: [[T:%.*]] = insertelement <1 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <1 x float> poison, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: ret <1 x float> [[T]] ; %t = insertelement <1 x float> undef, float %arg, i32 0 @@ -133,9 +133,9 @@ define <1 x float> @bad4(float %arg) { define <4 x float> @splat_undef3(float %arg) { ; CHECK-LABEL: @splat_undef3( ; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 -; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG:%.*]], i64 1 -; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG:%.*]], i64 2 -; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG:%.*]], i64 3 +; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 1 +; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 +; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: [[T7:%.*]] = fadd <4 x float> [[T6]], [[T4]] ; CHECK-NEXT: ret <4 x float> [[T7]] ; diff --git a/llvm/test/Transforms/InstCombine/bswap.ll b/llvm/test/Transforms/InstCombine/bswap.ll index 756e898b18eba..21eb170b8c58d 100644 --- a/llvm/test/Transforms/InstCombine/bswap.ll +++ b/llvm/test/Transforms/InstCombine/bswap.ll @@ -690,7 +690,7 @@ declare void @use(<4 x i8>) define i32 @shuf_4bytes_extra_use(<4 x i8> %x) { ; CHECK-LABEL: @shuf_4bytes_extra_use( -; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x i8> [[BSWAP]]) ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32 ; CHECK-NEXT: ret i32 [[CAST]] @@ -705,7 +705,7 @@ define i32 @shuf_4bytes_extra_use(<4 x i8> %x) { define i128 @shuf_16bytes(<16 x i8> %x) { ; CHECK-LABEL: @shuf_16bytes( -; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: [[CAST:%.*]] = bitcast <16 x i8> [[BSWAP]] to i128 ; CHECK-NEXT: ret i128 [[CAST]] ; @@ -718,7 +718,7 @@ define i128 @shuf_16bytes(<16 x i8> %x) { define i32 @shuf_2bytes_widening(<2 x i8> %x) { ; CHECK-LABEL: @shuf_2bytes_widening( -; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32 ; CHECK-NEXT: ret i32 [[CAST]] ; diff --git a/llvm/test/Transforms/InstCombine/extractelement.ll b/llvm/test/Transforms/InstCombine/extractelement.ll index 540a4c7d1722a..3d94106d7f888 100644 --- a/llvm/test/Transforms/InstCombine/extractelement.ll +++ b/llvm/test/Transforms/InstCombine/extractelement.ll @@ -269,7 +269,7 @@ define i16 @bitcasted_inselt_from_FP_uses2(double %x) { define float @bitcasted_inselt_to_and_from_FP(double %x) { ; ANY-LABEL: @bitcasted_inselt_to_and_from_FP( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float> ; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i64 1 ; ANY-NEXT: ret float [[R]] diff --git a/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll index a24766ad8ab61..9369d5564c3a2 100644 --- a/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll +++ b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll @@ -74,7 +74,7 @@ define i1 @test_i8_pattern_2(i8 %val) { ; Make sure we don't try to fold if the shufflemask has differing element values define i1 @test_i8_pattern_3(<4 x i8> %invec) { ; CHECK-LABEL: @test_i8_pattern_3( -; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INVEC:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INVEC:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32 ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696648 ; CHECK-NEXT: ret i1 [[COND]] @@ -88,7 +88,7 @@ define i1 @test_i8_pattern_3(<4 x i8> %invec) { ; Make sure we don't try to fold if the compared-to constant isn't a splatted value define i1 @test_i8_nopattern(i8 %val) { ; CHECK-LABEL: @test_i8_nopattern( -; CHECK-NEXT: [[INSVEC:%.*]] = insertelement <4 x i8> undef, i8 [[VAL:%.*]], i64 0 +; CHECK-NEXT: [[INSVEC:%.*]] = insertelement <4 x i8> poison, i8 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INSVEC]], <4 x i8> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32 ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696647 diff --git a/llvm/test/Transforms/InstCombine/icmp-vec.ll b/llvm/test/Transforms/InstCombine/icmp-vec.ll index 30bfb0daf7a3d..38e30334342b1 100644 --- a/llvm/test/Transforms/InstCombine/icmp-vec.ll +++ b/llvm/test/Transforms/InstCombine/icmp-vec.ll @@ -242,7 +242,7 @@ declare void @use_v4i8(<4 x i8>) define <4 x i1> @same_shuffle_inputs_icmp_extra_use1(<4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use1( -; CHECK-NEXT: [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i8> [[X]], [[Y:%.*]] ; CHECK-NEXT: [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: call void @use_v4i8(<4 x i8> [[SHUFX]]) @@ -259,7 +259,7 @@ declare void @use_v2i8(<2 x i8>) define <2 x i1> @same_shuffle_inputs_icmp_extra_use2(<4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use2( -; CHECK-NEXT: [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> +; CHECK-NEXT: [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[X:%.*]], [[Y]] ; CHECK-NEXT: [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <2 x i32> ; CHECK-NEXT: call void @use_v2i8(<2 x i8> [[SHUFY]]) @@ -339,7 +339,7 @@ define <4 x i1> @splat_fcmp_smaller_size(<5 x float> %x) { define <4 x i1> @splat_icmp_extra_use(<4 x i8> %x) { ; CHECK-LABEL: @splat_icmp_extra_use( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: call void @use_v4i8(<4 x i8> [[SPLATX]]) ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i8> [[SPLATX]], ; CHECK-NEXT: ret <4 x i1> [[CMP]] @@ -354,7 +354,7 @@ define <4 x i1> @splat_icmp_extra_use(<4 x i8> %x) { define <4 x i1> @not_splat_icmp(<4 x i8> %x) { ; CHECK-LABEL: @not_splat_icmp( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i8> [[SPLATX]], ; CHECK-NEXT: ret <4 x i1> [[CMP]] ; @@ -367,7 +367,7 @@ define <4 x i1> @not_splat_icmp(<4 x i8> %x) { define <4 x i1> @not_splat_icmp2(<4 x i8> %x) { ; CHECK-LABEL: @not_splat_icmp2( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i8> [[SPLATX]], ; CHECK-NEXT: ret <4 x i1> [[CMP]] ; diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll index 76ce605463c60..6cbb2a246f5a4 100644 --- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll @@ -3,7 +3,7 @@ define <1 x i8> @test1(<8 x i8> %in) { ; CHECK-LABEL: @test1( -; CHECK-NEXT: [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> undef, <1 x i32> +; CHECK-NEXT: [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> poison, <1 x i32> ; CHECK-NEXT: ret <1 x i8> [[VEC]] ; %val = extractelement <8 x i8> %in, i32 5 diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll index 233fd4b008045..5034f44b6a6f9 100644 --- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll +++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll @@ -3,7 +3,7 @@ define <1 x i8> @test1(<8 x i8> %in) { ; CHECK-LABEL: @test1( -; CHECK-NEXT: [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> undef, <1 x i32> +; CHECK-NEXT: [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> poison, <1 x i32> ; CHECK-NEXT: ret <1 x i8> [[VEC]] ; %val = extractelement <8 x i8> %in, i32 5 @@ -464,7 +464,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) { ; CHECK-LABEL: @insert_nonzero_index_splat_extra_use( ; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 2 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] ; %xv = insertelement <4 x float> undef, float %x, i32 2 @@ -478,7 +478,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) { define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> %y) { ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_base( ; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 2 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] ; %xv = insertelement <4 x float> %y, float %x, i32 2 @@ -491,7 +491,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) { ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index( ; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 [[INDEX:%.*]] -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] ; %xv = insertelement <4 x float> undef, float %x, i32 %index @@ -501,7 +501,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) define <4 x float> @insert_in_splat(float %x) { ; CHECK-LABEL: @insert_in_splat( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[R]] ; @@ -515,7 +515,7 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) { ; CHECK-LABEL: @insert_in_splat_extra_uses( ; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x float> [[SPLAT]]) ; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[R]] @@ -532,8 +532,8 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) { define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) { ; CHECK-LABEL: @insert_in_splat_variable_index( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 [[Y:%.*]] ; CHECK-NEXT: ret <4 x float> [[R]] ; @@ -547,7 +547,7 @@ define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) { define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) { ; CHECK-LABEL: @insert_in_nonsplat( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> ; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3 ; CHECK-NEXT: ret <4 x float> [[R]] @@ -563,7 +563,7 @@ define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) { define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) { ; CHECK-LABEL: @insert_in_nonsplat2( ; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3 ; CHECK-NEXT: ret <4 x float> [[R]] ; @@ -575,7 +575,7 @@ define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) { define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) { ; CHECK-LABEL: @shuf_identity_padding( -; CHECK-NEXT: [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2 ; CHECK-NEXT: ret <4 x i8> [[V2]] ; @@ -588,7 +588,7 @@ define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) { define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) { ; CHECK-LABEL: @shuf_identity_extract( -; CHECK-NEXT: [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <3 x i32> +; CHECK-NEXT: [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <3 x i32> ; CHECK-NEXT: [[V2:%.*]] = insertelement <3 x i8> [[V1]], i8 [[Y:%.*]], i64 2 ; CHECK-NEXT: ret <3 x i8> [[V2]] ; @@ -601,9 +601,9 @@ define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) { define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) { ; CHECK-LABEL: @shuf_identity_extract_extra_use( -; CHECK-NEXT: [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> undef, <4 x i32> +; CHECK-NEXT: [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x float> [[V0]]) -; CHECK-NEXT: [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> undef, <4 x i32> +; CHECK-NEXT: [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> poison, <4 x i32> ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[Y:%.*]], i64 1 ; CHECK-NEXT: ret <4 x float> [[V2]] ; @@ -619,7 +619,7 @@ define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) { define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %index) { ; CHECK-LABEL: @shuf_identity_padding_variable_index( -; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 [[INDEX:%.*]] ; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 [[INDEX]] ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2 @@ -636,7 +636,7 @@ define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %i define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x i8> %other) { ; CHECK-LABEL: @shuf_identity_padding_wrong_source_vec( -; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[OTHER:%.*]], i64 1 ; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 1 ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2 @@ -653,7 +653,7 @@ define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x define <4 x i8> @shuf_identity_padding_wrong_index(<2 x i8> %x, i8 %y) { ; CHECK-LABEL: @shuf_identity_padding_wrong_index( -; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[X]], i64 1 ; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 2 ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 3 diff --git a/llvm/test/Transforms/InstCombine/logical-select.ll b/llvm/test/Transforms/InstCombine/logical-select.ll index c3eec5eebf2eb..fcca9588767dd 100644 --- a/llvm/test/Transforms/InstCombine/logical-select.ll +++ b/llvm/test/Transforms/InstCombine/logical-select.ll @@ -677,11 +677,11 @@ define <4 x i32> @computesignbits_through_shuffles(<4 x float> %x, <4 x float> % ; CHECK-LABEL: @computesignbits_through_shuffles( ; CHECK-NEXT: [[CMP:%.*]] = fcmp ole <4 x float> [[X:%.*]], [[Y:%.*]] ; CHECK-NEXT: [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32> -; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> undef, <4 x i32> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[SHUF_OR1:%.*]] = or <4 x i32> [[S1]], [[S2]] -; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> undef, <4 x i32> -; CHECK-NEXT: [[S4:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[S4:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[SHUF_OR2:%.*]] = or <4 x i32> [[S3]], [[S4]] ; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i32> [[SHUF_OR2]] to <4 x i1> ; CHECK-NEXT: [[SEL_V:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[Z:%.*]], <4 x float> [[X]] diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll index af9406ce382ca..aee8f400ff43a 100644 --- a/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll @@ -59,7 +59,7 @@ define <2 x double> @load_lane0(ptr %ptr, double %pt) { define double @load_all(ptr %base, double %pt) { ; CHECK-LABEL: @load_all( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> -; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> undef) +; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> poison) ; CHECK-NEXT: [[ELT:%.*]] = extractelement <4 x double> [[RES]], i64 2 ; CHECK-NEXT: ret double [[ELT]] ; diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll index 615784d473483..2704905f7a358 100644 --- a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll @@ -59,7 +59,7 @@ define <2 x double> @load_lane0(ptr %ptr, double %pt) { define double @load_all(ptr %base, double %pt) { ; CHECK-LABEL: @load_all( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> -; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> undef) +; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> poison) ; CHECK-NEXT: [[ELT:%.*]] = extractelement <4 x double> [[RES]], i64 2 ; CHECK-NEXT: ret double [[ELT]] ; @@ -71,7 +71,7 @@ define double @load_all(ptr %base, double %pt) { define <2 x double> @load_generic(ptr %ptr, double %pt, <2 x i1> %mask) { ; CHECK-LABEL: @load_generic( -; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0(ptr [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]]) ; CHECK-NEXT: ret <2 x double> [[RES]] @@ -84,7 +84,7 @@ define <2 x double> @load_generic(ptr %ptr, double %pt, <2 x i1> %mask) { define <2 x double> @load_speculative(ptr dereferenceable(16) align 4 %ptr, double %pt, <2 x i1> %mask) nofree nosync { ; CHECK-LABEL: @load_speculative( -; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, ptr [[PTR:%.*]], align 4 ; CHECK-NEXT: [[RES:%.*]] = select <2 x i1> [[MASK:%.*]], <2 x double> [[UNMASKEDLOAD]], <2 x double> [[PTV2]] @@ -98,7 +98,7 @@ define <2 x double> @load_speculative(ptr dereferenceable(16) align 4 %ptr, doub define <2 x double> @load_speculative_less_aligned(ptr dereferenceable(16) %ptr, double %pt, <2 x i1> %mask) nofree nosync { ; CHECK-LABEL: @load_speculative_less_aligned( -; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, ptr [[PTR:%.*]], align 4 ; CHECK-NEXT: [[RES:%.*]] = select <2 x i1> [[MASK:%.*]], <2 x double> [[UNMASKEDLOAD]], <2 x double> [[PTV2]] @@ -114,7 +114,7 @@ define <2 x double> @load_speculative_less_aligned(ptr dereferenceable(16) %ptr, define <2 x double> @load_spec_neg_size(ptr dereferenceable(8) %ptr, double %pt, <2 x i1> %mask) nofree nosync { ; CHECK-LABEL: @load_spec_neg_size( -; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0(ptr nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]]) ; CHECK-NEXT: ret <2 x double> [[RES]] @@ -128,7 +128,7 @@ define <2 x double> @load_spec_neg_size(ptr dereferenceable(8) %ptr, double %pt, ; Can only speculate one lane (but it's the only one active) define <2 x double> @load_spec_lan0(ptr dereferenceable(8) %ptr, double %pt, <2 x i1> %mask) nofree nosync { ; CHECK-LABEL: @load_spec_lan0( -; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1 ; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0(ptr nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PTV2]]) @@ -160,7 +160,7 @@ define void @store_onemask(ptr %ptr, <2 x double> %val) { define void @store_demandedelts(ptr %ptr, double %val) { ; CHECK-LABEL: @store_demandedelts( -; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> undef, double [[VAL:%.*]], i64 0 +; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> poison, double [[VAL:%.*]], i64 0 ; CHECK-NEXT: call void @llvm.masked.store.v2f64.p0(<2 x double> [[VALVEC1]], ptr [[PTR:%.*]], i32 4, <2 x i1> ) ; CHECK-NEXT: ret void ; @@ -201,7 +201,7 @@ define <2 x double> @gather_onemask(<2 x ptr> %ptrs, <2 x double> %passthru) { define <4 x double> @gather_lane2(ptr %base, double %pt) { ; CHECK-LABEL: @gather_lane2( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> -; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <4 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <4 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> poison, <4 x i32> ; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> , <4 x double> [[PT_V2]]) ; CHECK-NEXT: ret <4 x double> [[RES]] @@ -216,7 +216,7 @@ define <4 x double> @gather_lane2(ptr %base, double %pt) { define <2 x double> @gather_lane0_maybe(ptr %base, double %pt, <2 x i1> %mask) { ; CHECK-LABEL: @gather_lane0_maybe( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <2 x i64> -; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1 ; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]]) @@ -233,7 +233,7 @@ define <2 x double> @gather_lane0_maybe(ptr %base, double %pt, <2 x i1> %mask) define <2 x double> @gather_lane0_maybe_spec(ptr %base, double %pt, <2 x i1> %mask) { ; CHECK-LABEL: @gather_lane0_maybe_spec( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <2 x i64> -; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0 +; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0 ; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1 ; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]]) @@ -259,7 +259,7 @@ define void @scatter_zeromask(<2 x ptr> %ptrs, <2 x double> %val) { define void @scatter_demandedelts(ptr %ptr, double %val) { ; CHECK-LABEL: @scatter_demandedelts( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[PTR:%.*]], <2 x i64> -; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> undef, double [[VAL:%.*]], i64 0 +; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> poison, double [[VAL:%.*]], i64 0 ; CHECK-NEXT: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> [[VALVEC1]], <2 x ptr> [[PTRS]], i32 8, <2 x i1> ) ; CHECK-NEXT: ret void ; @@ -412,7 +412,7 @@ define <2 x i64> @gather_v2i64_uniform_ptrs_all_active_mask(ptr %src) { define <2 x i64> @negative_gather_v2i64_non_uniform_ptrs_all_active_mask(<2 x ptr> %inVal, ptr %src ) { ; CHECK-LABEL: @negative_gather_v2i64_non_uniform_ptrs_all_active_mask( ; CHECK-NEXT: [[INSERT_VALUE:%.*]] = insertelement <2 x ptr> [[INVAL:%.*]], ptr [[SRC:%.*]], i64 1 -; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> [[INSERT_VALUE]], i32 8, <2 x i1> , <2 x i64> undef) +; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> [[INSERT_VALUE]], i32 8, <2 x i1> , <2 x i64> poison) ; CHECK-NEXT: ret <2 x i64> [[RES]] ; %insert.value = insertelement <2 x ptr> %inVal, ptr %src, i32 1 diff --git a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll index 889bad5f2f76d..74aa014970e97 100644 --- a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll +++ b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll @@ -240,7 +240,7 @@ define <12 x double> @fneg_with_multiple_uses(<15 x double> %a, <20 x double> %b ; CHECK-LABEL: @fneg_with_multiple_uses( ; CHECK-NEXT: [[A_NEG:%.*]] = fneg <15 x double> [[A:%.*]] ; CHECK-NEXT: [[RES:%.*]] = tail call <12 x double> @llvm.matrix.multiply.v12f64.v15f64.v20f64(<15 x double> [[A_NEG]], <20 x double> [[B:%.*]], i32 3, i32 5, i32 4) -; CHECK-NEXT: [[RES_2:%.*]] = shufflevector <15 x double> [[A_NEG]], <15 x double> undef, <12 x i32> +; CHECK-NEXT: [[RES_2:%.*]] = shufflevector <15 x double> [[A_NEG]], <15 x double> poison, <12 x i32> ; CHECK-NEXT: [[RES_3:%.*]] = fadd <12 x double> [[RES_2]], [[RES]] ; CHECK-NEXT: ret <12 x double> [[RES_3]] ; diff --git a/llvm/test/Transforms/InstCombine/nsw.ll b/llvm/test/Transforms/InstCombine/nsw.ll index 2d68236d93346..6ced39a88c0a6 100644 --- a/llvm/test/Transforms/InstCombine/nsw.ll +++ b/llvm/test/Transforms/InstCombine/nsw.ll @@ -102,7 +102,7 @@ define i8 @nopreserve4(i8 %A, i8 %B) { define <3 x i32> @shl_nuw_nsw_shuffle_splat_vec(<2 x i8> %x) { ; CHECK-LABEL: @shl_nuw_nsw_shuffle_splat_vec( ; CHECK-NEXT: [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32> -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> poison, <3 x i32> ; CHECK-NEXT: [[T3:%.*]] = shl nuw nsw <3 x i32> [[SHUF]], ; CHECK-NEXT: ret <3 x i32> [[T3]] ; @@ -118,7 +118,7 @@ define <3 x i32> @shl_nuw_nsw_shuffle_splat_vec(<2 x i8> %x) { define <3 x i32> @shl_nuw_nsw_shuffle_undef_elt_splat_vec(<2 x i8> %x) { ; CHECK-LABEL: @shl_nuw_nsw_shuffle_undef_elt_splat_vec( ; CHECK-NEXT: [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32> -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> poison, <3 x i32> ; CHECK-NEXT: [[T3:%.*]] = shl <3 x i32> [[SHUF]], ; CHECK-NEXT: ret <3 x i32> [[T3]] ; diff --git a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll index 5752612b0b6c9..949c41900c8dd 100644 --- a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll +++ b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll @@ -5,9 +5,7 @@ define void @test_undef(ptr %in_ptr, ptr %out_ptr) { ; CHECK-LABEL: define void @test_undef( ; CHECK-SAME: ptr [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) { ; CHECK-NEXT: [[A:%.*]] = load <4 x float>, ptr [[IN_PTR]], align 16 -; CHECK-NEXT: [[B:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <4 x i32> -; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[A]], <4 x i32> -; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[C]], <4 x float> [[A]], <4 x i32> +; CHECK-NEXT: [[D:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: store <4 x float> [[D]], ptr [[OUT_PTR]], align 16 ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll index 8a20b09c2990e..1fa0c09a9e987 100644 --- a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll +++ b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll @@ -571,7 +571,7 @@ define <2 x i8> @select_xor_icmp_vec_bad(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) define <2 x i32> @vec_select_no_equivalence(<2 x i32> %x) { ; CHECK-LABEL: @vec_select_no_equivalence( -; CHECK-NEXT: [[X10:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> +; CHECK-NEXT: [[X10:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> ; CHECK-NEXT: [[COND:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer ; CHECK-NEXT: [[S:%.*]] = select <2 x i1> [[COND]], <2 x i32> [[X10]], <2 x i32> [[X]] ; CHECK-NEXT: ret <2 x i32> [[S]] diff --git a/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll index 3b4977f1927ab..2348490265e1b 100644 --- a/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll @@ -21,7 +21,7 @@ define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) # ; CHECK-LABEL: @extract_two_select( ; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[C:%.*]], 0 ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]] -; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> +; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> poison, <2 x i32> ; CHECK-NEXT: ret <2 x float> [[BUILD2]] ; %cmp = icmp ne i32 %c, 0 @@ -85,7 +85,7 @@ define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32 ; CHECK-LABEL: @extract_two_vselect( ; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer ; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]] -; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> +; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> poison, <2 x i32> ; CHECK-NEXT: ret <2 x float> [[BUILD2]] ; %cmp = icmp ne <4 x i32> %c, zeroinitializer @@ -146,7 +146,7 @@ entry: define <4 x i32> @extract_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) { ; CHECK-LABEL: @extract_cond( -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[R]] ; @@ -200,7 +200,7 @@ define <4 x i32> @extract_cond_variable_index(<4 x i32> %x, <4 x i32> %y, <4 x i define <4 x i32> @extract_cond_type_mismatch(<4 x i32> %x, <4 x i32> %y, <5 x i1> %condv) { ; CHECK-LABEL: @extract_cond_type_mismatch( -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> undef, <4 x i32> +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[R]] ; diff --git a/llvm/test/Transforms/InstCombine/select-extractelement.ll b/llvm/test/Transforms/InstCombine/select-extractelement.ll index 51e0451cb6dcd..e5b4fe5051e10 100644 --- a/llvm/test/Transforms/InstCombine/select-extractelement.ll +++ b/llvm/test/Transforms/InstCombine/select-extractelement.ll @@ -21,7 +21,7 @@ define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) # ; CHECK-LABEL: @extract_two_select( ; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[C:%.*]], 0 ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]] -; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> +; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> poison, <2 x i32> ; CHECK-NEXT: ret <2 x float> [[BUILD2]] ; %cmp = icmp ne i32 %c, 0 @@ -85,7 +85,7 @@ define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32 ; CHECK-LABEL: @extract_two_vselect( ; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer ; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]] -; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> +; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> poison, <2 x i32> ; CHECK-NEXT: ret <2 x float> [[BUILD2]] ; %cmp = icmp ne <4 x i32> %c, zeroinitializer @@ -146,7 +146,7 @@ entry: define <4 x i32> @extract_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) { ; CHECK-LABEL: @extract_cond( -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[R]] ; @@ -157,7 +157,7 @@ define <4 x i32> @extract_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) { define <4 x i32> @splat_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) { ; CHECK-LABEL: @splat_cond( -; CHECK-NEXT: [[SPLATCOND:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[SPLATCOND:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[SPLATCOND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[R]] ; @@ -200,7 +200,7 @@ define <4 x i32> @extract_cond_variable_index(<4 x i32> %x, <4 x i32> %y, <4 x i define <4 x i32> @extract_cond_type_mismatch(<4 x i32> %x, <4 x i32> %y, <5 x i1> %condv) { ; CHECK-LABEL: @extract_cond_type_mismatch( -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> undef, <4 x i32> +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]] ; CHECK-NEXT: ret <4 x i32> [[R]] ; diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 4e9499ca7a586..6ea2718abb2bb 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -78,7 +78,7 @@ define <4 x i32> @lshr_C1_add_A_C2_v4i32(<4 x i32> %A) { define <4 x i32> @shl_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @shl_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 -; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 +; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0 ; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = shl <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] @@ -94,7 +94,7 @@ define <4 x i32> @shl_C1_add_A_C2_v4i32_splat(i16 %I) { define <4 x i32> @ashr_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @ashr_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 -; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 +; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0 ; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = ashr <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] @@ -110,7 +110,7 @@ define <4 x i32> @ashr_C1_add_A_C2_v4i32_splat(i16 %I) { define <4 x i32> @lshr_C1_add_A_C2_v4i32_splat(i16 %I) { ; CHECK-LABEL: @lshr_C1_add_A_C2_v4i32_splat( ; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32 -; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> undef, i32 [[A]], i64 0 +; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0 ; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[E:%.*]] = lshr <4 x i32> , [[C]] ; CHECK-NEXT: ret <4 x i32> [[E]] @@ -702,7 +702,7 @@ define <3 x i32> @add3_i96(<3 x i32> %0, <3 x i32> %1) { ; CHECK-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP11]], 32 ; CHECK-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32 ; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP14]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <3 x i32> undef, i32 [[ADD_NARROWED]], i64 0 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <3 x i32> poison, i32 [[ADD_NARROWED]], i64 0 ; CHECK-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP11]] to i32 ; CHECK-NEXT: [[TMP20:%.*]] = insertelement <3 x i32> [[TMP18]], i32 [[TMP19]], i64 1 ; CHECK-NEXT: [[TMP21:%.*]] = insertelement <3 x i32> [[TMP20]], i32 [[TMP17]], i64 2 diff --git a/llvm/test/Transforms/InstCombine/shuffle-cast.ll b/llvm/test/Transforms/InstCombine/shuffle-cast.ll index 677b9b057cdc0..281f41b576bce 100644 --- a/llvm/test/Transforms/InstCombine/shuffle-cast.ll +++ b/llvm/test/Transforms/InstCombine/shuffle-cast.ll @@ -9,7 +9,7 @@ define <4 x i16> @trunc_little_endian(<4 x i32> %x) { ; ; BE-LABEL: @trunc_little_endian( ; BE-NEXT: [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16> -; BE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> +; BE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> ; BE-NEXT: ret <4 x i16> [[R]] ; %b = bitcast <4 x i32> %x to <8 x i16> @@ -20,7 +20,7 @@ define <4 x i16> @trunc_little_endian(<4 x i32> %x) { define <4 x i16> @trunc_big_endian(<4 x i32> %x) { ; LE-LABEL: @trunc_big_endian( ; LE-NEXT: [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16> -; LE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> +; LE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> ; LE-NEXT: ret <4 x i16> [[R]] ; ; BE-LABEL: @trunc_big_endian( @@ -46,7 +46,7 @@ define <2 x i16> @trunc_little_endian_extra_use(<2 x i64> %x) { ; BE-LABEL: @trunc_little_endian_extra_use( ; BE-NEXT: [[B:%.*]] = bitcast <2 x i64> [[X:%.*]] to <8 x i16> ; BE-NEXT: call void @use_v8i16(<8 x i16> [[B]]) -; BE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <2 x i32> +; BE-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <2 x i32> ; BE-NEXT: ret <2 x i16> [[R]] ; %b = bitcast <2 x i64> %x to <8 x i16> @@ -63,7 +63,7 @@ define <4 x i11> @trunc_big_endian_extra_use(<4 x i33> %x) { ; LE-LABEL: @trunc_big_endian_extra_use( ; LE-NEXT: [[B:%.*]] = bitcast <4 x i33> [[X:%.*]] to <12 x i11> ; LE-NEXT: call void @use_v12i11(<12 x i11> [[B]]) -; LE-NEXT: [[R:%.*]] = shufflevector <12 x i11> [[B]], <12 x i11> undef, <4 x i32> +; LE-NEXT: [[R:%.*]] = shufflevector <12 x i11> [[B]], <12 x i11> poison, <4 x i32> ; LE-NEXT: ret <4 x i11> [[R]] ; ; BE-LABEL: @trunc_big_endian_extra_use( @@ -81,7 +81,7 @@ define <4 x i11> @trunc_big_endian_extra_use(<4 x i33> %x) { define <4 x i16> @wrong_cast1(i128 %x) { ; ANY-LABEL: @wrong_cast1( ; ANY-NEXT: [[B:%.*]] = bitcast i128 [[X:%.*]] to <8 x i16> -; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> +; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> ; ANY-NEXT: ret <4 x i16> [[R]] ; %b = bitcast i128 %x to <8 x i16> @@ -92,7 +92,7 @@ define <4 x i16> @wrong_cast1(i128 %x) { define <4 x i16> @wrong_cast2(<4 x float> %x) { ; ANY-LABEL: @wrong_cast2( ; ANY-NEXT: [[B:%.*]] = bitcast <4 x float> [[X:%.*]] to <8 x i16> -; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> +; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <4 x i32> ; ANY-NEXT: ret <4 x i16> [[R]] ; %b = bitcast <4 x float> %x to <8 x i16> @@ -103,7 +103,7 @@ define <4 x i16> @wrong_cast2(<4 x float> %x) { define <4 x half> @wrong_cast3(<4 x i32> %x) { ; ANY-LABEL: @wrong_cast3( ; ANY-NEXT: [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x half> -; ANY-NEXT: [[R:%.*]] = shufflevector <8 x half> [[B]], <8 x half> undef, <4 x i32> +; ANY-NEXT: [[R:%.*]] = shufflevector <8 x half> [[B]], <8 x half> poison, <4 x i32> ; ANY-NEXT: ret <4 x half> [[R]] ; %b = bitcast <4 x i32> %x to <8 x half> @@ -114,7 +114,7 @@ define <4 x half> @wrong_cast3(<4 x i32> %x) { define <2 x i16> @wrong_shuffle(<4 x i32> %x) { ; ANY-LABEL: @wrong_shuffle( ; ANY-NEXT: [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16> -; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <2 x i32> +; ANY-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> poison, <2 x i32> ; ANY-NEXT: ret <2 x i16> [[R]] ; %b = bitcast <4 x i32> %x to <8 x i16> diff --git a/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll index 98d81f1030bd4..24a28fcbd874d 100644 --- a/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll +++ b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll @@ -53,10 +53,10 @@ declare void @use_cmp(<4 x i1>) define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @narrow_shuffle_of_select_use1( -; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> +; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> ; CHECK-NEXT: [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]] ; CHECK-NEXT: call void @use(<4 x i8> [[WIDESEL]]) -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <2 x i32> ; CHECK-NEXT: ret <2 x i8> [[R]] ; %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> @@ -70,10 +70,10 @@ define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @narrow_shuffle_of_select_use2( -; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> +; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> ; CHECK-NEXT: call void @use_cmp(<4 x i1> [[WIDECMP]]) ; CHECK-NEXT: [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <2 x i32> ; CHECK-NEXT: ret <2 x i8> [[R]] ; %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> @@ -87,9 +87,9 @@ define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) { ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types1( -; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> +; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> ; CHECK-NEXT: [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> @@ -102,9 +102,9 @@ define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8 define <3 x i8> @narrow_shuffle_of_select_mismatch_types2(<4 x i1> %cmp, <6 x i8> %x, <6 x i8> %y) { ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types2( -; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <6 x i32> +; CHECK-NEXT: [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <6 x i32> ; CHECK-NEXT: [[WIDESEL:%.*]] = select <6 x i1> [[WIDECMP]], <6 x i8> [[X:%.*]], <6 x i8> [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <6 x i8> [[WIDESEL]], <6 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <6 x i8> [[WIDESEL]], <6 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %widecmp = shufflevector <4 x i1> %cmp, <4 x i1> undef, <6 x i32> diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll index 233d9b008bc79..12bf09f8aeb76 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll @@ -1521,7 +1521,7 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) { define <4 x i32> @PR41419(<4 x i32> %v) { ; CHECK-LABEL: @PR41419( -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[S]] ; %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll index cedb882d0d228..f20077243273c 100644 --- a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll +++ b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll @@ -25,7 +25,7 @@ define void @test(<16 x i8> %w, ptr %o1, ptr %o2) { define <4 x i16> @splat_bitcast_operand(<8 x i8> %x) { ; CHECK-LABEL: @splat_bitcast_operand( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> poison, <8 x i32> ; CHECK-NEXT: [[S2:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16> ; CHECK-NEXT: ret <4 x i16> [[S2]] ; @@ -39,7 +39,7 @@ define <4 x i16> @splat_bitcast_operand(<8 x i8> %x) { define <4 x i16> @splat_bitcast_operand_uses(<8 x i8> %x) { ; CHECK-LABEL: @splat_bitcast_operand_uses( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> poison, <8 x i32> ; CHECK-NEXT: [[BC:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16> ; CHECK-NEXT: call void @use(<4 x i16> [[BC]]) ; CHECK-NEXT: [[S2:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16> @@ -83,9 +83,9 @@ define <4 x i32> @shuf_bitcast_operand(<16 x i8> %x) { define <5 x i16> @splat_bitcast_operand_change_type(<8 x i8> %x) { ; CHECK-LABEL: @splat_bitcast_operand_change_type( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> poison, <8 x i32> ; CHECK-NEXT: [[BC:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <4 x i16> [[BC]], <4 x i16> undef, <5 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <4 x i16> [[BC]], <4 x i16> poison, <5 x i32> ; CHECK-NEXT: ret <5 x i16> [[S2]] ; %s1 = shufflevector <8 x i8> %x, <8 x i8> undef, <8 x i32> @@ -98,7 +98,7 @@ define <5 x i16> @splat_bitcast_operand_change_type(<8 x i8> %x) { define <4 x i16> @splat_bitcast_operand_wider_src_elt(<2 x i32> %x) { ; CHECK-LABEL: @splat_bitcast_operand_wider_src_elt( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> ; CHECK-NEXT: [[S2:%.*]] = bitcast <2 x i32> [[S1]] to <4 x i16> ; CHECK-NEXT: ret <4 x i16> [[S2]] ; @@ -112,7 +112,7 @@ define <4 x i16> @splat_bitcast_operand_wider_src_elt(<2 x i32> %x) { define <4 x i16> @splat_bitcast_operand_wider_src_elt_uses(<2 x i32> %x) { ; CHECK-LABEL: @splat_bitcast_operand_wider_src_elt_uses( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> ; CHECK-NEXT: [[BC:%.*]] = bitcast <2 x i32> [[S1]] to <4 x i16> ; CHECK-NEXT: call void @use(<4 x i16> [[BC]]) ; CHECK-NEXT: [[S2:%.*]] = bitcast <2 x i32> [[S1]] to <4 x i16> @@ -142,9 +142,9 @@ define <16 x i8> @shuf_bitcast_operand_wider_src(<4 x i32> %x) { define <16 x i8> @shuf_bitcast_operand_cannot_widen(<4 x i32> %x) { ; CHECK-LABEL: @shuf_bitcast_operand_cannot_widen( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x i32> [[S1]] to <16 x i8> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i8> [[S2]] ; %s1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> @@ -157,9 +157,9 @@ define <16 x i8> @shuf_bitcast_operand_cannot_widen(<4 x i32> %x) { define <16 x i8> @shuf_bitcast_operand_cannot_widen_undef(<4 x i32> %x) { ; CHECK-LABEL: @shuf_bitcast_operand_cannot_widen_undef( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x i32> [[S1]] to <16 x i8> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i8> [[S2]] ; %s1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> @@ -212,7 +212,7 @@ define <2 x i4> @shuf_bitcast_insert_use2(<2 x i8> %v, i8 %x, ptr %p) { define <2 x i4> @shuf_bitcast_insert_wrong_index(<2 x i8> %v, i8 %x) { ; CHECK-LABEL: @shuf_bitcast_insert_wrong_index( ; CHECK-NEXT: [[B:%.*]] = bitcast <2 x i8> [[V:%.*]] to <4 x i4> -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i4> [[B]], <4 x i4> undef, <2 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i4> [[B]], <4 x i4> poison, <2 x i32> ; CHECK-NEXT: ret <2 x i4> [[R]] ; %i = insertelement <2 x i8> %v, i8 %x, i32 1 @@ -227,7 +227,7 @@ define <3 x i4> @shuf_bitcast_wrong_size(<2 x i8> %v, i8 %x) { ; CHECK-LABEL: @shuf_bitcast_wrong_size( ; CHECK-NEXT: [[I:%.*]] = insertelement <2 x i8> [[V:%.*]], i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[B:%.*]] = bitcast <2 x i8> [[I]] to <4 x i4> -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i4> [[B]], <4 x i4> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i4> [[B]], <4 x i4> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i4> [[R]] ; %i = insertelement <2 x i8> %v, i8 %x, i32 0 diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll index 12d81e5d1944b..457300a25e769 100644 --- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll +++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll @@ -28,7 +28,7 @@ define <2 x i16> @test_srem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_srem( ; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], -; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x i16> [[T2]] ; @@ -43,7 +43,7 @@ define <2 x i16> @test_urem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_urem( ; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], -; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x i16> [[T2]] ; @@ -58,7 +58,7 @@ define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_sdiv( ; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], -; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x i16> [[T2]] ; @@ -73,7 +73,7 @@ define <2 x i16> @test_udiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_udiv( ; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], -; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x i16> [[T2]] ; @@ -88,7 +88,7 @@ define <2 x i16> @test_udiv(i16 %a, i1 %cmp) { ; shufflevector is eliminated here. define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) { ; CHECK-LABEL: @test_fdiv( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A:%.*]], i64 1 ; CHECK-NEXT: [[SPLAT_OP:%.*]] = fdiv <2 x float> [[TMP1]], ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> , <2 x float> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x float> [[T2]] @@ -105,7 +105,7 @@ define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) { ; shufflevector is eliminated here. define <2 x float> @test_frem(float %a, float %b, i1 %cmp) { ; CHECK-LABEL: @test_frem( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A:%.*]], i64 1 ; CHECK-NEXT: [[SPLAT_OP:%.*]] = frem <2 x float> [[TMP1]], ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> , <2 x float> [[SPLAT_OP]] ; CHECK-NEXT: ret <2 x float> [[T2]] diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll index 66105f4add181..790d551e5b1de 100644 --- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll +++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll @@ -831,7 +831,7 @@ define i8 @negate_zext_wrongwidth(i8 %x, i2 %y) { define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @negate_shufflevector_oneinput_reverse( ; CHECK-NEXT: [[T0_NEG:%.*]] = shl <2 x i4> , [[X:%.*]] -; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> undef, <2 x i32> +; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i4> [[T2]] ; @@ -843,7 +843,7 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef( ; CHECK-NEXT: [[T0_NEG:%.*]] = shl <2 x i4> , [[X:%.*]] -; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> undef, <2 x i32> +; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i4> [[T2]] ; @@ -869,7 +869,7 @@ define <2 x i4> @negate_shufflevector_twoinputs(<2 x i4> %x, <2 x i4> %y, <2 x i define <2 x i4> @negate_shufflevector_oneinput_extrause(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @negate_shufflevector_oneinput_extrause( ; CHECK-NEXT: [[T0:%.*]] = shl <2 x i4> , [[X:%.*]] -; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x i4> [[T0]], <2 x i4> undef, <2 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x i4> [[T0]], <2 x i4> poison, <2 x i32> ; CHECK-NEXT: call void @use_v2i4(<2 x i4> [[T1]]) ; CHECK-NEXT: [[T2:%.*]] = sub <2 x i4> [[Y:%.*]], [[T1]] ; CHECK-NEXT: ret <2 x i4> [[T2]] diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll index 6682e7f3a0d9d..e9e105b91f3c1 100644 --- a/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll @@ -182,7 +182,7 @@ define <4 x i64> @PR45314(<4 x i64> %x) { ; ; BE-LABEL: @PR45314( ; BE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> +; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> ; BE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64> ; BE-NEXT: ret <4 x i64> [[B]] ; diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll index 10fd7e5dc0e3d..5e62ca9cd591d 100644 --- a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll +++ b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll @@ -182,7 +182,7 @@ define <4 x i64> @PR45314(<4 x i64> %x) { ; ; BE-LABEL: @PR45314( ; BE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> +; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> ; BE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64> ; BE-NEXT: ret <4 x i64> [[B]] ; diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll index adcad30036818..c6bc06d666d0a 100644 --- a/llvm/test/Transforms/InstCombine/trunc.ll +++ b/llvm/test/Transforms/InstCombine/trunc.ll @@ -950,7 +950,7 @@ define <3 x i31> @wide_splat2(<3 x i33> %x) { define <3 x i31> @wide_splat3(<3 x i33> %x) { ; CHECK-LABEL: @wide_splat3( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> undef, <3 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> poison, <3 x i32> ; CHECK-NEXT: [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31> ; CHECK-NEXT: ret <3 x i31> [[TRUNC]] ; diff --git a/llvm/test/Transforms/InstCombine/type_pun.ll b/llvm/test/Transforms/InstCombine/type_pun.ll index a439f77902365..d99ac20d64e21 100644 --- a/llvm/test/Transforms/InstCombine/type_pun.ll +++ b/llvm/test/Transforms/InstCombine/type_pun.ll @@ -145,7 +145,7 @@ tail: ; should stay the same. define i40 @type_pun_unhandled(<16 x i8> %in) { ; CHECK-LABEL: @type_pun_unhandled( -; CHECK-NEXT: [[SROA:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> undef, <5 x i32> +; CHECK-NEXT: [[SROA:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> poison, <5 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <5 x i8> [[SROA]] to i40 ; CHECK-NEXT: ret i40 [[TMP1]] ; diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll index 717645c129e31..576af85b5ee27 100644 --- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll +++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll @@ -175,10 +175,10 @@ define <4 x float> @inselt_shuf_no_demand_commute(float %a1, float %a2, float %a define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) { ; CHECK-LABEL: @inselt_shuf_no_demand_multiuse( -; CHECK-NEXT: [[OUT0:%.*]] = insertelement <4 x i32> undef, i32 [[A0:%.*]], i64 0 +; CHECK-NEXT: [[OUT0:%.*]] = insertelement <4 x i32> poison, i32 [[A0:%.*]], i64 0 ; CHECK-NEXT: [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i64 1 ; CHECK-NEXT: [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]] -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[SHUFFLE]] ; %out0 = insertelement <4 x i32> undef, i32 %a0, i32 0 @@ -193,7 +193,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) { ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain( ; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]] -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SHUFFLE]] ; %out1 = insertelement <4 x float> undef, float %a1, i32 1 @@ -208,7 +208,7 @@ define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, define <3 x i8> @shuf_add(<3 x i8> %x) { ; CHECK-LABEL: @shuf_add( ; CHECK-NEXT: [[BO:%.*]] = add nsw <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = add nsw <3 x i8> %x, @@ -219,7 +219,7 @@ define <3 x i8> @shuf_add(<3 x i8> %x) { define <3 x i8> @shuf_sub(<3 x i8> %x) { ; CHECK-LABEL: @shuf_sub( ; CHECK-NEXT: [[BO:%.*]] = sub nuw <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = sub nuw <3 x i8> , %x @@ -230,7 +230,7 @@ define <3 x i8> @shuf_sub(<3 x i8> %x) { define <3 x i8> @shuf_mul(<3 x i8> %x) { ; CHECK-LABEL: @shuf_mul( ; CHECK-NEXT: [[BO:%.*]] = mul nsw <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = mul nsw <3 x i8> %x, @@ -241,7 +241,7 @@ define <3 x i8> @shuf_mul(<3 x i8> %x) { define <3 x i8> @shuf_and(<3 x i8> %x) { ; CHECK-LABEL: @shuf_and( ; CHECK-NEXT: [[BO:%.*]] = and <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = and <3 x i8> %x, @@ -252,7 +252,7 @@ define <3 x i8> @shuf_and(<3 x i8> %x) { define <3 x i8> @shuf_or(<3 x i8> %x) { ; CHECK-LABEL: @shuf_or( ; CHECK-NEXT: [[BO:%.*]] = or <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = or <3 x i8> %x, @@ -263,7 +263,7 @@ define <3 x i8> @shuf_or(<3 x i8> %x) { define <3 x i8> @shuf_xor(<3 x i8> %x) { ; CHECK-LABEL: @shuf_xor( ; CHECK-NEXT: [[BO:%.*]] = xor <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = xor <3 x i8> %x, @@ -274,7 +274,7 @@ define <3 x i8> @shuf_xor(<3 x i8> %x) { define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_lshr_const_op0( ; CHECK-NEXT: [[BO:%.*]] = lshr <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = lshr <3 x i8> , %x @@ -285,7 +285,7 @@ define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_lshr_const_op1( ; CHECK-NEXT: [[BO:%.*]] = lshr exact <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = lshr exact <3 x i8> %x, @@ -296,7 +296,7 @@ define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_ashr_const_op0( ; CHECK-NEXT: [[BO:%.*]] = lshr <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = ashr <3 x i8> , %x @@ -307,7 +307,7 @@ define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_ashr_const_op1( ; CHECK-NEXT: [[BO:%.*]] = ashr exact <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = ashr exact <3 x i8> %x, @@ -318,7 +318,7 @@ define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_shl_const_op0( ; CHECK-NEXT: [[BO:%.*]] = shl nsw <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = shl nsw <3 x i8> , %x @@ -329,7 +329,7 @@ define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_shl_const_op1( ; CHECK-NEXT: [[BO:%.*]] = shl nuw <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = shl nuw <3 x i8> %x, @@ -340,7 +340,7 @@ define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_sdiv_const_op0( ; CHECK-NEXT: [[BO:%.*]] = sdiv exact <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = sdiv exact <3 x i8> , %x @@ -351,7 +351,7 @@ define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_sdiv_const_op1( ; CHECK-NEXT: [[BO:%.*]] = sdiv <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = sdiv <3 x i8> %x, @@ -362,7 +362,7 @@ define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_srem_const_op0( ; CHECK-NEXT: [[BO:%.*]] = srem <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = srem <3 x i8> , %x @@ -373,7 +373,7 @@ define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_srem_const_op1( ; CHECK-NEXT: [[BO:%.*]] = srem <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = srem <3 x i8> %x, @@ -384,7 +384,7 @@ define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_udiv_const_op0( ; CHECK-NEXT: [[BO:%.*]] = udiv exact <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = udiv exact <3 x i8> , %x @@ -395,7 +395,7 @@ define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_udiv_const_op1( ; CHECK-NEXT: [[BO:%.*]] = udiv <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = udiv <3 x i8> %x, @@ -406,7 +406,7 @@ define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) { define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) { ; CHECK-LABEL: @shuf_urem_const_op0( ; CHECK-NEXT: [[BO:%.*]] = urem <3 x i8> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = urem <3 x i8> , %x @@ -417,7 +417,7 @@ define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) { define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) { ; CHECK-LABEL: @shuf_urem_const_op1( ; CHECK-NEXT: [[BO:%.*]] = urem <3 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i8> [[R]] ; %bo = urem <3 x i8> %x, @@ -428,7 +428,7 @@ define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) { define <3 x float> @shuf_fadd(<3 x float> %x) { ; CHECK-LABEL: @shuf_fadd( ; CHECK-NEXT: [[BO:%.*]] = fadd <3 x float> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = fadd <3 x float> %x, @@ -439,7 +439,7 @@ define <3 x float> @shuf_fadd(<3 x float> %x) { define <3 x float> @shuf_fsub(<3 x float> %x) { ; CHECK-LABEL: @shuf_fsub( ; CHECK-NEXT: [[BO:%.*]] = fsub fast <3 x float> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = fsub fast <3 x float> , %x @@ -450,7 +450,7 @@ define <3 x float> @shuf_fsub(<3 x float> %x) { define <3 x float> @shuf_fmul(<3 x float> %x) { ; CHECK-LABEL: @shuf_fmul( ; CHECK-NEXT: [[BO:%.*]] = fmul reassoc <3 x float> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = fmul reassoc <3 x float> %x, @@ -461,7 +461,7 @@ define <3 x float> @shuf_fmul(<3 x float> %x) { define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) { ; CHECK-LABEL: @shuf_fdiv_const_op0( ; CHECK-NEXT: [[BO:%.*]] = fdiv reassoc ninf <3 x float> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = fdiv ninf reassoc <3 x float> , %x @@ -472,7 +472,7 @@ define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) { define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) { ; CHECK-LABEL: @shuf_fdiv_const_op1( ; CHECK-NEXT: [[BO:%.*]] = fdiv nnan ninf <3 x float> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = fdiv ninf nnan <3 x float> %x, @@ -483,7 +483,7 @@ define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) { define <3 x float> @shuf_frem_const_op0(<3 x float> %x) { ; CHECK-LABEL: @shuf_frem_const_op0( ; CHECK-NEXT: [[BO:%.*]] = frem nnan <3 x float> , [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = frem nnan <3 x float> , %x @@ -494,7 +494,7 @@ define <3 x float> @shuf_frem_const_op0(<3 x float> %x) { define <3 x float> @shuf_frem_const_op1(<3 x float> %x) { ; CHECK-LABEL: @shuf_frem_const_op1( ; CHECK-NEXT: [[BO:%.*]] = frem reassoc ninf <3 x float> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> ; CHECK-NEXT: ret <3 x float> [[R]] ; %bo = frem ninf reassoc <3 x float> %x, @@ -688,7 +688,7 @@ define <4 x i8> @select_cond_with_eq_true_false_elts(<4 x i8> %x, <4 x i8> %y, < define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) { ; CHECK-LABEL: @select_cond_with_eq_true_false_elts2( -; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[Y:%.*]], <4 x i8> [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[SEL]], <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[R]] @@ -706,7 +706,7 @@ define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x fl ; CHECK-LABEL: @select_cond_with_eq_true_false_elts3( ; CHECK-NEXT: [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> ; CHECK-NEXT: [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> -; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[TVAL]], <4 x float> [[FVAL]] ; CHECK-NEXT: ret <4 x float> [[R]] ; @@ -720,7 +720,7 @@ define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x fl define <4 x i8> @select_cond_with_undef_true_false_elts(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) { ; CHECK-LABEL: @select_cond_with_undef_true_false_elts( ; CHECK-NEXT: [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> -; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> +; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[TVAL]], <4 x i8> [[X:%.*]] ; CHECK-NEXT: ret <4 x i8> [[R]] ; @@ -798,7 +798,7 @@ define <4 x float> @ins_of_ext_wrong_demand(<4 x float> %x, float %y) { define <4 x float> @ins_of_ext_wrong_type(<5 x float> %x, float %y) { ; CHECK-LABEL: @ins_of_ext_wrong_type( ; CHECK-NEXT: [[E0:%.*]] = extractelement <5 x float> [[X:%.*]], i64 0 -; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> undef, float [[E0]], i64 0 +; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0 ; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[Y:%.*]], i64 1 ; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i64 2 ; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[Y]], i64 3 diff --git a/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll index 79c3d37cd53c3..c43def83f58ac 100644 --- a/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll @@ -84,7 +84,7 @@ define void @nocopy(i64 %val, i32 %limit, ptr %ptr) { ; CHECK-LABEL: @nocopy( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[VAL:%.*]] to i32 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i32> undef, i32 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i32> poison, i32 [[TMP0]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <16 x i32> zeroinitializer ; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i32> [[TMP2]], ; CHECK-NEXT: br label [[LOOP:%.*]] diff --git a/llvm/test/Transforms/InstCombine/vec_phi_extract.ll b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll index 1bdc21724be51..84657c9d43896 100644 --- a/llvm/test/Transforms/InstCombine/vec_phi_extract.ll +++ b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll @@ -84,7 +84,7 @@ define void @nocopy(i64 %val, i32 %limit, ptr %ptr) { ; CHECK-LABEL: @nocopy( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[VAL:%.*]] to i32 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i32> undef, i32 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i32> poison, i32 [[TMP0]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <16 x i32> zeroinitializer ; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i32> [[TMP2]], ; CHECK-NEXT: br label [[LOOP:%.*]] diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index ab27d75e0b783..bef201196e615 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -81,7 +81,7 @@ define float @testvscale6( %X) { define <4 x float> @test7(<4 x float> %x) { ; CHECK-LABEL: @test7( -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[R]] ; %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 > @@ -108,8 +108,7 @@ define <4 x float> @test8(<4 x float> %x, <4 x float> %y) { ; different length then the second. define <4 x i8> @test9(<16 x i8> %t6) { ; CHECK-LABEL: @test9( -; CHECK-NEXT: [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> -; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T9:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[T9]] ; %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 13, i32 9, i32 4, i32 13 > @@ -123,8 +122,8 @@ define <4 x i8> @test9(<16 x i8> %t6) { define <4 x i8> @test9a(<16 x i8> %t6) { ; CHECK-LABEL: @test9a( -; CHECK-NEXT: [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> -; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <4 x i32> +; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[T9]] ; %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 undef, i32 9, i32 4, i32 8 > @@ -136,8 +135,7 @@ define <4 x i8> @test9a(<16 x i8> %t6) { ; different length then the second. define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) { ; CHECK-LABEL: @test9b( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <8 x i32> -; CHECK-NEXT: [[T9:%.*]] = shufflevector <8 x i8> [[T1]], <8 x i8> undef, <4 x i32> +; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[T9]] ; %t1 = shufflevector <4 x i8> %t6, <4 x i8> %t7, <8 x i32> @@ -148,7 +146,7 @@ define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) { ; Redundant vector splats should be removed. Radar 8597790. define <4 x i32> @test10(<4 x i32> %t5) { ; CHECK-LABEL: @test10( -; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[T7]] ; %t6 = shufflevector <4 x i32> %t5, <4 x i32> undef, <4 x i32> @@ -160,9 +158,7 @@ define <4 x i32> @test10(<4 x i32> %t5) { define <8 x i8> @test11(<16 x i8> %t6) { ; CHECK-LABEL: @test11( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> -; CHECK-NEXT: [[T2:%.*]] = shufflevector <16 x i8> [[T6]], <16 x i8> undef, <4 x i32> -; CHECK-NEXT: [[T3:%.*]] = shufflevector <4 x i8> [[T1]], <4 x i8> [[T2]], <8 x i32> +; CHECK-NEXT: [[T3:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i8> [[T3]] ; %t1 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> @@ -228,7 +224,7 @@ define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y ; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use( ; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> ; CHECK-NEXT: call void @use_v5i8(<5 x i8> [[SHUF]]) -; CHECK-NEXT: [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> +; CHECK-NEXT: [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i8> [[EXTRACT_SUBV]] ; %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> @@ -239,7 +235,7 @@ define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y define <2 x i8> @test13a(i8 %x1, i8 %x2) { ; CHECK-LABEL: @test13a( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X2:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[X2:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X1:%.*]], i64 1 ; CHECK-NEXT: [[D:%.*]] = add <2 x i8> [[TMP2]], ; CHECK-NEXT: ret <2 x i8> [[D]] @@ -255,10 +251,10 @@ define <2 x i8> @test13a(i8 %x1, i8 %x2) { define <3 x i32> @add_wider(i32 %y, i32 %z) { ; CHECK-LABEL: @add_wider( -; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i64 0 +; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 ; CHECK-NEXT: [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1 ; CHECK-NEXT: [[A:%.*]] = add <2 x i32> [[I1]], -; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> +; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[EXT]] ; %i0 = insertelement <2 x i32> undef, i32 %y, i32 0 @@ -272,10 +268,10 @@ define <3 x i32> @add_wider(i32 %y, i32 %z) { define <3 x i32> @div_wider(i32 %y, i32 %z) { ; CHECK-LABEL: @div_wider( -; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i64 0 +; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 ; CHECK-NEXT: [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1 ; CHECK-NEXT: [[A:%.*]] = sdiv <2 x i32> [[I1]], -; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> +; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> ; CHECK-NEXT: ret <3 x i32> [[EXT]] ; %i0 = insertelement <2 x i32> undef, i32 %y, i32 0 @@ -289,7 +285,7 @@ define <3 x i32> @div_wider(i32 %y, i32 %z) { define <3 x i8> @fold_inselts_with_widening_shuffle(i8 %x, i8 %y) { ; CHECK-LABEL: @fold_inselts_with_widening_shuffle( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i8> poison, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[WIDEN:%.*]] = insertelement <3 x i8> [[TMP1]], i8 [[Y:%.*]], i64 1 ; CHECK-NEXT: ret <3 x i8> [[WIDEN]] ; @@ -301,7 +297,7 @@ define <3 x i8> @fold_inselts_with_widening_shuffle(i8 %x, i8 %y) { define <2 x i8> @test13b(i8 %x) { ; CHECK-LABEL: @test13b( -; CHECK-NEXT: [[B:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[B:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: ret <2 x i8> [[B]] ; %A = insertelement <2 x i8> undef, i8 %x, i32 0 @@ -311,7 +307,7 @@ define <2 x i8> @test13b(i8 %x) { define <2 x i8> @test13c(i8 %x1, i8 %x2) { ; CHECK-LABEL: @test13c( -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[X1:%.*]], i64 0 ; CHECK-NEXT: [[C:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i64 1 ; CHECK-NEXT: ret <2 x i8> [[C]] ; @@ -419,7 +415,7 @@ declare void @use4(<4 x float>) define <2 x float> @shuffle_fadd_multiuse(<2 x float> %v1, <2 x float> %v2) { ; CHECK-LABEL: @shuffle_fadd_multiuse( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[V1]], [[V2:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: call void @use(<2 x float> [[T1]]) @@ -434,7 +430,7 @@ define <2 x float> @shuffle_fadd_multiuse(<2 x float> %v1, <2 x float> %v2) { define <2 x float> @shuffle_fdiv_multiuse(<2 x float> %v1, <2 x float> %v2) { ; CHECK-LABEL: @shuffle_fdiv_multiuse( -; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = fdiv <2 x float> [[V1:%.*]], [[V2]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: call void @use(<2 x float> [[T2]]) @@ -451,8 +447,8 @@ define <2 x float> @shuffle_fdiv_multiuse(<2 x float> %v1, <2 x float> %v2) { define <2 x float> @shuffle_fsub_multiuse(<2 x float> %v1, <2 x float> %v2) { ; CHECK-LABEL: @shuffle_fsub_multiuse( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> -; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> poison, <2 x i32> +; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[R:%.*]] = fsub <2 x float> [[T1]], [[T2]] ; CHECK-NEXT: call void @use(<2 x float> [[T1]]) ; CHECK-NEXT: call void @use(<2 x float> [[T2]]) @@ -540,7 +536,7 @@ define <4 x i32> @sub_const(<4 x i32> %v) { define <2 x float> @fadd_const_multiuse(<2 x float> %v) { ; CHECK-LABEL: @fadd_const_multiuse( -; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[T1]], ; CHECK-NEXT: call void @use(<2 x float> [[T1]]) ; CHECK-NEXT: ret <2 x float> [[R]] @@ -620,7 +616,7 @@ define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) { define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) { ; CHECK-LABEL: @widening_shuffle_add_invalid_constant( -; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = add <4 x i8> [[WIDEX]], ; CHECK-NEXT: ret <4 x i8> [[R]] ; @@ -633,7 +629,7 @@ define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) { define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) { ; CHECK-LABEL: @widening_shuffle_add_invalid_mask( -; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> +; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> ; CHECK-NEXT: [[R:%.*]] = add <4 x i8> [[WIDEX]], ; CHECK-NEXT: ret <4 x i8> [[R]] ; @@ -675,7 +671,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) { define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: [[BO:%.*]] = shl <4 x i16> [[SHUF]], ; CHECK-NEXT: ret <4 x i16> [[BO]] ; @@ -689,7 +685,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { define <4 x i16> @widening_shuffle_or(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_or( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: [[BO:%.*]] = or <4 x i16> [[SHUF]], ; CHECK-NEXT: ret <4 x i16> [[BO]] ; @@ -749,8 +745,8 @@ define <4 x i16> @pr19717a(<8 x i16> %in0, <8 x i16> %in1) { define <8 x i8> @pr19730(<16 x i8> %in0) { ; CHECK-LABEL: @pr19730( -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[IN0:%.*]], <16 x i8> undef, <8 x i32> -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <8 x i8> [[SHUFFLE]], <8 x i8> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[IN0:%.*]], <16 x i8> poison, <8 x i32> +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <8 x i8> [[SHUFFLE]], <8 x i8> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i8> [[SHUFFLE1]] ; %shuffle = shufflevector <16 x i8> %in0, <16 x i8> undef, <8 x i32> @@ -789,7 +785,7 @@ define <4 x i32> @pr20059(<4 x i32> %p1, <4 x i32> %p2) { define <4 x i32> @pr20114(<4 x i32> %__mask) { ; CHECK-LABEL: @pr20114( -; CHECK-NEXT: [[MASK01_I:%.*]] = shufflevector <4 x i32> [[__MASK:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[MASK01_I:%.*]] = shufflevector <4 x i32> [[__MASK:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[MASKED_NEW_I_I_I:%.*]] = and <4 x i32> [[MASK01_I]], bitcast (<2 x i64> to <4 x i32>) ; CHECK-NEXT: ret <4 x i32> [[MASKED_NEW_I_I_I]] ; @@ -800,7 +796,7 @@ define <4 x i32> @pr20114(<4 x i32> %__mask) { define <2 x ptr> @pr23113(<4 x ptr> %A) { ; CHECK-LABEL: @pr23113( -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[A:%.*]], <4 x ptr> undef, <2 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[A:%.*]], <4 x ptr> poison, <2 x i32> ; CHECK-NEXT: ret <2 x ptr> [[TMP1]] ; %1 = shufflevector <4 x ptr> %A, <4 x ptr> undef, <2 x i32> @@ -1035,7 +1031,7 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) { define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) { ; CHECK-LABEL: @and_constant_mask_undef( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], ; CHECK-NEXT: ret <4 x i16> [[AND]] ; @@ -1050,7 +1046,7 @@ entry: define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) { ; CHECK-LABEL: @and_constant_mask_undef_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], ; CHECK-NEXT: ret <4 x i16> [[AND]] ; @@ -1104,7 +1100,7 @@ entry: define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) { ; CHECK-LABEL: @or_constant_mask_undef( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], ; CHECK-NEXT: ret <4 x i16> [[OR]] ; @@ -1119,7 +1115,7 @@ entry: define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) { ; CHECK-LABEL: @or_constant_mask_undef_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], ; CHECK-NEXT: ret <4 x i16> [[OR]] ; @@ -1171,7 +1167,7 @@ entry: define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) { ; CHECK-LABEL: @shl_constant_mask_undef( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], ; CHECK-NEXT: ret <4 x i16> [[SHL]] ; @@ -1357,7 +1353,7 @@ define <2 x float> @frem_splat_constant1(<2 x float> %x) { define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) { ; CHECK-LABEL: @PR40734( ; CHECK-NEXT: [[WIDEN:%.*]] = shufflevector <1 x i1> zeroinitializer, <1 x i1> [[X:%.*]], <2 x i32> -; CHECK-NEXT: [[NARROW:%.*]] = shufflevector <4 x i1> [[Y:%.*]], <4 x i1> undef, <2 x i32> +; CHECK-NEXT: [[NARROW:%.*]] = shufflevector <4 x i1> [[Y:%.*]], <4 x i1> poison, <2 x i32> ; CHECK-NEXT: [[R:%.*]] = and <2 x i1> [[WIDEN]], [[NARROW]] ; CHECK-NEXT: ret <2 x i1> [[R]] ; @@ -1371,8 +1367,8 @@ define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) { define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) { ; CHECK-LABEL: @insert_subvector_shuffles( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> ; CHECK-NEXT: [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> ; CHECK-NEXT: ret <7 x i8> [[S3]] ; @@ -1398,8 +1394,8 @@ define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) { define <2 x i8> @insert_subvector_shuffles_narrowing(<3 x i8> %x, <3 x i8> %y) { ; CHECK-LABEL: @insert_subvector_shuffles_narrowing( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> ; CHECK-NEXT: [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <2 x i32> ; CHECK-NEXT: ret <2 x i8> [[S3]] ; @@ -1437,8 +1433,8 @@ define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) { define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) { ; CHECK-LABEL: @not_insert_subvector_shuffle( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> undef, <4 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> poison, <4 x i32> ; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> ; CHECK-NEXT: ret <4 x double> [[S3]] ; @@ -1452,8 +1448,8 @@ define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) { define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %x, <3 x double> %y) { ; CHECK-LABEL: @not_insert_subvector_shuffles_with_same_size( -; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> -; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> undef, <4 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> +; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> poison, <4 x i32> ; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> ; CHECK-NEXT: ret <4 x double> [[S3]] ; @@ -1468,7 +1464,7 @@ define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> % define <4 x float> @insert_subvector_crash_invalid_mask_elt(<2 x float> %x, ptr %p) { ; CHECK-LABEL: @insert_subvector_crash_invalid_mask_elt( -; CHECK-NEXT: [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <4 x i32> +; CHECK-NEXT: [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <4 x i32> ; CHECK-NEXT: [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> poison, <4 x i32> ; CHECK-NEXT: store <4 x float> [[I]], ptr [[P:%.*]], align 16 ; CHECK-NEXT: ret <4 x float> [[WIDEN]] @@ -1565,7 +1561,7 @@ define <4 x i32> @splat_assoc_add_undef_constant_elt_at_splat_index(<4 x i32> %x define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_undef_mask_elts_undef_constant_elts( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] @@ -1578,7 +1574,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32> define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] @@ -1591,7 +1587,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_e define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], ; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]] ; CHECK-NEXT: ret <4 x i32> [[R]] @@ -1649,8 +1645,8 @@ define <3 x i8> @splat_assoc_mul_undef_elt1(<3 x i8> %x, <3 x i8> %y, <3 x i8> % define <3 x i8> @splat_assoc_mul_undef_elt2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) { ; CHECK-LABEL: @splat_assoc_mul_undef_elt2( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> -; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> +; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]] ; CHECK-NEXT: ret <3 x i8> [[R]] @@ -1678,8 +1674,8 @@ define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index1(<3 x i8> %x, <3 x i8> define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) { ; CHECK-LABEL: @splat_assoc_mul_undef_elt_at_splat_index2( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> -; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> +; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]] ; CHECK-NEXT: ret <3 x i8> [[R]] @@ -1695,8 +1691,8 @@ define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index2(<3 x i8> %x, <3 x i8> define <3 x i8> @splat_assoc_or(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) { ; CHECK-LABEL: @splat_assoc_or( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> -; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> +; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> ; CHECK-NEXT: [[A:%.*]] = or <3 x i8> [[SPLATZ]], [[Y:%.*]] ; CHECK-NEXT: [[R:%.*]] = or <3 x i8> [[A]], [[SPLATX]] ; CHECK-NEXT: ret <3 x i8> [[R]] @@ -1727,7 +1723,7 @@ define <2 x float> @splat_assoc_fdiv(<2 x float> %x, <2 x float> %y) { define <2 x float> @splat_assoc_fadd(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @splat_assoc_fadd( -; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[A:%.*]] = fadd fast <2 x float> [[Y:%.*]], ; CHECK-NEXT: call void @use(<2 x float> [[A]]) ; CHECK-NEXT: [[R:%.*]] = fadd fast <2 x float> [[A]], [[SPLATX]] @@ -1790,7 +1786,7 @@ define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) { define <4 x i32> @PR46872(<4 x i32> %x) { ; CHECK-LABEL: @PR46872( -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[A:%.*]] = and <4 x i32> [[S]], bitcast (<2 x i64> to <4 x i32>) ; CHECK-NEXT: ret <4 x i32> [[A]] ; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll index 0342d6d186d55..bd0b7a8412b94 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll @@ -161,9 +161,9 @@ define void @test_shrink_zext_in_preheader(ptr noalias %src, ptr noalias %dst, i ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[A]] to i16 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> undef, i16 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i16> undef, i16 [[B]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i16> poison, i16 [[B]], i64 0 ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -188,8 +188,8 @@ define void @test_shrink_zext_in_preheader(ptr noalias %src, ptr noalias %dst, i ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] ; CHECK: vec.epilog.ph: ; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[A]] to i16 -; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> undef, i16 [[TMP15]], i64 0 -; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i16> undef, i16 [[B]], i64 0 +; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> poison, i16 [[TMP15]], i64 0 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i16> poison, i16 [[B]], i64 0 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX7:%.*]] = phi i32 [ 992, [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] @@ -243,7 +243,7 @@ define void @test_shrink_select(ptr noalias %src, ptr noalias %dst, i32 %A, i1 % ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[A]] to i16 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> undef, i16 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -264,7 +264,7 @@ define void @test_shrink_select(ptr noalias %src, ptr noalias %dst, i32 %A, i1 % ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] ; CHECK: vec.epilog.ph: ; CHECK-NEXT: [[TMP10:%.*]] = trunc i32 [[A]] to i16 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x i16> undef, i16 [[TMP10]], i64 0 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x i16> poison, i16 [[TMP10]], i64 0 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX3:%.*]] = phi i32 [ 992, [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] @@ -320,7 +320,7 @@ define void @trunc_invariant_sdiv_result(i32 %a, i32 %b, ptr noalias %src, ptr % ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[INVAR_DIV]] to i16 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> undef, i16 [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -340,7 +340,7 @@ define void @trunc_invariant_sdiv_result(i32 %a, i32 %b, ptr noalias %src, ptr % ; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] ; CHECK: vec.epilog.ph: ; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[INVAR_DIV]] to i16 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i16> undef, i16 [[TMP8]], i64 0 +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i16> poison, i16 [[TMP8]], i64 0 ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll index d48f656089be1..2b5ee9a76047f 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll @@ -236,13 +236,13 @@ define void @multiply_dont_hoist_phi(ptr noalias %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B:%.*]], i64 2 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 -; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]] ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP0]]) ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B]], align 8 -; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP2]]) @@ -280,11 +280,11 @@ define void @multiply_dont_hoist_cast_due_to_operand(ptr noalias %A, ptr %B, ptr ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]]) ; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]] -; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]]) ; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8 ; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8 @@ -313,11 +313,11 @@ define void @multiply_dont_hoist_load(ptr noalias %A, ptr %B, ptr %C.ptr) { ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]]) ; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]] -; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]]) ; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8 ; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8 @@ -346,11 +346,11 @@ define void @multiply_dont_hoist_call(ptr noalias %A, ptr %B) { ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]]) ; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]] -; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]]) ; CHECK-NEXT: [[C:%.*]] = call ptr @get_address() ; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8 diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll index 0bfe0152138f4..e0dbb21cac08b 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll @@ -40,11 +40,11 @@ define void @multiply_noalias_4x4(ptr noalias %A, ptr noalias %B, ptr noalias %C ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[RESULT_VEC_0]]) -; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP7]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP6]]) ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[RESULT_VEC_1]]) -; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP9]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP8]]) ; CHECK-NEXT: br label [[INNER_LATCH]] ; CHECK: inner.latch: @@ -117,13 +117,13 @@ define void @multiply_noalias_2x4(ptr noalias %A, ptr noalias %B, ptr noalias %C ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]] ; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP6]] -; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> ; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]] ; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[TMP7]], [[TMP8]] ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]] ; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP10]] -; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> ; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]] ; CHECK-NEXT: [[TMP13]] = add <2 x i64> [[TMP11]], [[TMP12]] ; CHECK-NEXT: br label [[INNER_LATCH]] @@ -203,13 +203,13 @@ define void @multiply_noalias_4x2_2x8(ptr noalias %A, ptr noalias %B, ptr noalia ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]] ; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP6]] -; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> ; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]] ; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[TMP7]], [[TMP8]] ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]] ; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP10]] -; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> ; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]] ; CHECK-NEXT: [[TMP13]] = add <2 x i64> [[TMP11]], [[TMP12]] ; CHECK-NEXT: br label [[INNER_LATCH]] @@ -319,11 +319,11 @@ define void @multiply_alias_2x2(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x float>, ptr [[VEC_GEP10]], align 4 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[RESULT_VEC_0]]) -; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[TMP15]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT15]], <2 x float> [[TMP14]]) ; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP16:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[RESULT_VEC_1]]) -; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> ; CHECK-NEXT: [[TMP17]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT22]], <2 x float> [[TMP16]]) ; CHECK-NEXT: br label [[INNER_LATCH]] ; CHECK: inner.latch: diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll index 7671fecef7b64..f7ac2d321439a 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll @@ -55,11 +55,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]]) ; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]] -; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]]) ; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[C]], align 8 ; CHECK-NEXT: [[VEC_GEP21:%.*]] = getelementptr double, ptr [[C]], i64 3 @@ -73,11 +73,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <2 x double>, ptr [[VEC_GEP26]], align 8 ; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT29]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> [[TMP13]]) ; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT35]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP16:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> [[TMP15]]) ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[C]], i64 2 ; CHECK-NEXT: store <1 x double> [[TMP14]], ptr [[TMP17]], align 8 @@ -90,7 +90,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[TMP18]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD41]], [[SPLAT_SPLAT47]] -; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD43]], <2 x double> [[SPLAT_SPLAT50]], <2 x double> [[TMP19]]) ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 6 ; CHECK-NEXT: store <2 x double> [[TMP20]], ptr [[TMP21]], align 8 @@ -102,7 +102,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD54:%.*]] = load <2 x double>, ptr [[TMP23]], align 8 ; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD51]], [[SPLAT_SPLATINSERT56]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD53]], <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> [[TMP24]]) ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 8 ; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8 @@ -163,11 +163,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD80:%.*]] = load <2 x double>, ptr [[VEC_GEP79]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP40:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT83]] -; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT86]], <2 x double> [[TMP40]]) ; CHECK-NEXT: [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP42:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT89]] -; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP42]]) ; CHECK-NEXT: store <2 x double> [[TMP41]], ptr [[C]], align 8 ; CHECK-NEXT: [[VEC_GEP93:%.*]] = getelementptr double, ptr [[C]], i64 3 @@ -181,11 +181,11 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[VEC_GEP98]], align 8 ; CHECK-NEXT: [[SPLAT_SPLATINSERT101:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP45:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT101]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT104:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT104:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP46:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT104]], <1 x double> [[TMP45]]) ; CHECK-NEXT: [[SPLAT_SPLATINSERT107:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP47:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT107]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT110:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT110:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP48:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT110]], <1 x double> [[TMP47]]) ; CHECK-NEXT: [[TMP49:%.*]] = getelementptr double, ptr [[C]], i64 2 ; CHECK-NEXT: store <1 x double> [[TMP46]], ptr [[TMP49]], align 8 @@ -198,7 +198,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP50]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT119:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP51:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT119]] -; CHECK-NEXT: [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT122]], <2 x double> [[TMP51]]) ; CHECK-NEXT: [[TMP53:%.*]] = getelementptr double, ptr [[C]], i64 6 ; CHECK-NEXT: store <2 x double> [[TMP52]], ptr [[TMP53]], align 8 @@ -210,7 +210,7 @@ define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) { ; CHECK-NEXT: [[COL_LOAD126:%.*]] = load <2 x double>, ptr [[TMP55]], align 8 ; CHECK-NEXT: [[SPLAT_SPLATINSERT128:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> poison, <1 x i32> zeroinitializer ; CHECK-NEXT: [[TMP56:%.*]] = fmul contract <1 x double> [[COL_LOAD123]], [[SPLAT_SPLATINSERT128]] -; CHECK-NEXT: [[SPLAT_SPLATINSERT131:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> +; CHECK-NEXT: [[SPLAT_SPLATINSERT131:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> poison, <1 x i32> ; CHECK-NEXT: [[TMP57:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD125]], <1 x double> [[SPLAT_SPLATINSERT131]], <1 x double> [[TMP56]]) ; CHECK-NEXT: [[TMP58:%.*]] = getelementptr double, ptr [[C]], i64 8 ; CHECK-NEXT: store <1 x double> [[TMP57]], ptr [[TMP58]], align 8 diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll index 3dd04b23f272d..81d1fc5861511 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll @@ -49,11 +49,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]]) ; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]] -; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 8 ; CHECK-NEXT: [[COL_LOAD21:%.*]] = load <2 x double>, ptr [[TMP12]], align 8 @@ -65,11 +65,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD26:%.*]] = load <2 x double>, ptr [[VEC_GEP25]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP9]]) -; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP14]]) ; CHECK-NEXT: [[SPLAT_SPLAT37:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP16:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT37]], <2 x double> [[TMP11]]) -; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT40]], <2 x double> [[TMP16]]) ; CHECK-NEXT: store <2 x double> [[TMP15]], ptr [[C]], align 8 ; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, ptr [[C]], i64 4 @@ -83,11 +83,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD47:%.*]] = load <2 x double>, ptr [[VEC_GEP46]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT50]] -; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT53]], <2 x double> [[TMP19]]) ; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT56]] -; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP22:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT59]], <2 x double> [[TMP21]]) ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP3]], i64 10 ; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <2 x double>, ptr [[TMP23]], align 8 @@ -99,11 +99,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD65:%.*]] = load <2 x double>, ptr [[VEC_GEP64]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP25:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP20]]) -; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP25]]) ; CHECK-NEXT: [[SPLAT_SPLAT76:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP27:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT76]], <2 x double> [[TMP22]]) -; CHECK-NEXT: [[SPLAT_SPLAT79:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT79:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP28:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT79]], <2 x double> [[TMP27]]) ; CHECK-NEXT: [[TMP29:%.*]] = getelementptr double, ptr [[C]], i64 2 ; CHECK-NEXT: store <2 x double> [[TMP26]], ptr [[TMP29]], align 8 @@ -118,11 +118,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD86:%.*]] = load <2 x double>, ptr [[VEC_GEP85]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP31:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT89]] -; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP31]]) ; CHECK-NEXT: [[SPLAT_SPLAT95:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP33:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT95]] -; CHECK-NEXT: [[SPLAT_SPLAT98:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT98:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP34:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT98]], <2 x double> [[TMP33]]) ; CHECK-NEXT: [[TMP35:%.*]] = getelementptr double, ptr [[TMP3]], i64 8 ; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[TMP35]], align 8 @@ -134,11 +134,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD104:%.*]] = load <2 x double>, ptr [[VEC_GEP103]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP32]]) -; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP38:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP37]]) ; CHECK-NEXT: [[SPLAT_SPLAT115:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT115]], <2 x double> [[TMP34]]) -; CHECK-NEXT: [[SPLAT_SPLAT118:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT118:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP40:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT118]], <2 x double> [[TMP39]]) ; CHECK-NEXT: [[TMP41:%.*]] = getelementptr double, ptr [[C]], i64 8 ; CHECK-NEXT: store <2 x double> [[TMP38]], ptr [[TMP41]], align 8 @@ -154,11 +154,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD125:%.*]] = load <2 x double>, ptr [[VEC_GEP124]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT128:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT128]] -; CHECK-NEXT: [[SPLAT_SPLAT131:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT131:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP45:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT131]], <2 x double> [[TMP44]]) ; CHECK-NEXT: [[SPLAT_SPLAT134:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP46:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT134]] -; CHECK-NEXT: [[SPLAT_SPLAT137:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT137:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP47:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT137]], <2 x double> [[TMP46]]) ; CHECK-NEXT: [[TMP48:%.*]] = getelementptr double, ptr [[TMP3]], i64 10 ; CHECK-NEXT: [[COL_LOAD138:%.*]] = load <2 x double>, ptr [[TMP48]], align 8 @@ -170,11 +170,11 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD143:%.*]] = load <2 x double>, ptr [[VEC_GEP142]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP50:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP45]]) -; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP51:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP50]]) ; CHECK-NEXT: [[SPLAT_SPLAT154:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT154]], <2 x double> [[TMP47]]) -; CHECK-NEXT: [[SPLAT_SPLAT157:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT157:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP53:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT157]], <2 x double> [[TMP52]]) ; CHECK-NEXT: [[TMP54:%.*]] = getelementptr double, ptr [[C]], i64 10 ; CHECK-NEXT: store <2 x double> [[TMP51]], ptr [[TMP54]], align 8 @@ -238,11 +238,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]]) ; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]] -; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]]) ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[A]], i64 8 ; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <2 x double>, ptr [[TMP4]], align 8 @@ -254,11 +254,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <2 x double>, ptr [[VEC_GEP18]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP1]]) -; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP6]]) ; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP3]]) -; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP8]]) ; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[C:%.*]], align 8 ; CHECK-NEXT: [[VEC_GEP34:%.*]] = getelementptr double, ptr [[C]], i64 4 @@ -272,11 +272,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, ptr [[VEC_GEP39]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT43]] -; CHECK-NEXT: [[SPLAT_SPLAT46:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT46:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP12:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT46]], <2 x double> [[TMP11]]) ; CHECK-NEXT: [[SPLAT_SPLAT49:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT49]] -; CHECK-NEXT: [[SPLAT_SPLAT52:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT52:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT52]], <2 x double> [[TMP13]]) ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, ptr [[A]], i64 10 ; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <2 x double>, ptr [[TMP15]], align 8 @@ -288,11 +288,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD58:%.*]] = load <2 x double>, ptr [[VEC_GEP57]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT62]], <2 x double> [[TMP12]]) -; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP18:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT65]], <2 x double> [[TMP17]]) ; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP14]]) -; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP19]]) ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 2 ; CHECK-NEXT: store <2 x double> [[TMP18]], ptr [[TMP21]], align 8 @@ -307,11 +307,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD79:%.*]] = load <2 x double>, ptr [[VEC_GEP78]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT82:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT82]] -; CHECK-NEXT: [[SPLAT_SPLAT85:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT85:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT85]], <2 x double> [[TMP23]]) ; CHECK-NEXT: [[SPLAT_SPLAT88:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT88]] -; CHECK-NEXT: [[SPLAT_SPLAT91:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT91:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT91]], <2 x double> [[TMP25]]) ; CHECK-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[A]], i64 8 ; CHECK-NEXT: [[COL_LOAD92:%.*]] = load <2 x double>, ptr [[TMP27]], align 8 @@ -323,11 +323,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[VEC_GEP96]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT101:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP29:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT101]], <2 x double> [[TMP24]]) -; CHECK-NEXT: [[SPLAT_SPLAT104:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT104:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP30:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT104]], <2 x double> [[TMP29]]) ; CHECK-NEXT: [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP31:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP26]]) -; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP31]]) ; CHECK-NEXT: [[TMP33:%.*]] = getelementptr double, ptr [[C]], i64 8 ; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[TMP33]], align 8 @@ -343,11 +343,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD118:%.*]] = load <2 x double>, ptr [[VEC_GEP117]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT121:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP36:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT121]] -; CHECK-NEXT: [[SPLAT_SPLAT124:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT124:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT124]], <2 x double> [[TMP36]]) ; CHECK-NEXT: [[SPLAT_SPLAT127:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP38:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT127]] -; CHECK-NEXT: [[SPLAT_SPLAT130:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT130:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT130]], <2 x double> [[TMP38]]) ; CHECK-NEXT: [[TMP40:%.*]] = getelementptr double, ptr [[A]], i64 10 ; CHECK-NEXT: [[COL_LOAD131:%.*]] = load <2 x double>, ptr [[TMP40]], align 8 @@ -355,11 +355,11 @@ define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) ; CHECK-NEXT: [[COL_LOAD133:%.*]] = load <2 x double>, ptr [[VEC_GEP132]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT140:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT140]], <2 x double> [[TMP37]]) -; CHECK-NEXT: [[SPLAT_SPLAT143:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT143:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP42:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT143]], <2 x double> [[TMP41]]) ; CHECK-NEXT: [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP39]]) -; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP44:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP43]]) ; CHECK-NEXT: [[TMP45:%.*]] = getelementptr double, ptr [[C]], i64 10 ; CHECK-NEXT: store <2 x double> [[TMP42]], ptr [[TMP45]], align 8 diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll index 27eb90520373d..a02ef276e43e4 100644 --- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll @@ -25,19 +25,19 @@ define void @multiply(ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <4 x double>, ptr [[VEC_GEP7]], align 8 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT11]], <2 x double> [[TMP0]]) -; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP1]]) -; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT17]], <2 x double> [[TMP2]]) ; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT20]] -; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP4]]) -; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP5]]) -; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> poison, <2 x i32> ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT29]], <2 x double> [[TMP6]]) ; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[C:%.*]], align 8 ; CHECK-NEXT: [[VEC_GEP30:%.*]] = getelementptr double, ptr [[C]], i64 2 diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll index 658388c216427..362708bacc20f 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll @@ -8,7 +8,7 @@ define <2 x i64> @PR61061(<2 x i64> noundef %vect) { ; CHECK-LABEL: define <2 x i64> @PR61061 ; CHECK-SAME: (<2 x i64> noundef [[VECT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[VECT]] to <16 x i8> -; CHECK-NEXT: [[PTR_SROA_0_15_VEC_INSERT:%.*]] = shufflevector <16 x i8> [[BC0]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[PTR_SROA_0_15_VEC_INSERT:%.*]] = shufflevector <16 x i8> [[BC0]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[PTR_SROA_0_15_VEC_INSERT]] to <2 x i64> ; CHECK-NEXT: ret <2 x i64> [[TMP1]] ; diff --git a/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll b/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll index dae3b591af696..c3131a41c2b2e 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll @@ -26,7 +26,7 @@ define <4 x i32> @square(<4 x i32> %num, i32 %y, i32 %x, i32 %h, i32 %k, i32 %w, ; CHECK-NEXT: [[OP_RDX13:%.*]] = add i32 [[OP_RDX10]], [[OP_RDX11]] ; CHECK-NEXT: [[OP_RDX14:%.*]] = add i32 [[OP_RDX12]], [[OP_RDX13]] ; CHECK-NEXT: [[OP_RDX15:%.*]] = add i32 [[OP_RDX14]], [[Y:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> , i32 [[OP_RDX15]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[OP_RDX15]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[ADD29:%.*]] = add <4 x i32> [[TMP2]], [[NUM:%.*]] ; CHECK-NEXT: ret <4 x i32> [[ADD29]] diff --git a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll index 6b30af6b179d5..039e735985f9f 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll @@ -165,7 +165,7 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32> ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[SHUFFLE2]] ; %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> @@ -181,7 +181,7 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[SHUFFLE2]] ; %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> @@ -197,7 +197,7 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> % ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i8> [[SHUFFLE2]] ; %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> @@ -213,7 +213,7 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i3 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: ret <16 x i8> [[SHUFFLE2]] ; %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> @@ -229,7 +229,7 @@ define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8> ; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <4 x i32> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[SHUFFLE2]] ; %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> @@ -245,7 +245,7 @@ define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> ; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[V1:%.*]] to <4 x i32> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[SHUFFLE2]] ; %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> @@ -259,9 +259,9 @@ define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_not_be_converted_up(<16 x i8> %v1) { ; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_not_be_converted_up( -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <4 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[SHUFFLE2]] ; %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> @@ -275,9 +275,9 @@ define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_not_be_converted_up(<16 x define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(<8 x i16> %v1) { ; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up( -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[V1:%.*]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[V1:%.*]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: [[BC1:%.*]] = bitcast <8 x i16> [[SHUFFLE1]] to <4 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[SHUFFLE2]] ; %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> @@ -293,7 +293,7 @@ define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> ; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up( ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <8 x i16> ; CHECK-NEXT: [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[SHUFFLE2]] ; %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> @@ -307,9 +307,9 @@ define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can_not_be_converted_up(<16 x i8> %v1) { ; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can_not_be_converted_up( -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> poison, <16 x i32> ; CHECK-NEXT: [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <8 x i16> -; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> +; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> poison, <8 x i32> ; CHECK-NEXT: ret <8 x i16> [[SHUFFLE2]] ; %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll index e62749c4c71f1..2038400a05869 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll @@ -12,7 +12,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) { ; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1 ; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]]) ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: ret <2 x i16> [[INS_1]] ; @@ -42,7 +42,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) { ; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1 ; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]]) ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: ret <2 x i16> [[INS_1]] ; @@ -72,7 +72,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) { ; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1 ; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]]) ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: ret <2 x i16> [[INS_1]] ; @@ -102,7 +102,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) { ; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1 ; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]]) ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: ret <2 x i16> [[INS_1]] ; @@ -132,7 +132,7 @@ define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) { ; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1 ; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]]) ; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]]) -; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0 +; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0 ; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1 ; GCN-NEXT: ret <2 x i32> [[INS_1]] ; @@ -157,7 +157,7 @@ define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) { ; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1 ; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]]) ; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]]) -; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0 +; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0 ; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1 ; GCN-NEXT: ret <2 x i32> [[INS_1]] ; @@ -182,7 +182,7 @@ define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) { ; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1 ; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]]) ; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]]) -; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0 +; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0 ; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1 ; GCN-NEXT: ret <2 x i32> [[INS_1]] ; @@ -207,7 +207,7 @@ define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) { ; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1 ; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]]) ; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]]) -; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0 +; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0 ; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1 ; GCN-NEXT: ret <2 x i32> [[INS_1]] ; @@ -235,7 +235,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) { ; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]]) ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) ; GFX7-NEXT: [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2 ; GFX7-NEXT: ret <3 x i16> [[INS_2]] @@ -283,7 +283,7 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) { ; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]]) ; GFX7-NEXT: [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]]) ; GFX7-NEXT: [[ADD_3:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_3]], i16 [[ARG1_3]]) -; GFX7-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0 +; GFX7-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0 ; GFX7-NEXT: [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1 ; GFX7-NEXT: [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2 ; GFX7-NEXT: [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll index 435c677c3afbc..bc5bcee361168 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll @@ -18,7 +18,7 @@ define <8 x float> @ceil_floor(<8 x float> %a) { ; SSE-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]]) ; SSE-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> ; SSE-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]]) -; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0 +; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0 ; SSE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> ; SSE-NEXT: [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> ; SSE-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3 @@ -39,7 +39,7 @@ define <8 x float> @ceil_floor(<8 x float> %a) { ; SLM-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]]) ; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> ; SLM-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]]) -; SLM-NEXT: [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0 +; SLM-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0 ; SLM-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> ; SLM-NEXT: [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> ; SLM-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3 @@ -62,7 +62,7 @@ define <8 x float> @ceil_floor(<8 x float> %a) { ; AVX-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP1]]) ; AVX-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> ; AVX-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP3]]) -; AVX-NEXT: [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0 +; AVX-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0 ; AVX-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1 ; AVX-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2 ; AVX-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i64 3 @@ -83,7 +83,7 @@ define <8 x float> @ceil_floor(<8 x float> %a) { ; AVX2-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]]) ; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> ; AVX2-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]]) -; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0 +; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0 ; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> ; AVX2-NEXT: [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> ; AVX2-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll index 3334d494621af..e690b041416b3 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll @@ -448,7 +448,7 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) { ; AVX2-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4 ; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> ; AVX2-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], -; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i32> , i32 [[AB1]], i64 1 +; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1 ; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> ; AVX2-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> ; AVX2-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5 @@ -465,7 +465,7 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) { ; AVX512-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4 ; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> ; AVX512-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], -; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x i32> , i32 [[AB1]], i64 1 +; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1 ; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> ; AVX512-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> ; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll index cac6845c43004..1dcabb7aafc46 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll @@ -220,7 +220,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b ; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> ; SSE-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]] ; SSE-NEXT: [[R3:%.*]] = fadd double [[B2]], [[B3]] -; SSE-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> undef, <4 x i32> +; SSE-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> ; SSE-NEXT: [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3 ; SSE-NEXT: ret <4 x double> [[R03]] ; @@ -231,7 +231,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b ; SLM-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> ; SLM-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]] ; SLM-NEXT: [[R3:%.*]] = fadd double [[B2]], [[B3]] -; SLM-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> undef, <4 x i32> +; SLM-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> ; SLM-NEXT: [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3 ; SLM-NEXT: ret <4 x double> [[R03]] ; @@ -242,7 +242,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b ; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> ; AVX1-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> ; AVX1-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]] -; AVX1-NEXT: [[R00:%.*]] = insertelement <4 x double> undef, double [[R0]], i64 0 +; AVX1-NEXT: [[R00:%.*]] = insertelement <4 x double> poison, double [[R0]], i64 0 ; AVX1-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> ; AVX1-NEXT: [[R031:%.*]] = shufflevector <4 x double> [[R00]], <4 x double> [[TMP4]], <4 x i32> ; AVX1-NEXT: ret <4 x double> [[R031]] @@ -254,7 +254,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b ; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> ; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> ; AVX2-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]] -; AVX2-NEXT: [[R00:%.*]] = insertelement <4 x double> undef, double [[R0]], i64 0 +; AVX2-NEXT: [[R00:%.*]] = insertelement <4 x double> poison, double [[R0]], i64 0 ; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> ; AVX2-NEXT: [[R031:%.*]] = shufflevector <4 x double> [[R00]], <4 x double> [[TMP4]], <4 x i32> ; AVX2-NEXT: ret <4 x double> [[R031]] @@ -266,7 +266,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b ; AVX512-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> ; AVX512-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]] ; AVX512-NEXT: [[R3:%.*]] = fadd double [[B2]], [[B3]] -; AVX512-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> undef, <4 x i32> +; AVX512-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> ; AVX512-NEXT: [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3 ; AVX512-NEXT: ret <4 x double> [[R03]] ; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll index b93174da6a569..500856a0d66a9 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll @@ -14,7 +14,7 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl ; SSE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4 ; SSE-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]] ; SSE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i64 0 +; SSE-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; SSE-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1 ; SSE-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2 ; SSE-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3 @@ -30,7 +30,7 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl ; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4 ; AVX-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]] ; AVX-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]] -; AVX-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i64 0 +; AVX-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; AVX-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1 ; AVX-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2 ; AVX-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3 @@ -46,7 +46,7 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl ; AVX2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4 ; AVX2-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]] ; AVX2-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]] -; AVX2-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i64 0 +; AVX2-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; AVX2-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1 ; AVX2-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2 ; AVX2-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3 @@ -62,7 +62,7 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl ; AVX512F-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4 ; AVX512F-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]] ; AVX512F-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]] -; AVX512F-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i64 0 +; AVX512F-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; AVX512F-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1 ; AVX512F-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2 ; AVX512F-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3 @@ -78,7 +78,7 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl ; AVX512VL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4 ; AVX512VL-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]] ; AVX512VL-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]] -; AVX512VL-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i64 0 +; AVX512VL-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; AVX512VL-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1 ; AVX512VL-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2 ; AVX512VL-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll index c62fb97da5c89..42a50384787c8 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll @@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu" define <4 x i32> @foo(<4 x i32> %x, i32 %f) { ; CHECK-LABEL: @foo( -; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x i32> undef, i32 [[F:%.*]], i64 0 +; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x i32> poison, i32 [[F:%.*]], i64 0 ; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[F]], 1 ; CHECK-NEXT: [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i64 1 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i64 0 @@ -31,8 +31,8 @@ define <4 x i32> @bar(<4 x i32> %x, i32 %f) { ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F:%.*]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> -; CHECK-NEXT: ret <4 x i32> [[SHUFFLE]] +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> +; CHECK-NEXT: ret <4 x i32> [[TMP4]] ; %add = add nsw i32 %f, 2 %vecinit = insertelement <4 x i32> undef, i32 %add, i32 0 From 87b45f1ca6d4ff4cb322c14b6375b2ea0063f82a Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 18 Dec 2023 09:22:22 -0600 Subject: [PATCH 126/884] [llvm-readobj] Print AMDGPU specific values in GNU mode (#75661) Summary: Currently, we don't emit any of the AMDGPU specific flags and options when printing in GNU-mode (the default). This patch adds extra code to handle outputting these values to match the output from GNU's `readelf` when used on an AMDGPU image. --- .../llvm-readobj/ELF/file-header-os-abi.test | 6 +- llvm/tools/llvm-readobj/ELFDumper.cpp | 295 ++++++++++-------- 2 files changed, 167 insertions(+), 134 deletions(-) diff --git a/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test b/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test index 31675928a0d92..f815e78e0c540 100644 --- a/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test +++ b/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test @@ -167,21 +167,21 @@ FileHeader: # RUN: llvm-readelf --file-headers %t.osabi.amd.hsa | FileCheck %s --match-full-lines --check-prefix=OSABI-AMDHSA-GNU # OSABI-AMDHSA-LLVM: OS/ABI: AMDGPU_HSA (0x40) -# OSABI-AMDHSA-GNU: OS/ABI: 40 +# OSABI-AMDHSA-GNU: OS/ABI: AMDGPU - HSA # RUN: yaml2obj %s -DOSABI=ELFOSABI_AMDGPU_PAL -DMACHINE=EM_AMDGPU -o %t.osabi.amd.pal # RUN: llvm-readobj --file-headers %t.osabi.amd.pal | FileCheck %s --match-full-lines --check-prefix=OSABI-AMDPAL-LLVM # RUN: llvm-readelf --file-headers %t.osabi.amd.pal | FileCheck %s --match-full-lines --check-prefix=OSABI-AMDPAL-GNU # OSABI-AMDPAL-LLVM: OS/ABI: AMDGPU_PAL (0x41) -# OSABI-AMDPAL-GNU: OS/ABI: 41 +# OSABI-AMDPAL-GNU: OS/ABI: AMDGPU - PAL # RUN: yaml2obj %s -DOSABI=ELFOSABI_AMDGPU_MESA3D -DMACHINE=EM_AMDGPU -o %t.osabi.amd.mesa3d # RUN: llvm-readobj --file-headers %t.osabi.amd.mesa3d | FileCheck %s --match-full-lines --check-prefix=OSABI-AMDMESA3D-LLVM # RUN: llvm-readelf --file-headers %t.osabi.amd.mesa3d | FileCheck %s --match-full-lines --check-prefix=OSABI-AMDMESA3D-GNU # OSABI-AMDMESA3D-LLVM: OS/ABI: AMDGPU_MESA3D (0x42) -# OSABI-AMDMESA3D-GNU: OS/ABI: 42 +# OSABI-AMDMESA3D-GNU: OS/ABI: AMDGPU - MESA3D # Check all EM_ARM specific values. diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index f2851a52671bc..abf7ba6ba1c38 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1558,139 +1558,139 @@ const EnumEntry ElfHeaderMipsFlags[] = { }; const EnumEntry ElfHeaderAMDGPUFlagsABIVersion3[] = { - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_NONE), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R600), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R630), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RS880), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV670), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV710), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV730), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV770), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CEDAR), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CYPRESS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_JUNIPER), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_REDWOOD), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_SUMO), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_BARTS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAICOS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAYMAN), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_TURKS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX600), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX601), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX602), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX700), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX701), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX702), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX703), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX704), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX705), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX801), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX802), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX803), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX805), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX810), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX900), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX902), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX904), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX906), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX908), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX909), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90A), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90C), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX940), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX941), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX942), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1010), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1011), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1012), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1013), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1030), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1031), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1032), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1033), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1034), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1035), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1036), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1100), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1101), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1102), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1103), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1150), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1151), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1200), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1201), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_V3), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_V3), + ENUM_ENT(EF_AMDGPU_MACH_NONE, "none"), + ENUM_ENT(EF_AMDGPU_MACH_R600_R600, "r600"), + ENUM_ENT(EF_AMDGPU_MACH_R600_R630, "r630"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RS880, "rs880"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV670, "rv670"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV710, "rv710"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV730, "rv730"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV770, "rv770"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CEDAR, "cedar"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CYPRESS, "cypress"), + ENUM_ENT(EF_AMDGPU_MACH_R600_JUNIPER, "juniper"), + ENUM_ENT(EF_AMDGPU_MACH_R600_REDWOOD, "redwood"), + ENUM_ENT(EF_AMDGPU_MACH_R600_SUMO, "sumo"), + ENUM_ENT(EF_AMDGPU_MACH_R600_BARTS, "barts"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CAICOS, "caicos"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CAYMAN, "cayman"), + ENUM_ENT(EF_AMDGPU_MACH_R600_TURKS, "turks"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX600, "gfx600"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX601, "gfx601"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX602, "gfx602"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX700, "gfx700"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX701, "gfx701"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX702, "gfx702"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX703, "gfx703"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX704, "gfx704"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX705, "gfx705"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX801, "gfx801"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX802, "gfx802"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX803, "gfx803"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX805, "gfx805"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX810, "gfx810"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX900, "gfx900"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX902, "gfx902"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX904, "gfx904"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX906, "gfx906"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX908, "gfx908"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX909, "gfx909"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX90A, "gfx90a"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX90C, "gfx90c"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX940, "gfx940"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX941, "gfx941"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX942, "gfx942"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1010, "gfx1010"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1011, "gfx1011"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1012, "gfx1012"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1013, "gfx1013"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1030, "gfx1030"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1031, "gfx1031"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1032, "gfx1032"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1033, "gfx1033"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1034, "gfx1034"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1035, "gfx1035"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1036, "gfx1036"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1100, "gfx1100"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1101, "gfx1101"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1102, "gfx1102"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1103, "gfx1103"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1150, "gfx1150"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"), + ENUM_ENT(EF_AMDGPU_FEATURE_XNACK_V3, "xnack"), + ENUM_ENT(EF_AMDGPU_FEATURE_SRAMECC_V3, "sramecc"), }; const EnumEntry ElfHeaderAMDGPUFlagsABIVersion4[] = { - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_NONE), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R600), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R630), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RS880), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV670), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV710), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV730), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV770), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CEDAR), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CYPRESS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_JUNIPER), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_REDWOOD), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_SUMO), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_BARTS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAICOS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAYMAN), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_TURKS), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX600), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX601), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX602), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX700), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX701), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX702), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX703), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX704), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX705), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX801), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX802), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX803), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX805), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX810), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX900), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX902), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX904), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX906), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX908), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX909), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90A), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90C), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX940), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX941), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX942), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1010), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1011), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1012), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1013), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1030), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1031), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1032), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1033), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1034), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1035), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1036), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1100), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1101), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1102), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1103), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1150), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1151), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1200), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1201), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_ANY_V4), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_OFF_V4), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_ON_V4), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_ANY_V4), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_OFF_V4), - LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_ON_V4), + ENUM_ENT(EF_AMDGPU_MACH_NONE, "none"), + ENUM_ENT(EF_AMDGPU_MACH_R600_R600, "r600"), + ENUM_ENT(EF_AMDGPU_MACH_R600_R630, "r630"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RS880, "rs880"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV670, "rv670"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV710, "rv710"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV730, "rv730"), + ENUM_ENT(EF_AMDGPU_MACH_R600_RV770, "rv770"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CEDAR, "cedar"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CYPRESS, "cypress"), + ENUM_ENT(EF_AMDGPU_MACH_R600_JUNIPER, "juniper"), + ENUM_ENT(EF_AMDGPU_MACH_R600_REDWOOD, "redwood"), + ENUM_ENT(EF_AMDGPU_MACH_R600_SUMO, "sumo"), + ENUM_ENT(EF_AMDGPU_MACH_R600_BARTS, "barts"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CAICOS, "caicos"), + ENUM_ENT(EF_AMDGPU_MACH_R600_CAYMAN, "cayman"), + ENUM_ENT(EF_AMDGPU_MACH_R600_TURKS, "turks"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX600, "gfx600"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX601, "gfx601"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX602, "gfx602"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX700, "gfx700"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX701, "gfx701"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX702, "gfx702"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX703, "gfx703"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX704, "gfx704"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX705, "gfx705"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX801, "gfx801"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX802, "gfx802"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX803, "gfx803"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX805, "gfx805"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX810, "gfx810"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX900, "gfx900"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX902, "gfx902"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX904, "gfx904"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX906, "gfx906"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX908, "gfx908"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX909, "gfx909"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX90A, "gfx90a"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX90C, "gfx90c"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX940, "gfx940"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX941, "gfx941"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX942, "gfx942"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1010, "gfx1010"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1011, "gfx1011"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1012, "gfx1012"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1013, "gfx1013"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1030, "gfx1030"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1031, "gfx1031"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1032, "gfx1032"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1033, "gfx1033"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1034, "gfx1034"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1035, "gfx1035"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1036, "gfx1036"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1100, "gfx1100"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1101, "gfx1101"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1102, "gfx1102"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1103, "gfx1103"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1150, "gfx1150"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"), + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"), + ENUM_ENT(EF_AMDGPU_FEATURE_XNACK_ANY_V4, "xnack"), + ENUM_ENT(EF_AMDGPU_FEATURE_XNACK_OFF_V4, "xnack-"), + ENUM_ENT(EF_AMDGPU_FEATURE_XNACK_ON_V4, "xnack+"), + ENUM_ENT(EF_AMDGPU_FEATURE_SRAMECC_ANY_V4, "sramecc"), + ENUM_ENT(EF_AMDGPU_FEATURE_SRAMECC_OFF_V4, "sramecc-"), + ENUM_ENT(EF_AMDGPU_FEATURE_SRAMECC_ON_V4, "sramecc+"), }; const EnumEntry ElfHeaderNVPTXFlags[] = { @@ -3599,7 +3599,18 @@ template void GNUELFDumper::printFileHeaders() { if (e.e_version == ELF::EV_CURRENT) OS << " (current)"; OS << "\n"; - Str = enumToString(e.e_ident[ELF::EI_OSABI], ArrayRef(ElfOSABI)); + auto OSABI = ArrayRef(ElfOSABI); + if (e.e_ident[ELF::EI_OSABI] >= ELF::ELFOSABI_FIRST_ARCH && + e.e_ident[ELF::EI_OSABI] <= ELF::ELFOSABI_LAST_ARCH) { + switch (e.e_machine) { + case ELF::EM_AMDGPU: + OSABI = ArrayRef(AMDGPUElfOSABI); + break; + default: + break; + } + } + Str = enumToString(e.e_ident[ELF::EI_OSABI], OSABI); printFields(OS, "OS/ABI:", Str); printFields(OS, "ABI Version:", std::to_string(e.e_ident[ELF::EI_ABIVERSION])); @@ -3646,6 +3657,28 @@ template void GNUELFDumper::printFileHeaders() { else if (e.e_machine == EM_CUDA) ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderNVPTXFlags), unsigned(ELF::EF_CUDA_SM)); + else if (e.e_machine == EM_AMDGPU) { + switch (e.e_ident[ELF::EI_ABIVERSION]) { + default: + break; + case 0: + // ELFOSABI_AMDGPU_PAL, ELFOSABI_AMDGPU_MESA3D support *_V3 flags. + [[fallthrough]]; + case ELF::ELFABIVERSION_AMDGPU_HSA_V3: + ElfFlags = + printFlags(e.e_flags, ArrayRef(ElfHeaderAMDGPUFlagsABIVersion3), + unsigned(ELF::EF_AMDGPU_MACH)); + break; + case ELF::ELFABIVERSION_AMDGPU_HSA_V4: + case ELF::ELFABIVERSION_AMDGPU_HSA_V5: + ElfFlags = + printFlags(e.e_flags, ArrayRef(ElfHeaderAMDGPUFlagsABIVersion4), + unsigned(ELF::EF_AMDGPU_MACH), + unsigned(ELF::EF_AMDGPU_FEATURE_XNACK_V4), + unsigned(ELF::EF_AMDGPU_FEATURE_SRAMECC_V4)); + break; + } + } Str = "0x" + utohexstr(e.e_flags); if (!ElfFlags.empty()) Str = Str + ", " + ElfFlags; From e60167927bb6153408dbaa081fcb606e7a8f493a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20W=C3=B3jt?= Date: Mon, 18 Dec 2023 16:25:50 +0100 Subject: [PATCH 127/884] [libc++] Add a picolibc test configuration with exceptions enabled (#75462) --- libcxx/cmake/caches/Armv7M-picolibc.cmake | 7 +- libcxx/utils/ci/buildkite-pipeline.yml | 7 ++ libcxx/utils/ci/run-buildbot | 78 +++++++++++-------- .../test/catch_multi_level_pointer.pass.cpp | 4 + 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/libcxx/cmake/caches/Armv7M-picolibc.cmake b/libcxx/cmake/caches/Armv7M-picolibc.cmake index 9f8863943444b..91cc32fd376e3 100644 --- a/libcxx/cmake/caches/Armv7M-picolibc.cmake +++ b/libcxx/cmake/caches/Armv7M-picolibc.cmake @@ -13,17 +13,18 @@ set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "") set(COMPILER_RT_DEFAULT_TARGET_ONLY ON CACHE BOOL "") set(LIBCXXABI_BAREMETAL ON CACHE BOOL "") set(LIBCXXABI_ENABLE_ASSERTIONS OFF CACHE BOOL "") -set(LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "") +set(LIBCXXABI_ENABLE_EXCEPTIONS ON CACHE BOOL "") set(LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "") set(LIBCXXABI_ENABLE_STATIC ON CACHE BOOL "") +set(LIBCXXABI_ENABLE_STATIC_UNWINDER ON CACHE BOOL "") set(LIBCXXABI_ENABLE_THREADS OFF CACHE BOOL "") set(LIBCXXABI_USE_COMPILER_RT ON CACHE BOOL "") set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "") -set(LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "") +set(LIBCXX_ENABLE_EXCEPTIONS ON CACHE BOOL "") set(LIBCXX_ENABLE_FILESYSTEM OFF CACHE STRING "") set(LIBCXX_ENABLE_MONOTONIC_CLOCK OFF CACHE BOOL "") set(LIBCXX_ENABLE_RANDOM_DEVICE OFF CACHE BOOL "") -set(LIBCXX_ENABLE_RTTI OFF CACHE BOOL "") +set(LIBCXX_ENABLE_RTTI ON CACHE BOOL "") set(LIBCXX_ENABLE_SHARED OFF CACHE BOOL "") set(LIBCXX_ENABLE_STATIC ON CACHE BOOL "") set(LIBCXX_ENABLE_THREADS OFF CACHE BOOL "") diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml index c63f6a78b1607..a48f8524ef63c 100644 --- a/libcxx/utils/ci/buildkite-pipeline.yml +++ b/libcxx/utils/ci/buildkite-pipeline.yml @@ -224,6 +224,13 @@ steps: arch: aarch64 <<: *common + - label: Armv7-M picolibc -fno-exceptions + command: libcxx/utils/ci/run-buildbot armv7m-picolibc-no-exceptions + agents: + queue: libcxx-builders-linaro-arm + arch: aarch64 + <<: *common + - group: AIX steps: - label: AIX (32-bit) diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot index ee890fb766996..672d5cb70ae81 100755 --- a/libcxx/utils/ci/run-buildbot +++ b/libcxx/utils/ci/run-buildbot @@ -203,6 +203,43 @@ function check-cxx-benchmarks() { ${NINJA} -vC "${BUILD_DIR}" check-cxx-benchmarks } +function test-armv7m-picolibc() { + clean + + # To make it easier to get this builder up and running, build picolibc + # from scratch. Anecdotally, the build-picolibc script takes about 16 seconds. + # This could be optimised by building picolibc into the Docker container. + ${MONOREPO_ROOT}/libcxx/utils/ci/build-picolibc.sh \ + --build-dir "${BUILD_DIR}" \ + --install-dir "${INSTALL_DIR}" \ + --target armv7m-none-eabi + + echo "--- Generating CMake" + flags="--sysroot=${INSTALL_DIR}" + ${CMAKE} \ + -S "${MONOREPO_ROOT}/compiler-rt" \ + -B "${BUILD_DIR}/compiler-rt" \ + -GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_C_FLAGS="${flags}" \ + -DCMAKE_CXX_FLAGS="${flags}" \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON \ + "${@}" + generate-cmake \ + -DLIBCXX_TEST_CONFIG="armv7m-picolibc-libc++.cfg.in" \ + -DLIBCXXABI_TEST_CONFIG="armv7m-picolibc-libc++abi.cfg.in" \ + -DLIBUNWIND_TEST_CONFIG="armv7m-picolibc-libunwind.cfg.in" \ + -DCMAKE_C_FLAGS="${flags}" \ + -DCMAKE_CXX_FLAGS="${flags}" \ + "${@}" + + ${NINJA} -vC "${BUILD_DIR}/compiler-rt" install + mv "${BUILD_DIR}/install/lib/armv7m-none-eabi"/* "${BUILD_DIR}/install/lib" + + check-runtimes +} + # Print the version of a few tools to aid diagnostics in some cases ${CMAKE} --version ${NINJA} --version @@ -616,39 +653,16 @@ armv7-no-exceptions) check-runtimes ;; armv7m-picolibc) - clean - - # To make it easier to get this builder up and running, build picolibc - # from scratch. Anecdotally, the build-picolibc script takes about 16 seconds. - # This could be optimised by building picolibc into the Docker container. - ${MONOREPO_ROOT}/libcxx/utils/ci/build-picolibc.sh \ - --build-dir "${BUILD_DIR}" \ - --install-dir "${INSTALL_DIR}" \ - --target armv7m-none-eabi - - echo "--- Generating CMake" - flags="--sysroot=${INSTALL_DIR}" - ${CMAKE} \ - -S "${MONOREPO_ROOT}/compiler-rt" \ - -B "${BUILD_DIR}/compiler-rt" \ - -GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + test-armv7m-picolibc \ + -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7M-picolibc.cmake" +;; +armv7m-picolibc-no-exceptions) + test-armv7m-picolibc \ -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7M-picolibc.cmake" \ - -DCMAKE_C_FLAGS="${flags}" \ - -DCMAKE_CXX_FLAGS="${flags}" \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON - generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7M-picolibc.cmake" \ - -DLIBCXX_TEST_CONFIG="armv7m-picolibc-libc++.cfg.in" \ - -DLIBCXXABI_TEST_CONFIG="armv7m-picolibc-libc++abi.cfg.in" \ - -DLIBUNWIND_TEST_CONFIG="armv7m-picolibc-libunwind.cfg.in" \ - -DCMAKE_C_FLAGS="${flags}" \ - -DCMAKE_CXX_FLAGS="${flags}" - - ${NINJA} -vC "${BUILD_DIR}/compiler-rt" install - mv "${BUILD_DIR}/install/lib/armv7m-none-eabi"/* "${BUILD_DIR}/install/lib" - - check-runtimes + -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=OFF \ + -DLIBCXX_ENABLE_EXCEPTIONS=OFF \ + -DLIBCXX_ENABLE_RTTI=OFF ;; clang-cl-dll) clean diff --git a/libcxxabi/test/catch_multi_level_pointer.pass.cpp b/libcxxabi/test/catch_multi_level_pointer.pass.cpp index 00ea66da30176..5f0e78a391b14 100644 --- a/libcxxabi/test/catch_multi_level_pointer.pass.cpp +++ b/libcxxabi/test/catch_multi_level_pointer.pass.cpp @@ -11,6 +11,10 @@ // 1b00fc5d8133 made it in the dylib in macOS 10.11 // XFAIL: stdlib=apple-libc++ && target={{.+}}-apple-macosx10.{{9|10}} +// mps2-an385 machine used for testing of picolibc has just 4 MB of "flash" +// memory and this test requires almost 5 MB +// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME + #include #include #include From 27767959a5e53ffa3e91ea011f8c44eeeaf59132 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Mon, 18 Dec 2023 16:27:13 +0100 Subject: [PATCH 128/884] [libc++][CI] Enable modules in no TZDB build. (#75704) This is one of the few build where we didn't test modules. --- libcxx/cmake/caches/Generic-no-tzdb.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/libcxx/cmake/caches/Generic-no-tzdb.cmake b/libcxx/cmake/caches/Generic-no-tzdb.cmake index 27c826edfecff..c5dc882e58442 100644 --- a/libcxx/cmake/caches/Generic-no-tzdb.cmake +++ b/libcxx/cmake/caches/Generic-no-tzdb.cmake @@ -1 +1,2 @@ +set(LIBCXX_ENABLE_STD_MODULES ON CACHE BOOL "") # TODO MODULES Remove when enabled automatically. set(LIBCXX_ENABLE_TIME_ZONE_DATABASE OFF CACHE BOOL "") From 1580877555f91c5ab7eed7926bb984b2f6b70bb4 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 18 Dec 2023 09:28:06 -0600 Subject: [PATCH 129/884] [Libomptarget] Remove bitcode image map used for JIT processing (#75672) Summary: Libomptarget supports JIT by treating an LLVM-IR file as a regular input image. The handling here used a global map to keep track of triples once it was parsed. This was done to same time, however this created a global constructor as well as an extra mutex to handle it. This patch removes the use of this map. Instead, we simply use the file magic to perform a quick check if the input image is valid bitcode. If not, we then create a lazy module. This should roughly equivalent to the old handling that create an IR symbol table. Here we can prevent the module from materializing everything but the single triple metadata we read in later. --- .../plugins-nextgen/common/src/JIT.cpp | 51 +++++++------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/src/JIT.cpp b/openmp/libomptarget/plugins-nextgen/common/src/JIT.cpp index 188fb6dac84b9..08080c9d6091b 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/JIT.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/JIT.cpp @@ -47,18 +47,19 @@ using namespace llvm::object; using namespace omp; using namespace omp::target; -static codegen::RegisterCodeGenFlags RCGF; - namespace { -/// A map from a bitcode image start address to its corresponding triple. If the -/// image is not in the map, it is not a bitcode image. -DenseMap BitcodeImageMap; -std::shared_mutex BitcodeImageMapMutex; +bool isImageBitcode(const __tgt_device_image &Image) { + StringRef Binary(reinterpret_cast(Image.ImageStart), + target::getPtrDiff(Image.ImageEnd, Image.ImageStart)); + + return identify_magic(Binary) == file_magic::bitcode; +} std::once_flag InitFlag; void init(Triple TT) { + codegen::RegisterCodeGenFlags(); #ifdef LIBOMPTARGET_JIT_NVPTX if (TT.isNVPTX()) { LLVMInitializeNVPTXTargetInfo(); @@ -323,44 +324,30 @@ JITEngine::process(const __tgt_device_image &Image, return Device.doJITPostProcessing(std::move(MB)); }; - { - std::shared_lock SharedLock(BitcodeImageMapMutex); - auto Itr = BitcodeImageMap.find(Image.ImageStart); - if (Itr != BitcodeImageMap.end() && Itr->second == TT.getArch()) - return compile(Image, ComputeUnitKind, PostProcessing); - } + if (isImageBitcode(Image)) + return compile(Image, ComputeUnitKind, PostProcessing); return &Image; } bool JITEngine::checkBitcodeImage(const __tgt_device_image &Image) { TimeTraceScope TimeScope("Check bitcode image"); - std::lock_guard Lock(BitcodeImageMapMutex); - { - auto Itr = BitcodeImageMap.find(Image.ImageStart); - if (Itr != BitcodeImageMap.end() && Itr->second == TT.getArch()) - return true; - } + if (!isImageBitcode(Image)) + return false; StringRef Data(reinterpret_cast(Image.ImageStart), target::getPtrDiff(Image.ImageEnd, Image.ImageStart)); - std::unique_ptr MB = MemoryBuffer::getMemBuffer( - Data, /* BufferName */ "", /* RequiresNullTerminator */ false); + auto MB = MemoryBuffer::getMemBuffer(Data, /*BufferName=*/"", + /*RequiresNullTerminator=*/false); if (!MB) return false; - Expected FOrErr = object::readIRSymtab(*MB); - if (!FOrErr) { - consumeError(FOrErr.takeError()); - return false; - } - - auto ActualTriple = FOrErr->TheReader.getTargetTriple(); - auto BitcodeTA = Triple(ActualTriple).getArch(); - BitcodeImageMap[Image.ImageStart] = BitcodeTA; - - DP("Is%s IR Image\n", BitcodeTA == TT.getArch() ? " " : " NOT"); + LLVMContext Context; + SMDiagnostic Diagnostic; + std::unique_ptr M = + llvm::getLazyIRModule(std::move(MB), Diagnostic, Context, + /*ShouldLazyLoadMetadata=*/true); - return BitcodeTA == TT.getArch(); + return M && Triple(M->getTargetTriple()).getArch() == TT.getArch(); } From 0f5eef1fb3e3f6491fb27dfb69b230946ffea69c Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Mon, 18 Dec 2023 10:29:26 -0500 Subject: [PATCH 130/884] [OpenMP][Clang] Force use of `num_teams` and `thread_limit` for bare kernel (#68373) This patch makes `num_teams` and `thread_limit` mandatory for bare kernels, similar to a reguar kernel language that when launching a kernel, the grid size has to be set explicitly. --- .../clang/Basic/DiagnosticSemaKinds.td | 2 + clang/lib/Sema/SemaOpenMP.cpp | 13 ++ .../nvptx_target_teams_ompx_bare_codegen.cpp | 2 +- clang/test/OpenMP/ompx_bare_messages.c | 7 +- clang/test/OpenMP/target_teams_ast_print.cpp | 4 +- clang/test/OpenMP/target_teams_codegen.cpp | 206 +++++++++--------- 6 files changed, 130 insertions(+), 104 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index c8e32a63684f2..3bde53a8ce863 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -11368,6 +11368,8 @@ def err_openmp_vla_in_task_untied : Error< def warn_omp_unterminated_declare_target : Warning< "expected '#pragma omp end declare target' at end of file to match '#pragma omp %0'">, InGroup; +def err_ompx_bare_no_grid : Error< + "'ompx_bare' clauses requires explicit grid size via 'num_teams' and 'thread_limit' clauses">; } // end of OpenMP category let CategoryName = "Related Result Type Issue" in { diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp index e400f248d15aa..3826994ef2126 100644 --- a/clang/lib/Sema/SemaOpenMP.cpp +++ b/clang/lib/Sema/SemaOpenMP.cpp @@ -14658,6 +14658,19 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDirective(ArrayRef Clauses, } setFunctionHasBranchProtectedScope(); + const OMPClause *BareClause = nullptr; + bool HasThreadLimitAndNumTeamsClause = hasClauses(Clauses, OMPC_num_teams) && + hasClauses(Clauses, OMPC_thread_limit); + bool HasBareClause = llvm::any_of(Clauses, [&](const OMPClause *C) { + BareClause = C; + return C->getClauseKind() == OMPC_ompx_bare; + }); + + if (HasBareClause && !HasThreadLimitAndNumTeamsClause) { + Diag(BareClause->getBeginLoc(), diag::err_ompx_bare_no_grid); + return StmtError(); + } + return OMPTargetTeamsDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt); } diff --git a/clang/test/OpenMP/nvptx_target_teams_ompx_bare_codegen.cpp b/clang/test/OpenMP/nvptx_target_teams_ompx_bare_codegen.cpp index 9f8046acb0970..2e6f0a9ce0169 100644 --- a/clang/test/OpenMP/nvptx_target_teams_ompx_bare_codegen.cpp +++ b/clang/test/OpenMP/nvptx_target_teams_ompx_bare_codegen.cpp @@ -10,7 +10,7 @@ template tx ftemplate(int n) { tx a = 0; - #pragma omp target teams ompx_bare + #pragma omp target teams ompx_bare num_teams(1) thread_limit(32) { a = 2; } diff --git a/clang/test/OpenMP/ompx_bare_messages.c b/clang/test/OpenMP/ompx_bare_messages.c index a1b3c38028528..19ceee5625fee 100644 --- a/clang/test/OpenMP/ompx_bare_messages.c +++ b/clang/test/OpenMP/ompx_bare_messages.c @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -verify -fopenmp -triple x86_64-unknown-unknown %s - // RUN: %clang_cc1 -verify -fopenmp-simd -triple x86_64-unknown-unknown %s - // RUN: %clang_cc1 -verify -fopenmp -triple x86_64-unknown-unknown -fopenmp-targets=nvptx64 %s +// RUN: %clang_cc1 -verify -fopenmp-simd -triple x86_64-unknown-unknown %s +// RUN: %clang_cc1 -verify -fopenmp -triple x86_64-unknown-unknown -fopenmp-targets=nvptx64 %s void foo() { } @@ -18,4 +18,7 @@ void bar() { #pragma omp target #pragma omp teams ompx_bare // expected-error {{unexpected OpenMP clause 'ompx_bare' in directive '#pragma omp teams'}} expected-note {{OpenMP extension clause 'ompx_bare' only allowed with '#pragma omp target teams'}} foo(); + +#pragma omp target teams ompx_bare // expected-error {{'ompx_bare' clauses requires explicit grid size via 'num_teams' and 'thread_limit' clauses}} + foo(); } diff --git a/clang/test/OpenMP/target_teams_ast_print.cpp b/clang/test/OpenMP/target_teams_ast_print.cpp index 5f1040be01a25..8eaf4cbf24933 100644 --- a/clang/test/OpenMP/target_teams_ast_print.cpp +++ b/clang/test/OpenMP/target_teams_ast_print.cpp @@ -111,8 +111,8 @@ int main (int argc, char **argv) { // CHECK-NEXT: #pragma omp target teams a=2; // CHECK-NEXT: a = 2; -#pragma omp target teams ompx_bare -// CHECK-NEXT: #pragma omp target teams ompx_bare +#pragma omp target teams ompx_bare num_teams(1) thread_limit(32) +// CHECK-NEXT: #pragma omp target teams ompx_bare num_teams(1) thread_limit(32) a=3; // CHECK-NEXT: a = 3; #pragma omp target teams default(none), private(argc,b) num_teams(f) firstprivate(argv) reduction(| : c, d) reduction(* : e) thread_limit(f+g) diff --git a/clang/test/OpenMP/target_teams_codegen.cpp b/clang/test/OpenMP/target_teams_codegen.cpp index b7c7add229c14..8790a0fc87cbb 100644 --- a/clang/test/OpenMP/target_teams_codegen.cpp +++ b/clang/test/OpenMP/target_teams_codegen.cpp @@ -121,7 +121,7 @@ int foo(int n) { aa += 1; } - #pragma omp target teams ompx_bare + #pragma omp target teams ompx_bare num_teams(1) thread_limit(1) { a += 1; aa += 1; @@ -588,12 +588,12 @@ int bar(int n){ // CHECK1-NEXT: [[TMP116:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 9 // CHECK1-NEXT: store i64 0, ptr [[TMP116]], align 8 // CHECK1-NEXT: [[TMP117:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 10 -// CHECK1-NEXT: store [3 x i32] zeroinitializer, ptr [[TMP117]], align 4 +// CHECK1-NEXT: store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP117]], align 4 // CHECK1-NEXT: [[TMP118:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 11 -// CHECK1-NEXT: store [3 x i32] zeroinitializer, ptr [[TMP118]], align 4 +// CHECK1-NEXT: store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP118]], align 4 // CHECK1-NEXT: [[TMP119:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 12 // CHECK1-NEXT: store i32 0, ptr [[TMP119]], align 4 -// CHECK1-NEXT: [[TMP120:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 0, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.region_id, ptr [[KERNEL_ARGS21]]) +// CHECK1-NEXT: [[TMP120:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.region_id, ptr [[KERNEL_ARGS21]]) // CHECK1-NEXT: [[TMP121:%.*]] = icmp ne i32 [[TMP120]], 0 // CHECK1-NEXT: br i1 [[TMP121]], label [[OMP_OFFLOAD_FAILED22:%.*]], label [[OMP_OFFLOAD_CONT23:%.*]] // CHECK1: omp_offload.failed22: @@ -899,64 +899,64 @@ int bar(int n){ // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META19:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META21:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META23:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !25 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !25 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META25:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META25]] // CHECK1-NEXT: call void [[TMP10]](ptr [[TMP11]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], align 8, !noalias !25 -// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]], align 8, !noalias !25 +// CHECK1-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], align 8, !noalias [[META25]] +// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP9]], i32 0, i32 1 // CHECK1-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP9]], i32 0, i32 2 // CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 // CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP17]], align 4 // CHECK1-NEXT: [[TMP20:%.*]] = insertvalue [3 x i32] zeroinitializer, i32 [[TMP18]], 0 // CHECK1-NEXT: [[TMP21:%.*]] = insertvalue [3 x i32] zeroinitializer, i32 [[TMP19]], 0 -// CHECK1-NEXT: store i32 2, ptr [[KERNEL_ARGS_I]], align 4, !noalias !25 +// CHECK1-NEXT: store i32 2, ptr [[KERNEL_ARGS_I]], align 4, !noalias [[META25]] // CHECK1-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 1 -// CHECK1-NEXT: store i32 3, ptr [[TMP22]], align 4, !noalias !25 +// CHECK1-NEXT: store i32 3, ptr [[TMP22]], align 4, !noalias [[META25]] // CHECK1-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 2 -// CHECK1-NEXT: store ptr [[TMP13]], ptr [[TMP23]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr [[TMP13]], ptr [[TMP23]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 3 -// CHECK1-NEXT: store ptr [[TMP14]], ptr [[TMP24]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr [[TMP14]], ptr [[TMP24]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 4 -// CHECK1-NEXT: store ptr [[TMP15]], ptr [[TMP25]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr [[TMP15]], ptr [[TMP25]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 5 -// CHECK1-NEXT: store ptr @.offload_maptypes, ptr [[TMP26]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr @.offload_maptypes, ptr [[TMP26]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 6 -// CHECK1-NEXT: store ptr null, ptr [[TMP27]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr null, ptr [[TMP27]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 7 -// CHECK1-NEXT: store ptr null, ptr [[TMP28]], align 8, !noalias !25 +// CHECK1-NEXT: store ptr null, ptr [[TMP28]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 8 -// CHECK1-NEXT: store i64 0, ptr [[TMP29]], align 8, !noalias !25 +// CHECK1-NEXT: store i64 0, ptr [[TMP29]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 9 -// CHECK1-NEXT: store i64 1, ptr [[TMP30]], align 8, !noalias !25 +// CHECK1-NEXT: store i64 1, ptr [[TMP30]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 10 -// CHECK1-NEXT: store [3 x i32] [[TMP20]], ptr [[TMP31]], align 4, !noalias !25 +// CHECK1-NEXT: store [3 x i32] [[TMP20]], ptr [[TMP31]], align 4, !noalias [[META25]] // CHECK1-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 11 -// CHECK1-NEXT: store [3 x i32] [[TMP21]], ptr [[TMP32]], align 4, !noalias !25 +// CHECK1-NEXT: store [3 x i32] [[TMP21]], ptr [[TMP32]], align 4, !noalias [[META25]] // CHECK1-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 12 -// CHECK1-NEXT: store i32 0, ptr [[TMP33]], align 4, !noalias !25 +// CHECK1-NEXT: store i32 0, ptr [[TMP33]], align 4, !noalias [[META25]] // CHECK1-NEXT: [[TMP34:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 [[TMP18]], i32 [[TMP19]], ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l101.region_id, ptr [[KERNEL_ARGS_I]]) // CHECK1-NEXT: [[TMP35:%.*]] = icmp ne i32 [[TMP34]], 0 // CHECK1-NEXT: br i1 [[TMP35]], label [[OMP_OFFLOAD_FAILED_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK1: omp_offload.failed.i: // CHECK1-NEXT: [[TMP36:%.*]] = load i16, ptr [[TMP12]], align 2 -// CHECK1-NEXT: store i16 [[TMP36]], ptr [[AA_CASTED_I]], align 2, !noalias !25 -// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[AA_CASTED_I]], align 8, !noalias !25 +// CHECK1-NEXT: store i16 [[TMP36]], ptr [[AA_CASTED_I]], align 2, !noalias [[META25]] +// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[AA_CASTED_I]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP16]], align 4 -// CHECK1-NEXT: store i32 [[TMP38]], ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias !25 -// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 8, !noalias !25 +// CHECK1-NEXT: store i32 [[TMP38]], ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias [[META25]] +// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 8, !noalias [[META25]] // CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP17]], align 4 -// CHECK1-NEXT: store i32 [[TMP40]], ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias !25 -// CHECK1-NEXT: [[TMP41:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 8, !noalias !25 +// CHECK1-NEXT: store i32 [[TMP40]], ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias [[META25]] +// CHECK1-NEXT: [[TMP41:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 8, !noalias [[META25]] // CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l101(i64 [[TMP37]], i64 [[TMP39]], i64 [[TMP41]]) #[[ATTR3]] // CHECK1-NEXT: br label [[DOTOMP_OUTLINED__EXIT]] // CHECK1: .omp_outlined..exit: @@ -1069,15 +1069,17 @@ int bar(int n){ // CHECK1-NEXT: [[AA_ADDR:%.*]] = alloca i64, align 8 // CHECK1-NEXT: [[A_CASTED:%.*]] = alloca i64, align 8 // CHECK1-NEXT: [[AA_CASTED:%.*]] = alloca i64, align 8 +// CHECK1-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) // CHECK1-NEXT: store i64 [[A]], ptr [[A_ADDR]], align 8 // CHECK1-NEXT: store i64 [[AA]], ptr [[AA_ADDR]], align 8 -// CHECK1-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4 -// CHECK1-NEXT: store i32 [[TMP0]], ptr [[A_CASTED]], align 4 -// CHECK1-NEXT: [[TMP1:%.*]] = load i64, ptr [[A_CASTED]], align 8 -// CHECK1-NEXT: [[TMP2:%.*]] = load i16, ptr [[AA_ADDR]], align 2 -// CHECK1-NEXT: store i16 [[TMP2]], ptr [[AA_CASTED]], align 2 -// CHECK1-NEXT: [[TMP3:%.*]] = load i64, ptr [[AA_CASTED]], align 8 -// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i64 [[TMP1]], i64 [[TMP3]]) +// CHECK1-NEXT: call void @__kmpc_push_num_teams(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 1) +// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4 +// CHECK1-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4 +// CHECK1-NEXT: [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8 +// CHECK1-NEXT: [[TMP3:%.*]] = load i16, ptr [[AA_ADDR]], align 2 +// CHECK1-NEXT: store i16 [[TMP3]], ptr [[AA_CASTED]], align 2 +// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[AA_CASTED]], align 8 +// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i64 [[TMP2]], i64 [[TMP4]]) // CHECK1-NEXT: ret void // // @@ -2180,12 +2182,12 @@ int bar(int n){ // CHECK3-NEXT: [[TMP114:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 9 // CHECK3-NEXT: store i64 0, ptr [[TMP114]], align 8 // CHECK3-NEXT: [[TMP115:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 10 -// CHECK3-NEXT: store [3 x i32] zeroinitializer, ptr [[TMP115]], align 4 +// CHECK3-NEXT: store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP115]], align 4 // CHECK3-NEXT: [[TMP116:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 11 -// CHECK3-NEXT: store [3 x i32] zeroinitializer, ptr [[TMP116]], align 4 +// CHECK3-NEXT: store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP116]], align 4 // CHECK3-NEXT: [[TMP117:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS21]], i32 0, i32 12 // CHECK3-NEXT: store i32 0, ptr [[TMP117]], align 4 -// CHECK3-NEXT: [[TMP118:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 0, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.region_id, ptr [[KERNEL_ARGS21]]) +// CHECK3-NEXT: [[TMP118:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.region_id, ptr [[KERNEL_ARGS21]]) // CHECK3-NEXT: [[TMP119:%.*]] = icmp ne i32 [[TMP118]], 0 // CHECK3-NEXT: br i1 [[TMP119]], label [[OMP_OFFLOAD_FAILED22:%.*]], label [[OMP_OFFLOAD_CONT23:%.*]] // CHECK3: omp_offload.failed22: @@ -2493,64 +2495,64 @@ int bar(int n){ // CHECK3-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META20:![0-9]+]]) // CHECK3-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META22:![0-9]+]]) // CHECK3-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META24:![0-9]+]]) -// CHECK3-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP10:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META26:![0-9]+]] +// CHECK3-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP10:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: call void [[TMP10]](ptr [[TMP11]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]]) #[[ATTR3]] -// CHECK3-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]], align 4, !noalias !26 +// CHECK3-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR2_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR3_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP9]], i32 0, i32 1 // CHECK3-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP9]], i32 0, i32 2 // CHECK3-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 // CHECK3-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP17]], align 4 // CHECK3-NEXT: [[TMP20:%.*]] = insertvalue [3 x i32] zeroinitializer, i32 [[TMP18]], 0 // CHECK3-NEXT: [[TMP21:%.*]] = insertvalue [3 x i32] zeroinitializer, i32 [[TMP19]], 0 -// CHECK3-NEXT: store i32 2, ptr [[KERNEL_ARGS_I]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 2, ptr [[KERNEL_ARGS_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 1 -// CHECK3-NEXT: store i32 3, ptr [[TMP22]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 3, ptr [[TMP22]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 2 -// CHECK3-NEXT: store ptr [[TMP13]], ptr [[TMP23]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr [[TMP13]], ptr [[TMP23]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 3 -// CHECK3-NEXT: store ptr [[TMP14]], ptr [[TMP24]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr [[TMP14]], ptr [[TMP24]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 4 -// CHECK3-NEXT: store ptr [[TMP15]], ptr [[TMP25]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr [[TMP15]], ptr [[TMP25]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 5 -// CHECK3-NEXT: store ptr @.offload_maptypes, ptr [[TMP26]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr @.offload_maptypes, ptr [[TMP26]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 6 -// CHECK3-NEXT: store ptr null, ptr [[TMP27]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr null, ptr [[TMP27]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 7 -// CHECK3-NEXT: store ptr null, ptr [[TMP28]], align 4, !noalias !26 +// CHECK3-NEXT: store ptr null, ptr [[TMP28]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 8 -// CHECK3-NEXT: store i64 0, ptr [[TMP29]], align 8, !noalias !26 +// CHECK3-NEXT: store i64 0, ptr [[TMP29]], align 8, !noalias [[META26]] // CHECK3-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 9 -// CHECK3-NEXT: store i64 1, ptr [[TMP30]], align 8, !noalias !26 +// CHECK3-NEXT: store i64 1, ptr [[TMP30]], align 8, !noalias [[META26]] // CHECK3-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 10 -// CHECK3-NEXT: store [3 x i32] [[TMP20]], ptr [[TMP31]], align 4, !noalias !26 +// CHECK3-NEXT: store [3 x i32] [[TMP20]], ptr [[TMP31]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 11 -// CHECK3-NEXT: store [3 x i32] [[TMP21]], ptr [[TMP32]], align 4, !noalias !26 +// CHECK3-NEXT: store [3 x i32] [[TMP21]], ptr [[TMP32]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS_I]], i32 0, i32 12 -// CHECK3-NEXT: store i32 0, ptr [[TMP33]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 0, ptr [[TMP33]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP34:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 [[TMP18]], i32 [[TMP19]], ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l101.region_id, ptr [[KERNEL_ARGS_I]]) // CHECK3-NEXT: [[TMP35:%.*]] = icmp ne i32 [[TMP34]], 0 // CHECK3-NEXT: br i1 [[TMP35]], label [[OMP_OFFLOAD_FAILED_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK3: omp_offload.failed.i: // CHECK3-NEXT: [[TMP36:%.*]] = load i16, ptr [[TMP12]], align 2 -// CHECK3-NEXT: store i16 [[TMP36]], ptr [[AA_CASTED_I]], align 2, !noalias !26 -// CHECK3-NEXT: [[TMP37:%.*]] = load i32, ptr [[AA_CASTED_I]], align 4, !noalias !26 +// CHECK3-NEXT: store i16 [[TMP36]], ptr [[AA_CASTED_I]], align 2, !noalias [[META26]] +// CHECK3-NEXT: [[TMP37:%.*]] = load i32, ptr [[AA_CASTED_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP16]], align 4 -// CHECK3-NEXT: store i32 [[TMP38]], ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 [[TMP38]], ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR__CASTED_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP17]], align 4 -// CHECK3-NEXT: store i32 [[TMP40]], ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias !26 -// CHECK3-NEXT: [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias !26 +// CHECK3-NEXT: store i32 [[TMP40]], ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias [[META26]] +// CHECK3-NEXT: [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR__CASTED4_I]], align 4, !noalias [[META26]] // CHECK3-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l101(i32 [[TMP37]], i32 [[TMP39]], i32 [[TMP41]]) #[[ATTR3]] // CHECK3-NEXT: br label [[DOTOMP_OUTLINED__EXIT]] // CHECK3: .omp_outlined..exit: @@ -2663,15 +2665,17 @@ int bar(int n){ // CHECK3-NEXT: [[AA_ADDR:%.*]] = alloca i32, align 4 // CHECK3-NEXT: [[A_CASTED:%.*]] = alloca i32, align 4 // CHECK3-NEXT: [[AA_CASTED:%.*]] = alloca i32, align 4 +// CHECK3-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) // CHECK3-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4 // CHECK3-NEXT: store i32 [[AA]], ptr [[AA_ADDR]], align 4 -// CHECK3-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4 -// CHECK3-NEXT: store i32 [[TMP0]], ptr [[A_CASTED]], align 4 -// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_CASTED]], align 4 -// CHECK3-NEXT: [[TMP2:%.*]] = load i16, ptr [[AA_ADDR]], align 2 -// CHECK3-NEXT: store i16 [[TMP2]], ptr [[AA_CASTED]], align 2 -// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[AA_CASTED]], align 4 -// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i32 [[TMP1]], i32 [[TMP3]]) +// CHECK3-NEXT: call void @__kmpc_push_num_teams(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 1) +// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4 +// CHECK3-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4 +// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4 +// CHECK3-NEXT: [[TMP3:%.*]] = load i16, ptr [[AA_ADDR]], align 2 +// CHECK3-NEXT: store i16 [[TMP3]], ptr [[AA_CASTED]], align 2 +// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[AA_CASTED]], align 4 +// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i32 [[TMP2]], i32 [[TMP4]]) // CHECK3-NEXT: ret void // // @@ -3613,16 +3617,18 @@ int bar(int n){ // CHECK9-NEXT: [[AA_ADDR:%.*]] = alloca i64, align 8 // CHECK9-NEXT: [[A_CASTED:%.*]] = alloca i64, align 8 // CHECK9-NEXT: [[AA_CASTED:%.*]] = alloca i64, align 8 +// CHECK9-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) // CHECK9-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8 // CHECK9-NEXT: store i64 [[A]], ptr [[A_ADDR]], align 8 // CHECK9-NEXT: store i64 [[AA]], ptr [[AA_ADDR]], align 8 -// CHECK9-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4 -// CHECK9-NEXT: store i32 [[TMP0]], ptr [[A_CASTED]], align 4 -// CHECK9-NEXT: [[TMP1:%.*]] = load i64, ptr [[A_CASTED]], align 8 -// CHECK9-NEXT: [[TMP2:%.*]] = load i16, ptr [[AA_ADDR]], align 2 -// CHECK9-NEXT: store i16 [[TMP2]], ptr [[AA_CASTED]], align 2 -// CHECK9-NEXT: [[TMP3:%.*]] = load i64, ptr [[AA_CASTED]], align 8 -// CHECK9-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i64 [[TMP1]], i64 [[TMP3]]) +// CHECK9-NEXT: call void @__kmpc_push_num_teams(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 1) +// CHECK9-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4 +// CHECK9-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4 +// CHECK9-NEXT: [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8 +// CHECK9-NEXT: [[TMP3:%.*]] = load i16, ptr [[AA_ADDR]], align 2 +// CHECK9-NEXT: store i16 [[TMP3]], ptr [[AA_CASTED]], align 2 +// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[AA_CASTED]], align 8 +// CHECK9-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i64 [[TMP2]], i64 [[TMP4]]) // CHECK9-NEXT: ret void // // @@ -4172,16 +4178,18 @@ int bar(int n){ // CHECK11-NEXT: [[AA_ADDR:%.*]] = alloca i32, align 4 // CHECK11-NEXT: [[A_CASTED:%.*]] = alloca i32, align 4 // CHECK11-NEXT: [[AA_CASTED:%.*]] = alloca i32, align 4 +// CHECK11-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) // CHECK11-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 4 // CHECK11-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4 // CHECK11-NEXT: store i32 [[AA]], ptr [[AA_ADDR]], align 4 -// CHECK11-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4 -// CHECK11-NEXT: store i32 [[TMP0]], ptr [[A_CASTED]], align 4 -// CHECK11-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_CASTED]], align 4 -// CHECK11-NEXT: [[TMP2:%.*]] = load i16, ptr [[AA_ADDR]], align 2 -// CHECK11-NEXT: store i16 [[TMP2]], ptr [[AA_CASTED]], align 2 -// CHECK11-NEXT: [[TMP3:%.*]] = load i32, ptr [[AA_CASTED]], align 4 -// CHECK11-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i32 [[TMP1]], i32 [[TMP3]]) +// CHECK11-NEXT: call void @__kmpc_push_num_teams(ptr @[[GLOB1]], i32 [[TMP0]], i32 1, i32 1) +// CHECK11-NEXT: [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4 +// CHECK11-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4 +// CHECK11-NEXT: [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4 +// CHECK11-NEXT: [[TMP3:%.*]] = load i16, ptr [[AA_ADDR]], align 2 +// CHECK11-NEXT: store i16 [[TMP3]], ptr [[AA_CASTED]], align 2 +// CHECK11-NEXT: [[TMP4:%.*]] = load i32, ptr [[AA_CASTED]], align 4 +// CHECK11-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB1]], i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l124.omp_outlined, i32 [[TMP2]], i32 [[TMP4]]) // CHECK11-NEXT: ret void // // From 9d25b28b9e412b66bb58993dff6646a8f08b8c8c Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 16:25:58 +0100 Subject: [PATCH 131/884] [InstCombine] Explicitly canonicalize splat shuffles to use poison RHS This is usually handled by demanded elements simplification. However, as that is not supported for scalable vectors, also handle it explicitly here. --- llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp | 5 +++++ llvm/test/Transforms/InstCombine/vec_shuffle.ll | 2 +- llvm/test/Transforms/InstCombine/vscale_extractelement.ll | 4 ++-- llvm/test/Transforms/InstCombine/vscale_insertelement.ll | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c18fd5d99d097..7a9a0950c2b6b 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -2774,6 +2774,11 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { if (Instruction *I = simplifyBinOpSplats(SVI)) return I; + // Canonicalize splat shuffle to use poison RHS. Handle this explicitly in + // order to support scalable vectors. + if (match(SVI.getShuffleMask(), m_ZeroMask()) && !isa(RHS)) + return replaceOperand(SVI, 1, PoisonValue::get(RHS->getType())); + if (isa(LHS->getType())) return nullptr; diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index bef201196e615..0c21dc88f8569 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -67,7 +67,7 @@ define float @test6(<4 x float> %X) { define float @testvscale6( %X) { ; CHECK-LABEL: @testvscale6( -; CHECK-NEXT: [[T2:%.*]] = shufflevector [[X:%.*]], undef, zeroinitializer +; CHECK-NEXT: [[T2:%.*]] = shufflevector [[X:%.*]], poison, zeroinitializer ; CHECK-NEXT: [[R:%.*]] = extractelement [[T2]], i64 0 ; CHECK-NEXT: ret float [[R]] ; diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll index 784229c948a6d..f696ff469e205 100644 --- a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll +++ b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll @@ -56,7 +56,7 @@ define i8 @extractelement_bitcast_useless_insert( %a, i32 %x) define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) { ; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range( ; CHECK-NEXT: [[IN:%.*]] = insertelement undef, i32 [[V:%.*]], i64 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], undef, zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], poison, zeroinitializer ; CHECK-NEXT: [[R:%.*]] = extractelement [[SPLAT]], i64 4 ; CHECK-NEXT: ret i32 [[R]] ; @@ -69,7 +69,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) { define i32 @extractelement_shuffle_invalid_index(i32 %v) { ; CHECK-LABEL: @extractelement_shuffle_invalid_index( ; CHECK-NEXT: [[IN:%.*]] = insertelement undef, i32 [[V:%.*]], i64 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], undef, zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], poison, zeroinitializer ; CHECK-NEXT: [[R:%.*]] = extractelement [[SPLAT]], i64 4294967295 ; CHECK-NEXT: ret i32 [[R]] ; diff --git a/llvm/test/Transforms/InstCombine/vscale_insertelement.ll b/llvm/test/Transforms/InstCombine/vscale_insertelement.ll index e9dc7f75eb543..db22b3b657fb3 100644 --- a/llvm/test/Transforms/InstCombine/vscale_insertelement.ll +++ b/llvm/test/Transforms/InstCombine/vscale_insertelement.ll @@ -89,7 +89,7 @@ define @insertelement_sequene_may_not_be_splat(float %x) { define void @ossfuzz_27416(i32 %v, ptr %p) { ; CHECK-LABEL: @ossfuzz_27416( ; CHECK-NEXT: [[IN:%.*]] = insertelement undef, i32 [[V:%.*]], i64 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], undef, zeroinitializer +; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[IN]], poison, zeroinitializer ; CHECK-NEXT: [[I1:%.*]] = insertelement [[SPLAT]], i32 undef, i64 128 ; CHECK-NEXT: store [[I1]], ptr [[P:%.*]], align 16 ; CHECK-NEXT: ret void From 8a233d8cfde4cd026b5dd12b56ea029749a02329 Mon Sep 17 00:00:00 2001 From: Abhina Sree <69635948+abhina-sree@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:33:23 -0500 Subject: [PATCH 132/884] [SystemZ][z/OS] Add guard for dl_info and dladdr (#75637) This patch fixes the following build error on z/OS `error: unknown type name 'Dl_info'` by adding a guard to check if we have dladdr. --- clang/tools/libclang/CIndexer.cpp | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/clang/tools/libclang/CIndexer.cpp b/clang/tools/libclang/CIndexer.cpp index 77da2e4fa5ead..0623ae69fe01e 100644 --- a/clang/tools/libclang/CIndexer.cpp +++ b/clang/tools/libclang/CIndexer.cpp @@ -17,6 +17,7 @@ #include "clang/Driver/Driver.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Config/config.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MD5.h" #include "llvm/Support/Path.h" @@ -125,21 +126,28 @@ const std::string &CIndexer::getClangResourcesPath() { #elif defined(_AIX) getClangResourcesPathImplAIX(LibClangPath); #else + bool PathFound = false; +#if defined(HAVE_DLFCN_H) && defined(HAVE_DLADDR) Dl_info info; - std::string Path; // This silly cast below avoids a C++ warning. if (dladdr((void *)(uintptr_t)clang_createTranslationUnit, &info) != 0) { // We now have the CIndex directory, locate clang relative to it. LibClangPath += info.dli_fname; - } else if (!(Path = llvm::sys::fs::getMainExecutable(nullptr, nullptr)).empty()) { - // If we can't get the path using dladdr, try to get the main executable - // path. This may be needed when we're statically linking libclang with - // musl libc, for example. - LibClangPath += Path; - } else { - // It's rather unlikely we end up here. But it could happen, so report an - // error instead of crashing. - llvm::report_fatal_error("could not locate Clang resource path"); + PathFound = true; + } +#endif + std::string Path; + if (!PathFound) { + if (!(Path = llvm::sys::fs::getMainExecutable(nullptr, nullptr)).empty()) { + // If we can't get the path using dladdr, try to get the main executable + // path. This may be needed when we're statically linking libclang with + // musl libc, for example. + LibClangPath += Path; + } else { + // It's rather unlikely we end up here. But it could happen, so report an + // error instead of crashing. + llvm::report_fatal_error("could not locate Clang resource path"); + } } #endif From ddd11537e2af5596e8a1d213ed6f939a3ecfc72b Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 16:39:06 +0100 Subject: [PATCH 133/884] [InstCombine] Match poison instead of undef in binop of same-mask shuffle fold --- llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 94f60719b78ca..20ce5e0ede9e5 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1728,8 +1728,8 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { // If both arguments of the binary operation are shuffles that use the same // mask and shuffle within a single vector, move the shuffle after the binop. - if (match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))) && - match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) && + if (match(LHS, m_Shuffle(m_Value(V1), m_Poison(), m_Mask(Mask))) && + match(RHS, m_Shuffle(m_Value(V2), m_Poison(), m_SpecificMask(Mask))) && V1->getType() == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) { // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask) From fd527def7edef8f8754b9e7812ed77f486bb144f Mon Sep 17 00:00:00 2001 From: Momchil Velikov Date: Mon, 18 Dec 2023 15:52:28 +0000 Subject: [PATCH 134/884] [Clang][SVE2.1] Add floating-point variants of `svrevd_XX` (#75117) --- clang/include/clang/Basic/arm_sve.td | 4 +- .../aarch64-sve2-intrinsics/acle_sve2_revd.c | 198 +++++++++++++++++- llvm/lib/Target/AArch64/SMEInstrFormats.td | 6 + .../CodeGen/AArch64/sve2-intrinsics-revd.ll | 41 ++++ 4 files changed, 245 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index a429a3c5fe378..cbc2af73d6052 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -2077,11 +2077,11 @@ def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aar def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; } -let TargetGuard = "sve2p1" in { +let TargetGuard = "sve2p1|sme" in { def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; -defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">; +defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; } let TargetGuard = "sve2p1|sme2" in { diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c index 9d912c5d9e276..74a90583a173a 100644 --- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c +++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c @@ -1,14 +1,15 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ -// RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: -target-feature +sme -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK - +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s #include #ifdef SVE_OVERLOADED_FORMS @@ -388,3 +389,196 @@ svuint32_t test_svrevd_u32_x(svbool_t pg, svuint32_t op) { svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) { return SVE_ACLE_FUNC(svrevd, _u64, _x, )(pg, op); } + + +// CHECK-LABEL: @test_svrevd_bf16_z( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z18test_svrevd_bf16_zu10__SVBool_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svbfloat16_t test_svrevd_bf16_z(svbool_t pg, svbfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _bf16, _z, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f16_z( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f16_zu10__SVBool_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat16_t test_svrevd_f16_z(svbool_t pg, svfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _f16, _z, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f32_z( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f32_zu10__SVBool_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat32_t test_svrevd_f32_z(svbool_t pg, svfloat32_t op) { + return SVE_ACLE_FUNC(svrevd, _f32, _z, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f64_z( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f64_zu10__SVBool_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( zeroinitializer, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat64_t test_svrevd_f64_z(svbool_t pg, svfloat64_t op) { + return SVE_ACLE_FUNC(svrevd, _f64, _z, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_bf16_m( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z18test_svrevd_bf16_mu14__SVBfloat16_tu10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svbfloat16_t test_svrevd_bf16_m(svbfloat16_t inactive, svbool_t pg, svbfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _bf16, _m, )(inactive, pg, op); +} + +// CHECK-LABEL: @test_svrevd_f16_m( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f16_mu13__SVFloat16_tu10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat16_t test_svrevd_f16_m(svfloat16_t inactive, svbool_t pg, svfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _f16, _m, )(inactive, pg, op); +} + +// CHECK-LABEL: @test_svrevd_f32_m( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f32_mu13__SVFloat32_tu10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat32_t test_svrevd_f32_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op) { + return SVE_ACLE_FUNC(svrevd, _f32, _m, )(inactive, pg, op); +} + +// CHECK-LABEL: @test_svrevd_f64_m( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f64_mu13__SVFloat64_tu10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( [[INACTIVE:%.*]], [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat64_t test_svrevd_f64_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op) { + return SVE_ACLE_FUNC(svrevd, _f64, _m, )(inactive, pg, op); +} + +// CHECK-LABEL: @test_svrevd_bf16_x( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( undef, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z18test_svrevd_bf16_xu10__SVBool_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8bf16( undef, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svbfloat16_t test_svrevd_bf16_x(svbool_t pg, svbfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _bf16, _x, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f16_x( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( undef, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f16_xu10__SVBool_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv8f16( undef, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat16_t test_svrevd_f16_x(svbool_t pg, svfloat16_t op) { + return SVE_ACLE_FUNC(svrevd, _f16, _x, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f32_x( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( undef, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f32_xu10__SVBool_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv4f32( undef, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat32_t test_svrevd_f32_x(svbool_t pg, svfloat32_t op) { + return SVE_ACLE_FUNC(svrevd, _f32, _x, )(pg, op); +} + +// CHECK-LABEL: @test_svrevd_f64_x( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( undef, [[TMP0]], [[OP:%.*]]) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svrevd_f64_xu10__SVBool_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.revd.nxv2f64( undef, [[TMP0]], [[OP:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svfloat64_t test_svrevd_f64_x(svbool_t pg, svfloat64_t op) { + return SVE_ACLE_FUNC(svrevd, _f64, _x, )(pg, op); +} diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td index 4f8917618ea40..659e2ddc5d543 100644 --- a/llvm/lib/Target/AArch64/SMEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td @@ -1259,6 +1259,12 @@ multiclass sve2_int_perm_revd { def : SVE_1_Op_Passthru_Pat(NAME)>; def : SVE_1_Op_Passthru_Pat(NAME)>; def : SVE_1_Op_Passthru_Pat(NAME)>; + + def : SVE_1_Op_Passthru_Pat(NAME)>; + def : SVE_1_Op_Passthru_Pat(NAME)>; + def : SVE_1_Op_Passthru_Pat(NAME)>; + def : SVE_1_Op_Passthru_Pat(NAME)>; + } class sve2_clamp sz, bit U, ZPRRegOp zpr_ty> diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll index 87b15baffa0d2..dd02e1a89e9ac 100644 --- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll +++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll @@ -37,7 +37,48 @@ define @test_revd_i64( %a, %res } +define @test_revd_bf16( %a, %pg, %b) { +; CHECK-LABEL: test_revd_bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: revd z0.q, p0/m, z1.q +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.revd.nxv8bf16( %a, %pg, %b) + ret %res +} + +define @test_revd_f16( %a, %pg, %b) { +; CHECK-LABEL: test_revd_f16: +; CHECK: // %bb.0: +; CHECK-NEXT: revd z0.q, p0/m, z1.q +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.revd.nxv8f16( %a, %pg, %b) + ret %res +} + +define @test_revd_f32( %a, %pg, %b) { +; CHECK-LABEL: test_revd_f32: +; CHECK: // %bb.0: +; CHECK-NEXT: revd z0.q, p0/m, z1.q +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.revd.nxv4f32( %a, %pg, %b) + ret %res +} + +define @test_revd_f64( %a, %pg, %b) { +; CHECK-LABEL: test_revd_f64: +; CHECK: // %bb.0: +; CHECK-NEXT: revd z0.q, p0/m, z1.q +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.revd.nxv2f64( %a, %pg, %b) + ret %res +} + declare @llvm.aarch64.sve.revd.nxv16i8(, , ) declare @llvm.aarch64.sve.revd.nxv8i16(, , ) declare @llvm.aarch64.sve.revd.nxv4i32(, , ) declare @llvm.aarch64.sve.revd.nxv2i64(, , ) + +declare @llvm.aarch64.sve.revd.nxv8bf16(, , ) +declare @llvm.aarch64.sve.revd.nxv8f16(, , ) +declare @llvm.aarch64.sve.revd.nxv4f32(, , ) +declare @llvm.aarch64.sve.revd.nxv2f64(, , ) From cd54c47424456a4dcf8161f3377234cbfa459e88 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 16:49:30 +0100 Subject: [PATCH 135/884] [InstCombine] Match poison instead of undef in foldVectorBinop() Some negative tests turn into positive tests, as the differences between undef and poison propagation allow additional transforms. --- .../InstCombine/InstructionCombining.cpp | 37 ++++----- .../InstCombine/vec_shuffle-inseltpoison.ll | 78 ++++++++----------- .../Transforms/InstCombine/vec_shuffle.ll | 75 ++++++++---------- 3 files changed, 81 insertions(+), 109 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 20ce5e0ede9e5..4188b5b46e87e 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1770,9 +1770,9 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { Constant *C; auto *InstVTy = dyn_cast(Inst.getType()); if (InstVTy && - match(&Inst, - m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))), - m_ImmConstant(C))) && + match(&Inst, m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Poison(), + m_Mask(Mask))), + m_ImmConstant(C))) && cast(V1->getType())->getNumElements() <= InstVTy->getNumElements()) { assert(InstVTy->getScalarType() == V1->getType()->getScalarType() && @@ -1787,8 +1787,8 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { ArrayRef ShMask = Mask; unsigned SrcVecNumElts = cast(V1->getType())->getNumElements(); - UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType()); - SmallVector NewVecC(SrcVecNumElts, UndefScalar); + PoisonValue *PoisonScalar = PoisonValue::get(C->getType()->getScalarType()); + SmallVector NewVecC(SrcVecNumElts, PoisonScalar); bool MayChange = true; unsigned NumElts = InstVTy->getNumElements(); for (unsigned I = 0; I < NumElts; ++I) { @@ -1801,29 +1801,29 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { // 2. The shuffle needs an element of the constant vector that can't // be mapped to a new constant vector. // 3. This is a widening shuffle that copies elements of V1 into the - // extended elements (extending with undef is allowed). - if (!CElt || (!isa(NewCElt) && NewCElt != CElt) || + // extended elements (extending with poison is allowed). + if (!CElt || (!isa(NewCElt) && NewCElt != CElt) || I >= SrcVecNumElts) { MayChange = false; break; } NewVecC[ShMask[I]] = CElt; } - // If this is a widening shuffle, we must be able to extend with undef - // elements. If the original binop does not produce an undef in the high + // If this is a widening shuffle, we must be able to extend with poison + // elements. If the original binop does not produce a poison in the high // lanes, then this transform is not safe. - // Similarly for undef lanes due to the shuffle mask, we can only - // transform binops that preserve undef. - // TODO: We could shuffle those non-undef constant values into the - // result by using a constant vector (rather than an undef vector) + // Similarly for poison lanes due to the shuffle mask, we can only + // transform binops that preserve poison. + // TODO: We could shuffle those non-poison constant values into the + // result by using a constant vector (rather than an poison vector) // as operand 1 of the new binop, but that might be too aggressive // for target-independent shuffle creation. if (I >= SrcVecNumElts || ShMask[I] < 0) { - Constant *MaybeUndef = + Constant *MaybePoison = ConstOp1 - ? ConstantFoldBinaryOpOperands(Opcode, UndefScalar, CElt, DL) - : ConstantFoldBinaryOpOperands(Opcode, CElt, UndefScalar, DL); - if (!MaybeUndef || !match(MaybeUndef, m_Undef())) { + ? ConstantFoldBinaryOpOperands(Opcode, PoisonScalar, CElt, DL) + : ConstantFoldBinaryOpOperands(Opcode, CElt, PoisonScalar, DL); + if (!MaybePoison || !isa(MaybePoison)) { MayChange = false; break; } @@ -1831,9 +1831,10 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { } if (MayChange) { Constant *NewC = ConstantVector::get(NewVecC); - // It may not be safe to execute a binop on a vector with undef elements + // It may not be safe to execute a binop on a vector with poison elements // because the entire instruction can be folded to undef or create poison // that did not exist in the original code. + // TODO: The shift case should not be necessary. if (Inst.isIntDivRem() || (Inst.isShift() && ConstOp1)) NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1); diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll index bedabf4e3456d..81f0a966dbc96 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll @@ -557,7 +557,7 @@ define <4 x i32> @mul_const_splat(<4 x i32> %v) { define <4 x i32> @lshr_const_half_splat(<4 x i32> %v) { ; CHECK-LABEL: @lshr_const_half_splat( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> , [[V:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> , [[V:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[R]] ; @@ -659,13 +659,12 @@ define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) { ret <4 x i16> %bo } -; A binop that does not produce undef in the high lanes can not be moved before the shuffle. -; This is not ok because 'shl undef, 1 (or 2)' --> 0' but moving the shuffle results in undef instead. +; This is valid for poison, but would not be valid for undef. define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> -; CHECK-NEXT: [[BO:%.*]] = shl <4 x i16> [[SHUF]], +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], +; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[BO]] ; %shuf = shufflevector <2 x i16> %v, <2 x i16> poison, <4 x i32> @@ -673,13 +672,10 @@ define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { ret <4 x i16> %bo } -; A binop that does not produce undef in the high lanes can not be moved before the shuffle. -; This is not ok because 'or -1, undef --> -1' but moving the shuffle results in undef instead. - define <4 x i16> @widening_shuffle_or(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_or( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> -; CHECK-NEXT: [[BO:%.*]] = or <4 x i16> [[SHUF]], +; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i16> [[V:%.*]], +; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[BO]] ; %shuf = shufflevector <2 x i16> %v, <2 x i16> poison, <4 x i32> @@ -856,7 +852,7 @@ define <2 x i32> @mul_splat_constant(<2 x i32> %x) { define <2 x i32> @shl_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @shl_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -878,7 +874,7 @@ define <2 x i32> @shl_splat_constant1(<2 x i32> %x) { define <2 x i32> @ashr_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @ashr_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -900,7 +896,7 @@ define <2 x i32> @ashr_splat_constant1(<2 x i32> %x) { define <2 x i32> @lshr_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @lshr_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -1019,13 +1015,10 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) { ret <2 x i32> %r } -; AND does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef( +define <4 x i16> @and_constant_mask_poison(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[AND]] ; entry: @@ -1034,13 +1027,10 @@ entry: ret <4 x i16> %and } -; AND does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_2( +define <4 x i16> @and_constant_mask_poison_2(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[AND]] ; entry: @@ -1050,8 +1040,8 @@ entry: } ; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes. -define <4 x i16> @and_constant_mask_undef_3(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_3( +define <4 x i16> @and_constant_mask_poison_3(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_3( ; CHECK-NEXT: entry: ; CHECK-NEXT: ret <4 x i16> ; @@ -1062,8 +1052,8 @@ entry: } ; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes. -define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_4( +define <4 x i16> @and_constant_mask_poison_4(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_4( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], ; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> @@ -1088,13 +1078,10 @@ entry: ret <4 x i16> %and } -; OR does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef( +define <4 x i16> @or_constant_mask_poison(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[OR]] ; entry: @@ -1103,13 +1090,10 @@ entry: ret <4 x i16> %or } -; OR does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_2( +define <4 x i16> @or_constant_mask_poison_2(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[OR]] ; entry: @@ -1119,8 +1103,8 @@ entry: } ; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes. -define <4 x i16> @or_constant_mask_undef_3(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_3( +define <4 x i16> @or_constant_mask_poison_3(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_3( ; CHECK-NEXT: entry: ; CHECK-NEXT: ret <4 x i16> ; @@ -1131,8 +1115,8 @@ entry: } ; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes. -define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_4( +define <4 x i16> @or_constant_mask_poison_4(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_4( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], ; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> @@ -1160,8 +1144,8 @@ entry: define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) { ; CHECK-LABEL: @shl_constant_mask_undef( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i16> [[IN:%.*]], +; CHECK-NEXT: [[SHL:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[SHL]] ; entry: diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index 0c21dc88f8569..8eff837d6e1a3 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -564,7 +564,7 @@ define <4 x i32> @mul_const_splat(<4 x i32> %v) { define <4 x i32> @lshr_const_half_splat(<4 x i32> %v) { ; CHECK-LABEL: @lshr_const_half_splat( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> , [[V:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> , [[V:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[R]] ; @@ -666,13 +666,12 @@ define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) { ret <4 x i16> %bo } -; A binop that does not produce undef in the high lanes can not be moved before the shuffle. -; This is not ok because 'shl undef, 1 (or 2)' --> 0' but moving the shuffle results in undef instead. +; This is valid for poison, but would not be valid for undef. define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> -; CHECK-NEXT: [[BO:%.*]] = shl <4 x i16> [[SHUF]], +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], +; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[BO]] ; %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> @@ -685,8 +684,8 @@ define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) { define <4 x i16> @widening_shuffle_or(<2 x i16> %v) { ; CHECK-LABEL: @widening_shuffle_or( -; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> -; CHECK-NEXT: [[BO:%.*]] = or <4 x i16> [[SHUF]], +; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i16> [[V:%.*]], +; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[BO]] ; %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> @@ -863,7 +862,7 @@ define <2 x i32> @mul_splat_constant(<2 x i32> %x) { define <2 x i32> @shl_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @shl_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -885,7 +884,7 @@ define <2 x i32> @shl_splat_constant1(<2 x i32> %x) { define <2 x i32> @ashr_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @ashr_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -907,7 +906,7 @@ define <2 x i32> @ashr_splat_constant1(<2 x i32> %x) { define <2 x i32> @lshr_splat_constant0(<2 x i32> %x) { ; CHECK-LABEL: @lshr_splat_constant0( -; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> , [[X:%.*]] ; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: ret <2 x i32> [[R]] ; @@ -1026,13 +1025,10 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) { ret <2 x i32> %r } -; AND does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef( +define <4 x i16> @and_constant_mask_poison(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[AND]] ; entry: @@ -1041,13 +1037,10 @@ entry: ret <4 x i16> %and } -; AND does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_2( +define <4 x i16> @and_constant_mask_poison_2(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[AND]] ; entry: @@ -1057,8 +1050,8 @@ entry: } ; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes. -define <4 x i16> @and_constant_mask_undef_3(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_3( +define <4 x i16> @and_constant_mask_poison_3(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_3( ; CHECK-NEXT: entry: ; CHECK-NEXT: ret <4 x i16> ; @@ -1069,8 +1062,8 @@ entry: } ; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes. -define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) { -; CHECK-LABEL: @and_constant_mask_undef_4( +define <4 x i16> @and_constant_mask_poison_4(<4 x i16> %add) { +; CHECK-LABEL: @and_constant_mask_poison_4( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], ; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> @@ -1095,13 +1088,10 @@ entry: ret <4 x i16> %and } -; OR does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef( +define <4 x i16> @or_constant_mask_poison(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[OR]] ; entry: @@ -1110,13 +1100,10 @@ entry: ret <4 x i16> %or } -; OR does not fold to undef for undef operands, we cannot move it -; across a shuffle with undef masks. -define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_2( +define <4 x i16> @or_constant_mask_poison_2(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[OR]] ; entry: @@ -1126,8 +1113,8 @@ entry: } ; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes. -define <4 x i16> @or_constant_mask_undef_3(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_3( +define <4 x i16> @or_constant_mask_poison_3(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_3( ; CHECK-NEXT: entry: ; CHECK-NEXT: ret <4 x i16> ; @@ -1138,8 +1125,8 @@ entry: } ; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes. -define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) { -; CHECK-LABEL: @or_constant_mask_undef_4( +define <4 x i16> @or_constant_mask_poison_4(<4 x i16> %in) { +; CHECK-LABEL: @or_constant_mask_poison_4( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], ; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> @@ -1167,8 +1154,8 @@ entry: define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) { ; CHECK-LABEL: @shl_constant_mask_undef( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> -; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], +; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i16> [[IN:%.*]], +; CHECK-NEXT: [[SHL:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i16> [[SHL]] ; entry: From d0285a31c85a9931ee0d9cbb8486f313dc21ada9 Mon Sep 17 00:00:00 2001 From: Nathan Sidwell Date: Mon, 18 Dec 2023 11:02:44 -0500 Subject: [PATCH 136/884] aarch64: fix testcase (#75723) Add missing < %s to RUN line. --- .../CodeGen/AArch64/stack-probing-dynamic-no-frame-setup.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/AArch64/stack-probing-dynamic-no-frame-setup.ll b/llvm/test/CodeGen/AArch64/stack-probing-dynamic-no-frame-setup.ll index 96f2f63d703c7..e1303285ee6b6 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-dynamic-no-frame-setup.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-dynamic-no-frame-setup.ll @@ -1,4 +1,4 @@ -; RUN: llc --stop-after=finalize-isel -o - | FileCheck %s +; RUN: llc < %s --stop-after=finalize-isel -o - | FileCheck %s target triple = "aarch64-linux" ; Check dynamic stack allocation and probing instructions do not have From 4d9d105c7053d67507a3ab1e7cf34bf6d521cc44 Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Mon, 18 Dec 2023 17:11:21 +0100 Subject: [PATCH 137/884] [mlir] fix filecheck prefixes in a dataflow test (#75794) -SAME and -LITERAL do not compose in CHECK commands. --- .../Analysis/DataFlow/test-next-access.mlir | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/mlir/test/Analysis/DataFlow/test-next-access.mlir b/mlir/test/Analysis/DataFlow/test-next-access.mlir index de0788fb6a176..70069b10a9398 100644 --- a/mlir/test/Analysis/DataFlow/test-next-access.mlir +++ b/mlir/test/Analysis/DataFlow/test-next-access.mlir @@ -398,29 +398,29 @@ func.func @conditonal_call(%arg0: memref, %cond: i1) { // "caller" -> "call" -> "callee" -> "post" func.func private @callee(%arg0: memref) { - // IP: name = "callee" - // IP-SAME-LITERAL: next_access = [["post"]] - // LOCAL: name = "callee" - // LOCAL-SAME: next_access = ["unknown"] + // IP: name = "callee" + // IP-SAME: next_access = {{\[}}["post"]] + // LOCAL: name = "callee" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee"} : memref return } // CHECK-LABEL: @call_and_store_before func.func @call_and_store_before(%arg0: memref) { - // IP: name = "caller" - // IP-SAME-LITERAL: next_access = [["call"]] - // LOCAL: name = "caller" - // LOCAL-SAME: next_access = ["unknown"] - // LC_AR: name = "caller" - // LC_AR-SAME: next_access = {{\[}}["call"]] + // IP: name = "caller" + // IP-SAME: next_access = {{\[}}["call"]] + // LOCAL: name = "caller" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "caller" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "caller"} : memref // Note that the access after the entire call is "post". - // CHECK: name = "call" - // CHECK-SAME-LITERAL: next_access = [["post"], ["post"]] + // CHECK: name = "call" + // CHECK-SAME: next_access = {{\[}}["post"], ["post"]] test.call_and_store @callee(%arg0), %arg0 {name = "call", store_before_call = true} : (memref, memref) -> () - // CHECK: name = "post" - // CHECK-SAME-LITERAL: next_access = ["unknown"] + // CHECK: name = "post" + // CHECK-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "post"} : memref return } @@ -432,28 +432,28 @@ func.func @call_and_store_before(%arg0: memref) { // "caller" -> "callee" -> "call" -> "post" func.func private @callee(%arg0: memref) { - // IP: name = "callee" - // IP-SAME-LITERAL: next_access = [["call"]] - // LOCAL: name = "callee" - // LOCAL-SAME: next_access = ["unknown"] + // IP: name = "callee" + // IP-SAME: next_access = {{\[}}["call"]] + // LOCAL: name = "callee" + // LOCAL-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "callee"} : memref return } // CHECK-LABEL: @call_and_store_after func.func @call_and_store_after(%arg0: memref) { - // IP: name = "caller" - // IP-SAME-LITERAL: next_access = [["callee"]] - // LOCAL: name = "caller" - // LOCAL-SAME: next_access = ["unknown"] - // LC_AR: name = "caller" - // LC_AR-SAME: next_access = {{\[}}["call"]] + // IP: name = "caller" + // IP-SAME: next_access = {{\[}}["callee"]] + // LOCAL: name = "caller" + // LOCAL-SAME: next_access = ["unknown"] + // LC_AR: name = "caller" + // LC_AR-SAME: next_access = {{\[}}["call"]] memref.load %arg0[] {name = "caller"} : memref - // CHECK: name = "call" - // CHECK-SAME-LITERAL: next_access = [["post"], ["post"]] + // CHECK: name = "call" + // CHECK-SAME: next_access = {{\[}}["post"], ["post"]] test.call_and_store @callee(%arg0), %arg0 {name = "call", store_before_call = false} : (memref, memref) -> () - // CHECK: name = "post" - // CHECK-SAME-LITERAL: next_access = ["unknown"] + // CHECK: name = "post" + // CHECK-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "post"} : memref return } @@ -466,12 +466,12 @@ func.func @call_and_store_after(%arg0: memref) { // - at the entry of the block, the next access is "post". // CHECK-LABEL: @store_with_a_region func.func @store_with_a_region_before(%arg0: memref) { - // CHECK: name = "pre" - // CHECK-SAME-LITERAL: next_access = [["region"]] + // CHECK: name = "pre" + // CHECK-SAME: next_access = {{\[}}["region"]] memref.load %arg0[] {name = "pre"} : memref // CHECK: name = "region" - // CHECK-SAME-LITERAL: next_access = [["post"]] - // CHECK-SAME-LITERAL: next_at_entry_point = [[["post"]]] + // CHECK-SAME: next_access = {{\[}}["post"]] + // CHECK-SAME: next_at_entry_point = {{\[}}{{\[}}["post"]]] test.store_with_a_region %arg0 attributes { name = "region", store_before_region = true } { test.store_with_a_region_terminator } : memref @@ -485,12 +485,12 @@ func.func @store_with_a_region_before(%arg0: memref) { // - at the entry of the block, the next access is "region". // CHECK-LABEL: @store_with_a_region func.func @store_with_a_region_after(%arg0: memref) { - // CHECK: name = "pre" - // CHECK-SAME-LITERAL: next_access = [["region"]] + // CHECK: name = "pre" + // CHECK-SAME: next_access = {{\[}}["region"]] memref.load %arg0[] {name = "pre"} : memref - // CHECK: name = "region" - // CHECK-SAME-LITERAL: next_access = [["post"]] - // CHECK-SAME-LITERAL: next_at_entry_point = [[["region"]]] + // CHECK: name = "region" + // CHECK-SAME: next_access = {{\[}}["post"]] + // CHECK-SAME: next_at_entry_point = {{\[}}{{\[}}["region"]]] test.store_with_a_region %arg0 attributes { name = "region", store_before_region = false } { test.store_with_a_region_terminator } : memref @@ -509,20 +509,20 @@ func.func @store_with_a_region_after(%arg0: memref) { // That is, the order of access is: "pre" -> "region" -> "inner" -> "post". // CHECK-LABEL: @store_with_a_region_before_containing_a_load func.func @store_with_a_region_before_containing_a_load(%arg0: memref) { - // CHECK: name = "pre" - // CHECK-SAME-LITERAL: next_access = [["region"]] + // CHECK: name = "pre" + // CHECK-SAME: next_access = {{\[}}["region"]] memref.load %arg0[] {name = "pre"} : memref - // CHECK: name = "region" - // CHECK-SAME-LITERAL: next_access = [["post"]] - // CHECK-SAME-LITERAL: next_at_entry_point = [[["inner"]]] + // CHECK: name = "region" + // CHECK-SAME: next_access = {{\[}}["post"]] + // CHECK-SAME: next_at_entry_point = {{\[}}{{\[}}["inner"]]] test.store_with_a_region %arg0 attributes { name = "region", store_before_region = true } { - // CHECK: name = "inner" - // CHECK-SAME-LITERAL: next_access = [["post"]] + // CHECK: name = "inner" + // CHECK-SAME: next_access = {{\[}}["post"]] memref.load %arg0[] {name = "inner"} : memref test.store_with_a_region_terminator } : memref - // CHECK: name = "post" - // CHECK-SAME-LITERAL: next_access = ["unknown"] + // CHECK: name = "post" + // CHECK-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "post"} : memref return } @@ -538,20 +538,20 @@ func.func @store_with_a_region_before_containing_a_load(%arg0: memref) { // That is, the order of access is "pre" -> "inner" -> "region" -> "post". // CHECK-LABEL: @store_with_a_region_after_containing_a_load func.func @store_with_a_region_after_containing_a_load(%arg0: memref) { - // CHECK: name = "pre" - // CHECK-SAME-LITERAL: next_access = [["inner"]] + // CHECK: name = "pre" + // CHECK-SAME: next_access = {{\[}}["inner"]] memref.load %arg0[] {name = "pre"} : memref - // CHECK: name = "region" - // CHECK-SAME-LITERAL: next_access = [["post"]] - // CHECK-SAME-LITERAL: next_at_entry_point = [[["inner"]]] + // CHECK: name = "region" + // CHECK-SAME: next_access = {{\[}}["post"]] + // CHECK-SAME: next_at_entry_point = {{\[}}{{\[}}["inner"]]] test.store_with_a_region %arg0 attributes { name = "region", store_before_region = false } { - // CHECK: name = "inner" - // CHECK-SAME-LITERAL: next_access = [["region"]] + // CHECK: name = "inner" + // CHECK-SAME: next_access = {{\[}}["region"]] memref.load %arg0[] {name = "inner"} : memref test.store_with_a_region_terminator } : memref - // CHECK: name = "post" - // CHECK-SAME-LITERAL: next_access = ["unknown"] + // CHECK: name = "post" + // CHECK-SAME: next_access = ["unknown"] memref.load %arg0[] {name = "post"} : memref return } From a0a3c793d212ffc70fdba4c94b024114d11532af Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Mon, 18 Dec 2023 16:14:25 +0000 Subject: [PATCH 138/884] [Clang][SME] Warn when a function doesn't have ZA state (#75805) This patch adds a warning that's emitted when a builtin call uses ZA state but the calling function doesn't provide any. Patch by David Sherwood . --- clang/include/clang/Basic/CMakeLists.txt | 3 + .../clang/Basic/DiagnosticSemaKinds.td | 3 + clang/lib/Sema/SemaChecking.cpp | 24 +++ .../aarch64-sme-intrinsics/acle_sme_add-i32.c | 16 +- .../aarch64-sme-intrinsics/acle_sme_add-i64.c | 16 +- .../aarch64-sme-intrinsics/acle_sme_ld1.c | 20 +- .../acle_sme_ld1_vnum.c | 20 +- .../aarch64-sme-intrinsics/acle_sme_ldr.c | 10 +- .../acle_sme_mopa-za32.c | 14 +- .../acle_sme_mopa-za64.c | 10 +- .../acle_sme_mops-za32.c | 14 +- .../acle_sme_mops-za64.c | 10 +- .../aarch64-sme-intrinsics/acle_sme_read.c | 192 +++++++++--------- .../aarch64-sme-intrinsics/acle_sme_st1.c | 20 +- .../acle_sme_st1_vnum.c | 20 +- .../aarch64-sme-intrinsics/acle_sme_str.c | 10 +- .../aarch64-sme-intrinsics/acle_sme_write.c | 192 +++++++++--------- .../aarch64-sme-intrinsics/acle_sme_zero.c | 8 +- .../Sema/aarch64-incompat-sm-builtin-calls.c | 5 + .../aarch64-sme-intrinsics/acle_sme_imm.cpp | 14 +- .../aarch64-sme-intrinsics/acle_sme_target.c | 8 +- clang/utils/TableGen/SveEmitter.cpp | 32 +++ clang/utils/TableGen/TableGen.cpp | 6 + clang/utils/TableGen/TableGenBackends.h | 1 + 24 files changed, 371 insertions(+), 297 deletions(-) diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt index 73fd521aeeec3..28baa2e45e423 100644 --- a/clang/include/clang/Basic/CMakeLists.txt +++ b/clang/include/clang/Basic/CMakeLists.txt @@ -103,6 +103,9 @@ clang_tablegen(arm_sme_sema_rangechecks.inc -gen-arm-sme-sema-rangechecks clang_tablegen(arm_sme_streaming_attrs.inc -gen-arm-sme-streaming-attrs SOURCE arm_sme.td TARGET ClangARMSmeStreamingAttrs) +clang_tablegen(arm_sme_builtins_za_state.inc -gen-arm-sme-builtin-za-state + SOURCE arm_sme.td + TARGET ClangARMSmeBuiltinsZAState) clang_tablegen(arm_cde_builtins.inc -gen-arm-cde-builtin-def SOURCE arm_cde.td TARGET ClangARMCdeBuiltinsDef) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 3bde53a8ce863..6e6f56ff75e5f 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3155,6 +3155,9 @@ def err_attribute_arm_feature_sve_bits_unsupported : Error< def warn_attribute_arm_sm_incompat_builtin : Warning< "builtin call has undefined behaviour when called from a %0 function">, InGroup>; +def warn_attribute_arm_za_builtin_no_za_state : Warning< + "builtin call is not valid when calling from a function without active ZA state">, + InGroup>; def err_sve_vector_in_non_sve_target : Error< "SVE vector type %0 cannot be used in a target without sve">; def err_attribute_riscv_rvv_bits_unsupported : Error< diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 49be88051e654..42e29e4309378 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -3174,6 +3174,25 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, } } +static bool hasSMEZAState(const FunctionDecl *FD) { + if (FD->hasAttr()) + return true; + if (const auto *T = FD->getType()->getAs()) + if (T->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask) + return true; + return false; +} + +static bool hasSMEZAState(unsigned BuiltinID) { + switch (BuiltinID) { + default: + return false; +#define GET_SME_BUILTIN_HAS_ZA_STATE +#include "clang/Basic/arm_sme_builtins_za_state.inc" +#undef GET_SME_BUILTIN_HAS_ZA_STATE + } +} + bool Sema::CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { if (const FunctionDecl *FD = getCurFunctionDecl()) { std::optional BuiltinType; @@ -3186,6 +3205,11 @@ bool Sema::CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { if (BuiltinType) checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType); + + if (hasSMEZAState(BuiltinID) && !hasSMEZAState(FD)) + Diag(TheCall->getBeginLoc(), + diag::warn_attribute_arm_za_builtin_no_za_state) + << TheCall->getSourceRange(); } // Range check SME intrinsics that take immediate values. diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c index ee6c1c9dd566b..55d2e355897f7 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { +void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za32, _u32, _m)(0, pn, pm, zn); } @@ -50,7 +50,7 @@ void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_stream // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { +void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za32, _u32, _m)(3, pn, pm, zn); } @@ -70,7 +70,7 @@ void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { +void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za32, _s32, _m)(0, pn, pm, zn); } @@ -90,7 +90,7 @@ void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streami // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { +void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za32, _s32, _m)(3, pn, pm, zn); } @@ -110,7 +110,7 @@ void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_strea // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { +void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za32, _u32, _m)(0, pn, pm, zn); } @@ -130,7 +130,7 @@ void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_stream // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming { +void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za32, _u32, _m)(3, pn, pm, zn); } @@ -150,7 +150,7 @@ void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { +void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za32, _s32, _m)(0, pn, pm, zn); } @@ -170,7 +170,7 @@ void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streami // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming { +void test_svaddva_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za32, _s32, _m)(3, pn, pm, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c index 254ea89d22c50..8e9c2e7da46a3 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { +void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za64, _u64, _m)(0, pn, pm, zn); } @@ -50,7 +50,7 @@ void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_stream // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { +void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za64, _u64, _m)(7, pn, pm, zn); } @@ -70,7 +70,7 @@ void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { +void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za64, _s64, _m)(0, pn, pm, zn); } @@ -90,7 +90,7 @@ void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streami // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { +void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddha_za64, _s64, _m)(7, pn, pm, zn); } @@ -110,7 +110,7 @@ void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_strea // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { +void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za64, _u64, _m)(0, pn, pm, zn); } @@ -130,7 +130,7 @@ void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_stream // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming { +void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za64, _u64, _m)(7, pn, pm, zn); } @@ -150,7 +150,7 @@ void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { +void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za64, _s64, _m)(0, pn, pm, zn); } @@ -170,7 +170,7 @@ void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streami // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, [[TMP0]], [[TMP1]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svaddva_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming { +void test_svaddva_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svaddva_za64, _s64, _m)(7, pn, pm, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c index 5622568c4cd76..e17782db222b6 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c @@ -22,7 +22,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1b.horiz( [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za8(0, slice_base, pg, ptr); svld1_hor_za8(0, slice_base + 15, pg, ptr); } @@ -45,7 +45,7 @@ void test_svld1_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1h.horiz( [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_hor_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za16(0, slice_base, pg, ptr); svld1_hor_za16(1, slice_base + 7, pg, ptr); } @@ -68,7 +68,7 @@ void test_svld1_hor_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1w.horiz( [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_hor_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za32(0, slice_base, pg, ptr); svld1_hor_za32(3, slice_base + 3, pg, ptr); } @@ -91,7 +91,7 @@ void test_svld1_hor_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1d.horiz( [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_hor_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za64(0, slice_base, pg, ptr); svld1_hor_za64(7, slice_base + 1, pg, ptr); } @@ -112,7 +112,7 @@ void test_svld1_hor_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1q.horiz( [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_za128(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_hor_za128(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za128(0, slice_base, pg, ptr); svld1_hor_za128(15, slice_base, pg, ptr); } @@ -133,7 +133,7 @@ void test_svld1_hor_za128(uint32_t slice_base, svbool_t pg, const void *ptr) __a // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1b.vert( [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_ver_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_ver_za8(0, slice_base, pg, ptr); svld1_ver_za8(0, slice_base + 15, pg, ptr); } @@ -156,7 +156,7 @@ void test_svld1_ver_za8(uint32_t slice_base, svbool_t pg, const void *ptr) __arm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1h.vert( [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_ver_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_ver_za16(0, slice_base, pg, ptr); svld1_ver_za16(1, slice_base + 7, pg, ptr); } @@ -179,7 +179,7 @@ void test_svld1_ver_za16(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1w.vert( [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_ver_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_ver_za32(0, slice_base, pg, ptr); svld1_ver_za32(3, slice_base + 3, pg, ptr); } @@ -202,7 +202,7 @@ void test_svld1_ver_za32(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1d.vert( [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_ver_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_ver_za64(0, slice_base, pg, ptr); svld1_ver_za64(7, slice_base + 1, pg, ptr); } @@ -223,7 +223,7 @@ void test_svld1_ver_za64(uint32_t slice_base, svbool_t pg, const void *ptr) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1q.vert( [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_za128(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming { +void test_svld1_ver_za128(uint32_t slice_base, svbool_t pg, const void *ptr) __arm_streaming __arm_shared_za { svld1_ver_za128(0, slice_base, pg, ptr); svld1_ver_za128(15, slice_base, pg, ptr); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c index 0fe7dcfc0a799..0fa77e1144a7d 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c @@ -28,7 +28,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1b.horiz( [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_hor_vnum_za8(0, slice_base, pg, ptr, vnum); svld1_hor_vnum_za8(0, slice_base + 15, pg, ptr, vnum); } @@ -57,7 +57,7 @@ void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1h.horiz( [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_hor_vnum_za16(0, slice_base, pg, ptr, vnum); svld1_hor_vnum_za16(1, slice_base + 7, pg, ptr, vnum); } @@ -86,7 +86,7 @@ void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1w.horiz( [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_hor_vnum_za32(0, slice_base, pg, ptr, vnum); svld1_hor_vnum_za32(3, slice_base + 3, pg, ptr, vnum); } @@ -115,7 +115,7 @@ void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1d.horiz( [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_hor_vnum_za64(0, slice_base, pg, ptr, vnum); svld1_hor_vnum_za64(7, slice_base + 1, pg, ptr, vnum); } @@ -142,7 +142,7 @@ void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1q.horiz( [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_hor_vnum_za128(0, slice_base, pg, ptr, vnum); svld1_hor_vnum_za128(15, slice_base, pg, ptr, vnum); } @@ -169,7 +169,7 @@ void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1b.vert( [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_ver_vnum_za8(0, slice_base, pg, ptr, vnum); svld1_ver_vnum_za8(0, slice_base + 15, pg, ptr, vnum); } @@ -198,7 +198,7 @@ void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, i // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1h.vert( [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_ver_vnum_za16(0, slice_base, pg, ptr, vnum); svld1_ver_vnum_za16(1, slice_base + 7, pg, ptr, vnum); } @@ -227,7 +227,7 @@ void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1w.vert( [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_ver_vnum_za32(0, slice_base, pg, ptr, vnum); svld1_ver_vnum_za32(3, slice_base + 3, pg, ptr, vnum); } @@ -256,7 +256,7 @@ void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1d.vert( [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_ver_vnum_za64(0, slice_base, pg, ptr, vnum); svld1_ver_vnum_za64(7, slice_base + 1, pg, ptr, vnum); } @@ -283,7 +283,7 @@ void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.ld1q.vert( [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svld1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming { +void test_svld1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svld1_ver_vnum_za128(0, slice_base, pg, ptr, vnum); svld1_ver_vnum_za128(15, slice_base, pg, ptr, vnum); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c index 9af0778e89c5e..314c9645dd4f7 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c @@ -12,7 +12,7 @@ // CHECK-NEXT: tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 0) // CHECK-NEXT: ret void // -void test_svldr_vnum_za(uint32_t slice_base, const void *ptr) { +void test_svldr_vnum_za(uint32_t slice_base, const void *ptr) __arm_shared_za { svldr_vnum_za(slice_base, ptr, 0); } @@ -22,7 +22,7 @@ void test_svldr_vnum_za(uint32_t slice_base, const void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 15) // CHECK-NEXT: ret void // -void test_svldr_vnum_za_1(uint32_t slice_base, const void *ptr) { +void test_svldr_vnum_za_1(uint32_t slice_base, const void *ptr) __arm_shared_za { svldr_vnum_za(slice_base, ptr, 15); } @@ -32,7 +32,7 @@ void test_svldr_vnum_za_1(uint32_t slice_base, const void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 0) // CHECK-NEXT: ret void // -void test_svldr_za(uint32_t slice_base, const void *ptr) { +void test_svldr_za(uint32_t slice_base, const void *ptr) __arm_shared_za { svldr_za(slice_base, ptr); } @@ -43,7 +43,7 @@ void test_svldr_za(uint32_t slice_base, const void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 [[TMP0:%.*]]) // CHECK-NEXT: ret void // -void test_svldr_vnum_za_var(uint32_t slice_base, const void *ptr, int64_t vnum) { +void test_svldr_vnum_za_var(uint32_t slice_base, const void *ptr, int64_t vnum) __arm_shared_za { svldr_vnum_za(slice_base, ptr, vnum); } @@ -53,6 +53,6 @@ void test_svldr_vnum_za_var(uint32_t slice_base, const void *ptr, int64_t vnum) // CHECK-NEXT: tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 16) // CHECK-NEXT: ret void // -void test_svldr_vnum_za_2(uint32_t slice_base, const void *ptr) { +void test_svldr_vnum_za_2(uint32_t slice_base, const void *ptr) __arm_shared_za { svldr_vnum_za(slice_base, ptr, 16); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c index b90c9be4a6e09..e84f31c2dfa92 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming { +void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -42,7 +42,7 @@ void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __a // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming { +void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za32, _u8, _m)(0, pn, pm, zn, zm); } @@ -62,7 +62,7 @@ void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.wide.nxv8bf16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { +void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za32, _bf16, _m)(0, pn, pm, zn, zm); } @@ -82,7 +82,7 @@ void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.wide.nxv8f16(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming { +void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za32, _f16, _m)(1, pn, pm, zn, zm); } @@ -102,7 +102,7 @@ void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming { +void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za32, _f32, _m)(1, pn, pm, zn, zm); } @@ -118,7 +118,7 @@ void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming { +void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svsumopa_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -134,7 +134,7 @@ void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmopa.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming { +void test_svusmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svusmopa_za32, _u8, _m)(0, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c index a56ce4d17f126..1b22eb64e9e36 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smopa.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming { +void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za64, _s16, _m)(7, pn, pm, zn, zm); } @@ -50,7 +50,7 @@ void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umopa.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming { +void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za64, _u16, _m)(0, pn, pm, zn, zm); } @@ -70,7 +70,7 @@ void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mopa.nxv2f64(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming { +void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmopa_za64, _f64, _m)(7, pn, pm, zn, zm); } @@ -90,7 +90,7 @@ void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumopa.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming { +void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svsumopa_za64, _s16, _m)(0, pn, pm, zn, zm); } @@ -110,7 +110,7 @@ void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmopa.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming { +void test_svusmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svusmopa_za64, _u16, _m)(7, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c index abcf4c2e698d7..0ff97ff92f714 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming { +void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -42,7 +42,7 @@ void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) __a // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming { +void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za32, _u8, _m)(0, pn, pm, zn, zm); } @@ -62,7 +62,7 @@ void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.wide.nxv8bf16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { +void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za32, _bf16, _m)(0, pn, pm, zn, zm); } @@ -82,7 +82,7 @@ void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.wide.nxv8f16(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming { +void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za32, _f16, _m)(1, pn, pm, zn, zm); } @@ -102,7 +102,7 @@ void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.nxv4f32(i32 1, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming { +void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za32, _f32, _m)(1, pn, pm, zn, zm); } @@ -118,7 +118,7 @@ void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming { +void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svsumops_za32, _s8, _m)(0, pn, pm, zn, zm); } @@ -134,7 +134,7 @@ void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmops.wide.nxv16i8(i32 0, [[PN]], [[PM]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming { +void test_svusmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svusmops_za32, _u8, _m)(0, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c index b26b9e4e51e05..5b190a7f9b748 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c @@ -30,7 +30,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.smops.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming { +void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za64, _s16, _m)(7, pn, pm, zn, zm); } @@ -50,7 +50,7 @@ void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.umops.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming { +void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za64, _u16, _m)(0, pn, pm, zn, zm); } @@ -70,7 +70,7 @@ void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.mops.nxv2f64(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming { +void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svmops_za64, _f64, _m)(7, pn, pm, zn, zm); } @@ -90,7 +90,7 @@ void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.sumops.wide.nxv8i16(i32 0, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming { +void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svsumops_za64, _s16, _m)(0, pn, pm, zn, zm); } @@ -110,7 +110,7 @@ void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.usmops.wide.nxv8i16(i32 7, [[TMP0]], [[TMP1]], [[ZN]], [[ZM]]) // CHECK-CXX-NEXT: ret void // -void test_svusmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming { +void test_svusmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svusmops_za64, _u16, _m)(7, pn, pm, zn, zm); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c index a15599d186a87..843a96da90278 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice_base); } @@ -44,7 +44,7 @@ svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) _ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice); } @@ -63,7 +63,7 @@ svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 0, slice_base); } @@ -83,7 +83,7 @@ svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 1, slice); } @@ -102,7 +102,7 @@ svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 0, slice_base); } @@ -122,7 +122,7 @@ svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 3, slice); } @@ -141,7 +141,7 @@ svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 0, slice_base); } @@ -161,7 +161,7 @@ svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 7, slice); } @@ -178,7 +178,7 @@ svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice_base); } @@ -196,7 +196,7 @@ svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice); } @@ -215,7 +215,7 @@ svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 0, slice_base); } @@ -235,7 +235,7 @@ svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 1, slice); } @@ -254,7 +254,7 @@ svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 0, slice_base); } @@ -274,7 +274,7 @@ svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 3, slice); } @@ -293,7 +293,7 @@ svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 0, slice_base); } @@ -313,7 +313,7 @@ svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 7, slice); } @@ -332,7 +332,7 @@ svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 0, slice_base); } @@ -352,7 +352,7 @@ svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 1, slice); } @@ -371,7 +371,7 @@ svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 0, slice_base); } @@ -391,7 +391,7 @@ svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 1, slice); } @@ -410,7 +410,7 @@ svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 0, slice_base); } @@ -430,7 +430,7 @@ svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 3, slice); } @@ -449,7 +449,7 @@ svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 0, slice_base); } @@ -469,7 +469,7 @@ svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 7, slice); } @@ -486,7 +486,7 @@ svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 0, slice_base); } @@ -502,7 +502,7 @@ svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 15, slice_base); } @@ -520,7 +520,7 @@ svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 0, slice_base); } @@ -538,7 +538,7 @@ svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 15, slice_base); } @@ -556,7 +556,7 @@ svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 0, slice_base); } @@ -574,7 +574,7 @@ svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 15, slice_base); } @@ -592,7 +592,7 @@ svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 0, slice_base); } @@ -610,7 +610,7 @@ svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 15, slice_base); } @@ -626,7 +626,7 @@ svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 0, slice_base); } @@ -642,7 +642,7 @@ svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 15, slice_base); } @@ -660,7 +660,7 @@ svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 0, slice_base); } @@ -678,7 +678,7 @@ svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 15, slice_base); } @@ -696,7 +696,7 @@ svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 0, slice_base); } @@ -714,7 +714,7 @@ svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 15, slice_base); } @@ -732,7 +732,7 @@ svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 0, slice_base); } @@ -750,7 +750,7 @@ svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 15, slice_base); } @@ -768,7 +768,7 @@ svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 0, slice_base); } @@ -786,7 +786,7 @@ svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8f16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 15, slice_base); } @@ -804,7 +804,7 @@ svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 0, slice_base); } @@ -822,7 +822,7 @@ svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t s // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv8bf16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 15, slice_base); } @@ -840,7 +840,7 @@ svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 0, slice_base); } @@ -858,7 +858,7 @@ svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv4f32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 15, slice_base); } @@ -876,7 +876,7 @@ svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 0, slice_base); } @@ -894,7 +894,7 @@ svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.horiz.nxv2f64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 15, slice_base); } @@ -910,7 +910,7 @@ svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice_base); } @@ -928,7 +928,7 @@ svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) _ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice); } @@ -947,7 +947,7 @@ svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 0, slice_base); } @@ -967,7 +967,7 @@ svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 1, slice); } @@ -986,7 +986,7 @@ svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 0, slice_base); } @@ -1006,7 +1006,7 @@ svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 3, slice); } @@ -1025,7 +1025,7 @@ svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 0, slice_base); } @@ -1045,7 +1045,7 @@ svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 7, slice); } @@ -1062,7 +1062,7 @@ svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice_base); } @@ -1080,7 +1080,7 @@ svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice); } @@ -1099,7 +1099,7 @@ svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 0, slice_base); } @@ -1119,7 +1119,7 @@ svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8i16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 1, slice); } @@ -1138,7 +1138,7 @@ svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 0, slice_base); } @@ -1158,7 +1158,7 @@ svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4i32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 3, slice); } @@ -1177,7 +1177,7 @@ svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 0, slice_base); } @@ -1197,7 +1197,7 @@ svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2i64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 7, slice); } @@ -1216,7 +1216,7 @@ svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 0, slice_base); } @@ -1236,7 +1236,7 @@ svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8f16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 1, slice); } @@ -1255,7 +1255,7 @@ svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 0, slice_base); } @@ -1275,7 +1275,7 @@ svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 1, slice); } @@ -1294,7 +1294,7 @@ svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 0, slice_base); } @@ -1314,7 +1314,7 @@ svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv4f32( [[ZD]], [[TMP0]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 3, slice); } @@ -1333,7 +1333,7 @@ svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 0, slice_base); } @@ -1353,7 +1353,7 @@ svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.read.vert.nxv2f64( [[ZD]], [[TMP0]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 7, slice); } @@ -1370,7 +1370,7 @@ svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t sli // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 0, slice_base); } @@ -1386,7 +1386,7 @@ svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 15, slice_base); } @@ -1404,7 +1404,7 @@ svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 0, slice_base); } @@ -1422,7 +1422,7 @@ svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 15, slice_base); } @@ -1440,7 +1440,7 @@ svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 0, slice_base); } @@ -1458,7 +1458,7 @@ svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 15, slice_base); } @@ -1476,7 +1476,7 @@ svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 0, slice_base); } @@ -1494,7 +1494,7 @@ svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_ba // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 15, slice_base); } @@ -1510,7 +1510,7 @@ svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 0, slice_base); } @@ -1526,7 +1526,7 @@ svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_bas // CHECK-CXX-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv16i8( [[ZD]], [[PG]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP0]] // -svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 15, slice_base); } @@ -1544,7 +1544,7 @@ svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_b // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 0, slice_base); } @@ -1562,7 +1562,7 @@ svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8i16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 15, slice_base); } @@ -1580,7 +1580,7 @@ svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 0, slice_base); } @@ -1598,7 +1598,7 @@ svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4i32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 15, slice_base); } @@ -1616,7 +1616,7 @@ svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 0, slice_base); } @@ -1634,7 +1634,7 @@ svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_ // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2i64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 15, slice_base); } @@ -1652,7 +1652,7 @@ svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8f16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 0, slice_base); } @@ -1670,7 +1670,7 @@ svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8f16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 15, slice_base); } @@ -1688,7 +1688,7 @@ svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 0, slice_base); } @@ -1706,7 +1706,7 @@ svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t s // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv8bf16( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 15, slice_base); } @@ -1724,7 +1724,7 @@ svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4f32( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 0, slice_base); } @@ -1742,7 +1742,7 @@ svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv4f32( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 15, slice_base); } @@ -1760,7 +1760,7 @@ svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t sl // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2f64( [[ZD]], [[TMP0]], i32 0, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 0, slice_base); } @@ -1778,7 +1778,7 @@ svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slic // CHECK-CXX-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sme.readq.vert.nxv2f64( [[ZD]], [[TMP0]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret [[TMP1]] // -svfloat64_t test_svread_ver_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { +svfloat64_t test_svread_ver_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming __arm_shared_za { return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 15, slice_base); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c index 9ed158aedf7e3..98ebbefc2e74c 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c @@ -22,7 +22,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1b.horiz( [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_hor_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_hor_za8(0, slice_base, pg, ptr); svst1_hor_za8(0, slice_base + 15, pg, ptr); } @@ -45,7 +45,7 @@ void test_svst1_hor_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_strea // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1h.horiz( [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_hor_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_hor_za16(0, slice_base, pg, ptr); svst1_hor_za16(1, slice_base + 7, pg, ptr); } @@ -68,7 +68,7 @@ void test_svst1_hor_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1w.horiz( [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_hor_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_hor_za32(0, slice_base, pg, ptr); svst1_hor_za32(3, slice_base + 3, pg, ptr); } @@ -91,7 +91,7 @@ void test_svst1_hor_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1d.horiz( [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_hor_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_hor_za64(0, slice_base, pg, ptr); svst1_hor_za64(7, slice_base + 1, pg, ptr); } @@ -112,7 +112,7 @@ void test_svst1_hor_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1q.horiz( [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_za128(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_hor_za128(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_hor_za128(0, slice_base, pg, ptr); svst1_hor_za128(15, slice_base, pg, ptr); } @@ -133,7 +133,7 @@ void test_svst1_hor_za128(uint32_t slice_base, svbool_t pg, void *ptr) __arm_str // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1b.vert( [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_ver_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za8(0, slice_base, pg, ptr); svst1_ver_za8(0, slice_base + 15, pg, ptr); } @@ -156,7 +156,7 @@ void test_svst1_ver_za8(uint32_t slice_base, svbool_t pg, void *ptr) __arm_strea // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1h.vert( [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_ver_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za16(0, slice_base, pg, ptr); svst1_ver_za16(1, slice_base + 7, pg, ptr); } @@ -179,7 +179,7 @@ void test_svst1_ver_za16(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1w.vert( [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_ver_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za32(0, slice_base, pg, ptr); svst1_ver_za32(3, slice_base + 3, pg, ptr); } @@ -202,7 +202,7 @@ void test_svst1_ver_za32(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1d.vert( [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_ver_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za64(0, slice_base, pg, ptr); svst1_ver_za64(7, slice_base + 1, pg, ptr); } @@ -223,7 +223,7 @@ void test_svst1_ver_za64(uint32_t slice_base, svbool_t pg, void *ptr) __arm_stre // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1q.vert( [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_za128(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming { +void test_svst1_ver_za128(uint32_t slice_base, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za128(0, slice_base, pg, ptr); svst1_ver_za128(15, slice_base, pg, ptr); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c index 627098d9365bd..938e62a15c771 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c @@ -28,7 +28,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1b.horiz( [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_hor_vnum_za8(0, slice_base, pg, ptr, vnum); svst1_hor_vnum_za8(0, slice_base + 15, pg, ptr, vnum); } @@ -57,7 +57,7 @@ void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1h.horiz( [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_hor_vnum_za16(0, slice_base, pg, ptr, vnum); svst1_hor_vnum_za16(1, slice_base + 7, pg, ptr, vnum); } @@ -86,7 +86,7 @@ void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1w.horiz( [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_hor_vnum_za32(0, slice_base, pg, ptr, vnum); svst1_hor_vnum_za32(3, slice_base + 3, pg, ptr, vnum); } @@ -115,7 +115,7 @@ void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1d.horiz( [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_hor_vnum_za64(0, slice_base, pg, ptr, vnum); svst1_hor_vnum_za64(7, slice_base + 1, pg, ptr, vnum); } @@ -142,7 +142,7 @@ void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1q.horiz( [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_hor_vnum_za128(0, slice_base, pg, ptr, vnum); svst1_hor_vnum_za128(15, slice_base, pg, ptr, vnum); } @@ -169,7 +169,7 @@ void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int6 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1b.vert( [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_ver_vnum_za8(0, slice_base, pg, ptr, vnum); svst1_ver_vnum_za8(0, slice_base + 15, pg, ptr, vnum); } @@ -198,7 +198,7 @@ void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1h.vert( [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_ver_vnum_za16(0, slice_base, pg, ptr, vnum); svst1_ver_vnum_za16(1, slice_base + 7, pg, ptr, vnum); } @@ -227,7 +227,7 @@ void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1w.vert( [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_ver_vnum_za32(0, slice_base, pg, ptr, vnum); svst1_ver_vnum_za32(3, slice_base + 3, pg, ptr, vnum); } @@ -256,7 +256,7 @@ void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1d.vert( [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_ver_vnum_za64(0, slice_base, pg, ptr, vnum); svst1_ver_vnum_za64(7, slice_base + 1, pg, ptr, vnum); } @@ -283,7 +283,7 @@ void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64 // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.st1q.vert( [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]]) // CHECK-CXX-NEXT: ret void // -void test_svst1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming { +void test_svst1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_shared_za { svst1_ver_vnum_za128(0, slice_base, pg, ptr, vnum); svst1_ver_vnum_za128(15, slice_base, pg, ptr, vnum); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c index baadfc18563a0..bcf368bc8dce4 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c @@ -12,7 +12,7 @@ // CHECK-NEXT: tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 0) // CHECK-NEXT: ret void // -void test_svstr_vnum_za(uint32_t slice_base, void *ptr) { +void test_svstr_vnum_za(uint32_t slice_base, void *ptr) __arm_shared_za { svstr_vnum_za(slice_base, ptr, 0); } @@ -22,7 +22,7 @@ void test_svstr_vnum_za(uint32_t slice_base, void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 15) // CHECK-NEXT: ret void // -void test_svstr_vnum_za_1(uint32_t slice_base, void *ptr) { +void test_svstr_vnum_za_1(uint32_t slice_base, void *ptr) __arm_shared_za { svstr_vnum_za(slice_base, ptr, 15); } @@ -32,7 +32,7 @@ void test_svstr_vnum_za_1(uint32_t slice_base, void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 0) // CHECK-NEXT: ret void // -void test_svstr_za(uint32_t slice_base, void *ptr) { +void test_svstr_za(uint32_t slice_base, void *ptr) __arm_shared_za { svstr_za(slice_base, ptr); } @@ -43,7 +43,7 @@ void test_svstr_za(uint32_t slice_base, void *ptr) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 [[TMP0:%.*]]) // CHECK-NEXT: ret void // -void test_svstr_vnum_za_var(uint32_t slice_base, void *ptr, int64_t vnum) { +void test_svstr_vnum_za_var(uint32_t slice_base, void *ptr, int64_t vnum) __arm_shared_za { svstr_vnum_za(slice_base, ptr, vnum); } @@ -53,6 +53,6 @@ void test_svstr_vnum_za_var(uint32_t slice_base, void *ptr, int64_t vnum) { // CHECK-NEXT: tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]], i32 16) // CHECK-NEXT: ret void // -void test_svstr_vnum_za_2(uint32_t slice_base, void *ptr) { +void test_svstr_vnum_za_2(uint32_t slice_base, void *ptr) __arm_shared_za { svstr_vnum_za(slice_base, ptr, 16); } diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c index 2cc338add314b..38c8402c3d0fa 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c @@ -26,7 +26,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice_base, pg, zn); } @@ -44,7 +44,7 @@ void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice, pg, zn); } @@ -63,7 +63,7 @@ void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(0, slice_base, pg, zn); } @@ -83,7 +83,7 @@ void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(1, slice, pg, zn); } @@ -102,7 +102,7 @@ void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(0, slice_base, pg, zn); } @@ -122,7 +122,7 @@ void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(3, slice, pg, zn); } @@ -141,7 +141,7 @@ void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(0, slice_base, pg, zn); } @@ -161,7 +161,7 @@ void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(7, slice, pg, zn); } @@ -178,7 +178,7 @@ void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice_base, pg, zn); } @@ -196,7 +196,7 @@ void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __a // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice, pg, zn); } @@ -215,7 +215,7 @@ void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(0, slice_base, pg, zn); } @@ -235,7 +235,7 @@ void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(1, slice, pg, zn); } @@ -254,7 +254,7 @@ void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(0, slice_base, pg, zn); } @@ -274,7 +274,7 @@ void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(3, slice, pg, zn); } @@ -293,7 +293,7 @@ void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(0, slice_base, pg, zn); } @@ -313,7 +313,7 @@ void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(7, slice, pg, zn); } @@ -332,7 +332,7 @@ void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(0, slice_base, pg, zn); } @@ -352,7 +352,7 @@ void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(1, slice, pg, zn); } @@ -371,7 +371,7 @@ void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(0, slice_base, pg, zn); } @@ -391,7 +391,7 @@ void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(1, slice, pg, zn); } @@ -410,7 +410,7 @@ void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(0, slice_base, pg, zn); } @@ -430,7 +430,7 @@ void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(3, slice, pg, zn); } @@ -449,7 +449,7 @@ void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(0, slice_base, pg, zn); } @@ -469,7 +469,7 @@ void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(7, slice, pg, zn); } @@ -486,7 +486,7 @@ void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(0, slice_base, pg, zn); } @@ -502,7 +502,7 @@ void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(15, slice_base, pg, zn); } @@ -520,7 +520,7 @@ void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(0, slice_base, pg, zn); } @@ -538,7 +538,7 @@ void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(15, slice_base, pg, zn); } @@ -556,7 +556,7 @@ void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(0, slice_base, pg, zn); } @@ -574,7 +574,7 @@ void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(15, slice_base, pg, zn); } @@ -592,7 +592,7 @@ void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(0, slice_base, pg, zn); } @@ -610,7 +610,7 @@ void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(15, slice_base, pg, zn); } @@ -626,7 +626,7 @@ void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(0, slice_base, pg, zn); } @@ -642,7 +642,7 @@ void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(15, slice_base, pg, zn); } @@ -660,7 +660,7 @@ void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(0, slice_base, pg, zn); } @@ -678,7 +678,7 @@ void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(15, slice_base, pg, zn); } @@ -696,7 +696,7 @@ void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(0, slice_base, pg, zn); } @@ -714,7 +714,7 @@ void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(15, slice_base, pg, zn); } @@ -732,7 +732,7 @@ void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(0, slice_base, pg, zn); } @@ -750,7 +750,7 @@ void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(15, slice_base, pg, zn); } @@ -768,7 +768,7 @@ void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(0, slice_base, pg, zn); } @@ -786,7 +786,7 @@ void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(15, slice_base, pg, zn); } @@ -804,7 +804,7 @@ void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(0, slice_base, pg, zn); } @@ -822,7 +822,7 @@ void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(15, slice_base, pg, zn); } @@ -840,7 +840,7 @@ void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(0, slice_base, pg, zn); } @@ -858,7 +858,7 @@ void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(15, slice_base, pg, zn); } @@ -876,7 +876,7 @@ void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(0, slice_base, pg, zn); } @@ -894,7 +894,7 @@ void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(15, slice_base, pg, zn); } @@ -910,7 +910,7 @@ void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice_base, pg, zn); } @@ -928,7 +928,7 @@ void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __ar // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice, pg, zn); } @@ -947,7 +947,7 @@ void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(0, slice_base, pg, zn); } @@ -967,7 +967,7 @@ void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(1, slice, pg, zn); } @@ -986,7 +986,7 @@ void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(0, slice_base, pg, zn); } @@ -1006,7 +1006,7 @@ void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(3, slice, pg, zn); } @@ -1025,7 +1025,7 @@ void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(0, slice_base, pg, zn); } @@ -1045,7 +1045,7 @@ void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(7, slice, pg, zn); } @@ -1062,7 +1062,7 @@ void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice_base, pg, zn); } @@ -1080,7 +1080,7 @@ void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __a // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 15; SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice, pg, zn); } @@ -1099,7 +1099,7 @@ void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(0, slice_base, pg, zn); } @@ -1119,7 +1119,7 @@ void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(1, slice, pg, zn); } @@ -1138,7 +1138,7 @@ void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(0, slice_base, pg, zn); } @@ -1158,7 +1158,7 @@ void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(3, slice, pg, zn); } @@ -1177,7 +1177,7 @@ void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(0, slice_base, pg, zn); } @@ -1197,7 +1197,7 @@ void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(7, slice, pg, zn); } @@ -1216,7 +1216,7 @@ void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(0, slice_base, pg, zn); } @@ -1236,7 +1236,7 @@ void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(1, slice, pg, zn); } @@ -1255,7 +1255,7 @@ void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(0, slice_base, pg, zn); } @@ -1275,7 +1275,7 @@ void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 1, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 7; SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(1, slice, pg, zn); } @@ -1294,7 +1294,7 @@ void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(0, slice_base, pg, zn); } @@ -1314,7 +1314,7 @@ void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 3, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 3; SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(3, slice, pg, zn); } @@ -1333,7 +1333,7 @@ void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(0, slice_base, pg, zn); } @@ -1353,7 +1353,7 @@ void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 7, i32 [[ADD]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { uint32_t slice = slice_base + 1; SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(7, slice, pg, zn); } @@ -1370,7 +1370,7 @@ void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(0, slice_base, pg, zn); } @@ -1386,7 +1386,7 @@ void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) __ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming { +void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(15, slice_base, pg, zn); } @@ -1404,7 +1404,7 @@ void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(0, slice_base, pg, zn); } @@ -1422,7 +1422,7 @@ void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming { +void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(15, slice_base, pg, zn); } @@ -1440,7 +1440,7 @@ void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(0, slice_base, pg, zn); } @@ -1458,7 +1458,7 @@ void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming { +void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(15, slice_base, pg, zn); } @@ -1476,7 +1476,7 @@ void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(0, slice_base, pg, zn); } @@ -1494,7 +1494,7 @@ void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming { +void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(15, slice_base, pg, zn); } @@ -1510,7 +1510,7 @@ void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(0, slice_base, pg, zn); } @@ -1526,7 +1526,7 @@ void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) _ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], [[PG]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming { +void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(15, slice_base, pg, zn); } @@ -1544,7 +1544,7 @@ void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(0, slice_base, pg, zn); } @@ -1562,7 +1562,7 @@ void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming { +void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(15, slice_base, pg, zn); } @@ -1580,7 +1580,7 @@ void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(0, slice_base, pg, zn); } @@ -1598,7 +1598,7 @@ void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming { +void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(15, slice_base, pg, zn); } @@ -1616,7 +1616,7 @@ void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(0, slice_base, pg, zn); } @@ -1634,7 +1634,7 @@ void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming { +void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(15, slice_base, pg, zn); } @@ -1652,7 +1652,7 @@ void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t z // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(0, slice_base, pg, zn); } @@ -1670,7 +1670,7 @@ void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(15, slice_base, pg, zn); } @@ -1688,7 +1688,7 @@ void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(0, slice_base, pg, zn); } @@ -1706,7 +1706,7 @@ void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming { +void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(15, slice_base, pg, zn); } @@ -1724,7 +1724,7 @@ void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(0, slice_base, pg, zn); } @@ -1742,7 +1742,7 @@ void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming { +void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(15, slice_base, pg, zn); } @@ -1760,7 +1760,7 @@ void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(0, slice_base, pg, zn); } @@ -1778,7 +1778,7 @@ void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 [[SLICE_BASE]], [[TMP0]], [[ZN]]) // CHECK-CXX-NEXT: ret void // -void test_svwrite_ver_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming { +void test_svwrite_ver_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) __arm_streaming __arm_shared_za { SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(15, slice_base, pg, zn); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c index 750eead7c705e..ddd9602369538 100644 --- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c +++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c @@ -18,7 +18,7 @@ // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.zero(i32 0) // CHECK-CXX-NEXT: ret void // -void test_svzero_mask_za() { +void test_svzero_mask_za(void) __arm_shared_za { svzero_mask_za(0); } @@ -34,7 +34,7 @@ void test_svzero_mask_za() { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.zero(i32 176) // CHECK-CXX-NEXT: ret void // -void test_svzero_mask_za_1() { +void test_svzero_mask_za_1(void) __arm_shared_za { svzero_mask_za(176); } @@ -50,7 +50,7 @@ void test_svzero_mask_za_1() { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.zero(i32 255) // CHECK-CXX-NEXT: ret void // -void test_svzero_mask_za_2() { +void test_svzero_mask_za_2(void) __arm_shared_za { svzero_mask_za(255); } @@ -66,7 +66,7 @@ void test_svzero_mask_za_2() { // CHECK-CXX-NEXT: tail call void @llvm.aarch64.sme.zero(i32 255) // CHECK-CXX-NEXT: ret void // -void test_svzero_za() { +void test_svzero_za(void) __arm_shared_za { svzero_za(); } //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c index 361a9e82a3adb..e63d9f0a84757 100644 --- a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c +++ b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c @@ -97,3 +97,8 @@ svbool_t streaming_caller_ptrue(void) __arm_streaming { // expected-no-warning return svand_z(svptrue_b16(), svptrue_pat_b16(SV_ALL), svptrue_pat_b16(SV_VL4)); } + +svint8_t missing_za(svint8_t zd, svbool_t pg, uint32_t slice_base) __arm_streaming { + // expected-warning@+1 {{builtin call is not valid when calling from a function without active ZA state}} + return svread_hor_za8_s8_m(zd, pg, 0, slice_base); +} diff --git a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp index 47c7210206b05..529d0d2d1e625 100644 --- a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp +++ b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp @@ -12,7 +12,7 @@ #include -void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 0]}} SVE_ACLE_FUNC(svld1_hor_za8,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 1 is outside the valid range [0, 0]}} @@ -32,7 +32,7 @@ void test_range_0_0(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svwrite_ver_za8, _s8, _m,)(1, slice, pg, svundef_s8()); } -void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}} SVE_ACLE_FUNC(svld1_hor_za16,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}} @@ -52,7 +52,7 @@ void test_range_0_1(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svwrite_ver_za16, _s16, _m,)(2, slice, pg, svundef_s16()); } -void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}} SVE_ACLE_FUNC(svld1_hor_za32,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}} @@ -90,7 +90,7 @@ void test_range_0_3(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svusmops_za32, _u8, _m,)(-1, pg, pg, svundef_u8(), svundef_s8()); } -void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}} SVE_ACLE_FUNC(svld1_hor_za64,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 8 is outside the valid range [0, 7]}} @@ -133,7 +133,7 @@ void test_range_0_7(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svmops_za64, _f64, _m,)(-1, pg, pg, svundef_f64(), svundef_f64()); } -void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 15]}} SVE_ACLE_FUNC(svld1_hor_za128,,,)(-1, slice, pg, ptr); // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}} @@ -153,14 +153,14 @@ void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) __arm_streaming { SVE_ACLE_FUNC(svwrite_ver_za128, _s8, _m,)(16, slice, pg, svundef_s8()); } -void test_range_0_255(svbool_t pg, void *ptr) __arm_streaming { +void test_range_0_255(svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { // expected-error@+1 {{argument value 256 is outside the valid range [0, 255]}} SVE_ACLE_FUNC(svzero_mask_za,,,)(256); // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 255]}} SVE_ACLE_FUNC(svzero_mask_za,,,)(-1); } -void test_constant(uint64_t u64, svbool_t pg, void *ptr) __arm_streaming { +void test_constant(uint64_t u64, svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { SVE_ACLE_FUNC(svld1_hor_za8,,,)(u64, u64, pg, ptr); // expected-error {{argument to 'svld1_hor_za8' must be a constant integer}} SVE_ACLE_FUNC(svst1_hor_za32,,,)(u64, 0, pg, ptr); // expected-error {{argument to 'svst1_hor_za32' must be a constant integer}} SVE_ACLE_FUNC(svld1_hor_vnum_za8,,,)(u64, 0, pg, ptr, u64); // expected-error {{argument to 'svld1_hor_vnum_za8' must be a constant integer}} diff --git a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c index 7cfe9fdfbd24f..95bb6be2d2d34 100644 --- a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c +++ b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_target.c @@ -6,21 +6,21 @@ #include __attribute__((target("sme"))) -void test_sme(svbool_t pg, void *ptr) __arm_streaming { +void test_sme(svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svld1_hor_za8(0, 0, pg, ptr); } __attribute__((target("arch=armv8-a+sme"))) -void test_arch_sme(svbool_t pg, void *ptr) __arm_streaming { +void test_arch_sme(svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svld1_hor_vnum_za32(0, 0, pg, ptr, 0); } __attribute__((target("+sme"))) -void test_plus_sme(svbool_t pg, void *ptr) __arm_streaming { +void test_plus_sme(svbool_t pg, void *ptr) __arm_streaming __arm_shared_za { svst1_ver_za16(0, 0, pg, ptr); } __attribute__((target("+sme"))) -void undefined(svbool_t pg, void *ptr) { +void undefined(svbool_t pg, void *ptr) __arm_shared_za { svst1_ver_vnum_za64(0, 0, pg, ptr, 0); // expected-warning {{builtin call has undefined behaviour when called from a non-streaming function}} } diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp index a59b7099d5adf..311c6b09dc790 100644 --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -385,6 +385,9 @@ class SVEEmitter { /// Emit all the range checks for the immediates. void createSMERangeChecks(raw_ostream &o); + /// Create a table for a builtin's requirement for PSTATE.ZA. + void createBuiltinZAState(raw_ostream &OS); + /// Create intrinsic and add it to \p Out void createIntrinsic(Record *R, SmallVectorImpl> &Out); @@ -1705,6 +1708,31 @@ void SVEEmitter::createSMERangeChecks(raw_ostream &OS) { OS << "#endif\n\n"; } +void SVEEmitter::createBuiltinZAState(raw_ostream &OS) { + std::vector RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + std::map> DefsZAState; + + uint64_t IsSharedZAFlag = getEnumValueForFlag("IsSharedZA"); + for (auto &Def : Defs) { + bool HasZAState = Def->isFlagSet(IsSharedZAFlag); + DefsZAState[HasZAState].insert(Def->getMangledName()); + } + + OS << "#ifdef GET_SME_BUILTIN_HAS_ZA_STATE\n"; + + for (auto HasZA : {true, false}) { + auto Names = DefsZAState[HasZA]; + for (auto Name : Names) + OS << "case SME::BI__builtin_sme_" << Name << ":\n"; + OS << " return " << (HasZA ? "true" : "false") << ";\n"; + } + OS << "#endif\n\n"; +} + void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { std::vector RV = Records.getAllDerivedDefinitions("Inst"); SmallVector, 128> Defs; @@ -1794,4 +1822,8 @@ void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) { void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) { SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME); } + +void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createBuiltinZAState(OS); +} } // End namespace clang diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp index 9043d90d7cb42..c1f2ca15b595c 100644 --- a/clang/utils/TableGen/TableGen.cpp +++ b/clang/utils/TableGen/TableGen.cpp @@ -92,6 +92,7 @@ enum ActionType { GenArmSmeBuiltinCG, GenArmSmeRangeChecks, GenArmSmeStreamingAttrs, + GenArmSmeBuiltinZAState, GenArmCdeHeader, GenArmCdeBuiltinDef, GenArmCdeBuiltinSema, @@ -260,6 +261,8 @@ cl::opt Action( "Generate arm_sme_sema_rangechecks.inc for clang"), clEnumValN(GenArmSmeStreamingAttrs, "gen-arm-sme-streaming-attrs", "Generate arm_sme_streaming_attrs.inc for clang"), + clEnumValN(GenArmSmeBuiltinZAState, "gen-arm-sme-builtin-za-state", + "Generate arm_sme_builtins_za_state.inc for clang"), clEnumValN(GenArmMveHeader, "gen-arm-mve-header", "Generate arm_mve.h for clang"), clEnumValN(GenArmMveBuiltinDef, "gen-arm-mve-builtin-def", @@ -518,6 +521,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenArmSmeStreamingAttrs: EmitSmeStreamingAttrs(Records, OS); break; + case GenArmSmeBuiltinZAState: + EmitSmeBuiltinZAState(Records, OS); + break; case GenArmCdeHeader: EmitCdeHeader(Records, OS); break; diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h index 6ec51776d637c..35f2f04c1e818 100644 --- a/clang/utils/TableGen/TableGenBackends.h +++ b/clang/utils/TableGen/TableGenBackends.h @@ -112,6 +112,7 @@ void EmitSmeBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeRangeChecks(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSmeStreamingAttrs(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSmeBuiltinZAState(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveBuiltinDef(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); From 7b1e4239b396e57919a56a965282c30711b6b4e8 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Mon, 18 Dec 2023 16:21:11 +0000 Subject: [PATCH 139/884] [DAG] Fold (vt trunc (extload (vt x))) -> (vt load x) (#75229) We were only folding cases which remained extloads, but DAG.getExtLoad can also handle the cases which don't need to extend at all (we just can't do truncloads). reduceLoadWidth can handle this for scalar loads, but not for vectors. Noticed while triaging D152928 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 6 +- llvm/test/CodeGen/AMDGPU/ctpop16.ll | 42 ++- llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll | 12 +- llvm/test/CodeGen/AMDGPU/kernel-args.ll | 296 ++++-------------- 4 files changed, 94 insertions(+), 262 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c782ad117ce60..5b7629fc8cbe8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -14818,11 +14818,11 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { if (SDValue Reduced = reduceLoadWidth(N)) return Reduced; - // Handle the case where the load remains an extending load even - // after truncation. + // Handle the case where the truncated result is at least as wide as the + // loaded type. if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { auto *LN0 = cast(N0); - if (LN0->isSimple() && LN0->getMemoryVT().bitsLT(VT)) { + if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) { SDValue NewLoad = DAG.getExtLoad( LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(), LN0->getBasePtr(), LN0->getMemoryVT(), LN0->getMemOperand()); diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll index 78c657049fcb2..c7396f25fba65 100644 --- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll +++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll @@ -1553,50 +1553,48 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace( ; EG: ; %bb.0: ; %entry ; EG-NEXT: ALU 0, @20, KC0[], KC1[] ; EG-NEXT: TEX 0 @14 -; EG-NEXT: ALU_PUSH_BEFORE 6, @21, KC0[], KC1[] +; EG-NEXT: ALU_PUSH_BEFORE 4, @21, KC0[], KC1[] ; EG-NEXT: JUMP @7 POP:1 -; EG-NEXT: ALU 0, @28, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @26, KC0[CB0:0-32], KC1[] ; EG-NEXT: TEX 0 @16 -; EG-NEXT: ALU_POP_AFTER 1, @29, KC0[], KC1[] -; EG-NEXT: ALU_PUSH_BEFORE 2, @31, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU_POP_AFTER 1, @27, KC0[], KC1[] +; EG-NEXT: ALU_PUSH_BEFORE 2, @29, KC0[CB0:0-32], KC1[] ; EG-NEXT: JUMP @11 POP:1 ; EG-NEXT: TEX 0 @18 -; EG-NEXT: ALU_POP_AFTER 0, @34, KC0[], KC1[] -; EG-NEXT: ALU 11, @35, KC0[], KC1[] +; EG-NEXT: ALU_POP_AFTER 0, @32, KC0[], KC1[] +; EG-NEXT: ALU 11, @33, KC0[], KC1[] ; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X ; EG-NEXT: CF_END ; EG-NEXT: Fetch clause starting at 14: -; EG-NEXT: VTX_READ_16 T1.X, T0.X, 46, #3 +; EG-NEXT: VTX_READ_16 T2.X, T1.X, 46, #3 ; EG-NEXT: Fetch clause starting at 16: -; EG-NEXT: VTX_READ_16 T1.X, T1.X, 2, #1 +; EG-NEXT: VTX_READ_16 T0.X, T0.X, 2, #1 ; EG-NEXT: Fetch clause starting at 18: -; EG-NEXT: VTX_READ_16 T0.X, T0.X, 44, #3 +; EG-NEXT: VTX_READ_16 T0.X, T1.X, 44, #3 ; EG-NEXT: ALU clause starting at 20: -; EG-NEXT: MOV * T0.X, 0.0, +; EG-NEXT: MOV * T1.X, 0.0, ; EG-NEXT: ALU clause starting at 21: -; EG-NEXT: AND_INT * T0.W, T1.X, literal.x, -; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) -; EG-NEXT: MOV T1.X, literal.x, +; EG-NEXT: MOV T0.X, literal.x, ; EG-NEXT: MOV T1.W, literal.y, -; EG-NEXT: SETNE_INT * T0.W, PV.W, 0.0, +; EG-NEXT: SETNE_INT * T0.W, T2.X, 0.0, ; EG-NEXT: 0(0.000000e+00), 1(1.401298e-45) ; EG-NEXT: PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0, -; EG-NEXT: ALU clause starting at 28: -; EG-NEXT: MOV * T1.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 29: +; EG-NEXT: ALU clause starting at 26: +; EG-NEXT: MOV * T0.X, KC0[2].Z, +; EG-NEXT: ALU clause starting at 27: ; EG-NEXT: MOV * T1.W, literal.x, ; EG-NEXT: 0(0.000000e+00), 0(0.000000e+00) -; EG-NEXT: ALU clause starting at 31: +; EG-NEXT: ALU clause starting at 29: ; EG-NEXT: MOV T0.W, KC0[2].Y, ; EG-NEXT: SETE_INT * T1.W, T1.W, 0.0, ; EG-NEXT: PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0, -; EG-NEXT: ALU clause starting at 34: -; EG-NEXT: BCNT_INT * T1.X, T0.X, -; EG-NEXT: ALU clause starting at 35: +; EG-NEXT: ALU clause starting at 32: +; EG-NEXT: BCNT_INT * T0.X, T0.X, +; EG-NEXT: ALU clause starting at 33: ; EG-NEXT: LSHL * T1.W, T0.W, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: AND_INT T1.W, PV.W, literal.x, -; EG-NEXT: AND_INT * T2.W, T1.X, literal.y, +; EG-NEXT: AND_INT * T2.W, T0.X, literal.y, ; EG-NEXT: 24(3.363116e-44), 65535(9.183409e-41) ; EG-NEXT: LSHL T1.X, PS, PV.W, ; EG-NEXT: LSHL * T1.W, literal.x, PV.W, diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll index 855b5fff11fe5..7e8c28fa44750 100644 --- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll +++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll @@ -331,7 +331,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i8_with_select(ptr addrspace(1) noa ; EG: ; %bb.0: ; EG-NEXT: ALU 0, @8, KC0[], KC1[] ; EG-NEXT: TEX 0 @6 -; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; EG-NEXT: CF_END ; EG-NEXT: PAD @@ -340,9 +340,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i8_with_select(ptr addrspace(1) noa ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, 0.0, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: BFE_INT * T0.W, T0.X, 0.0, literal.x, -; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00) -; EG-NEXT: FFBL_INT T0.W, PV.W, +; EG-NEXT: FFBL_INT T0.W, T0.X, ; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, @@ -402,7 +400,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i16_with_select(ptr addrspace(1) no ; EG: ; %bb.0: ; EG-NEXT: ALU 0, @8, KC0[], KC1[] ; EG-NEXT: TEX 0 @6 -; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; EG-NEXT: CF_END ; EG-NEXT: PAD @@ -411,9 +409,7 @@ define amdgpu_kernel void @s_cttz_zero_undef_i16_with_select(ptr addrspace(1) no ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, 0.0, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: BFE_INT * T0.W, T0.X, 0.0, literal.x, -; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) -; EG-NEXT: FFBL_INT T0.W, PV.W, +; EG-NEXT: FFBL_INT T0.W, T0.X, ; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll index d37819ac69412..5d6972dcaea13 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll @@ -602,64 +602,54 @@ define amdgpu_kernel void @v2i8_arg(ptr addrspace(1) %out, <2 x i8> %in) { ; ; EG-LABEL: v2i8_arg: ; EG: ; %bb.0: ; %entry -; EG-NEXT: ALU 0, @10, KC0[], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 15, @11, KC0[CB0:0-32], KC1[] -; EG-NEXT: MEM_RAT MSKOR T4.XW, T5.X +; EG-NEXT: ALU 0, @8, KC0[], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 11, @9, KC0[CB0:0-32], KC1[] +; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_8 T5.X, T4.X, 41, #3 -; EG-NEXT: VTX_READ_8 T4.X, T4.X, 40, #3 -; EG-NEXT: ALU clause starting at 10: -; EG-NEXT: MOV * T4.X, 0.0, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: LSHL T0.W, T5.X, literal.x, -; EG-NEXT: AND_INT * T1.W, T4.X, literal.y, -; EG-NEXT: 8(1.121039e-44), 255(3.573311e-43) -; EG-NEXT: AND_INT T2.W, KC0[2].Y, literal.x, -; EG-NEXT: OR_INT * T0.W, PV.W, PS, +; EG-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3 +; EG-NEXT: ALU clause starting at 8: +; EG-NEXT: MOV * T0.X, 0.0, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x, +; EG-NEXT: AND_INT * T1.W, T0.X, literal.y, +; EG-NEXT: 3(4.203895e-45), 65535(9.183409e-41) +; EG-NEXT: LSHL * T0.W, PV.W, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) -; EG-NEXT: AND_INT T0.W, PS, literal.x, -; EG-NEXT: LSHL * T1.W, PV.W, literal.y, -; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45) -; EG-NEXT: LSHL T4.X, PV.W, PS, -; EG-NEXT: LSHL * T4.W, literal.x, PS, +; EG-NEXT: LSHL T0.X, T1.W, PV.W, +; EG-NEXT: LSHL * T0.W, literal.x, PV.W, ; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) -; EG-NEXT: MOV T4.Y, 0.0, -; EG-NEXT: MOV * T4.Z, 0.0, -; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; EG-NEXT: MOV T0.Y, 0.0, +; EG-NEXT: MOV * T0.Z, 0.0, +; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) ; ; CM-LABEL: v2i8_arg: ; CM: ; %bb.0: ; %entry -; CM-NEXT: ALU 0, @10, KC0[], KC1[] -; CM-NEXT: TEX 1 @6 -; CM-NEXT: ALU 15, @11, KC0[CB0:0-32], KC1[] -; CM-NEXT: MEM_RAT MSKOR T4.XW, T5.X +; CM-NEXT: ALU 0, @8, KC0[], KC1[] +; CM-NEXT: TEX 0 @6 +; CM-NEXT: ALU 11, @9, KC0[CB0:0-32], KC1[] +; CM-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; CM-NEXT: CF_END ; CM-NEXT: PAD ; CM-NEXT: Fetch clause starting at 6: -; CM-NEXT: VTX_READ_8 T5.X, T4.X, 41, #3 -; CM-NEXT: VTX_READ_8 T4.X, T4.X, 40, #3 -; CM-NEXT: ALU clause starting at 10: -; CM-NEXT: MOV * T4.X, 0.0, -; CM-NEXT: ALU clause starting at 11: -; CM-NEXT: LSHL T0.Z, T5.X, literal.x, -; CM-NEXT: AND_INT * T0.W, T4.X, literal.y, BS:VEC_120/SCL_212 -; CM-NEXT: 8(1.121039e-44), 255(3.573311e-43) -; CM-NEXT: AND_INT T1.Z, KC0[2].Y, literal.x, -; CM-NEXT: OR_INT * T0.W, PV.Z, PV.W, +; CM-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3 +; CM-NEXT: ALU clause starting at 8: +; CM-NEXT: MOV * T0.X, 0.0, +; CM-NEXT: ALU clause starting at 9: +; CM-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x, ; CM-NEXT: 3(4.203895e-45), 0(0.000000e+00) -; CM-NEXT: AND_INT T0.Z, PV.W, literal.x, -; CM-NEXT: LSHL * T0.W, PV.Z, literal.y, +; CM-NEXT: AND_INT T0.Z, T0.X, literal.x, +; CM-NEXT: LSHL * T0.W, PV.W, literal.y, ; CM-NEXT: 65535(9.183409e-41), 3(4.203895e-45) -; CM-NEXT: LSHL T4.X, PV.Z, PV.W, -; CM-NEXT: LSHL * T4.W, literal.x, PV.W, +; CM-NEXT: LSHL T0.X, PV.Z, PV.W, +; CM-NEXT: LSHL * T0.W, literal.x, PV.W, ; CM-NEXT: 65535(9.183409e-41), 0(0.000000e+00) -; CM-NEXT: MOV T4.Y, 0.0, -; CM-NEXT: MOV * T4.Z, 0.0, -; CM-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; CM-NEXT: MOV T0.Y, 0.0, +; CM-NEXT: MOV * T0.Z, 0.0, +; CM-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; CM-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: store <2 x i8> %in, ptr addrspace(1) %out @@ -701,44 +691,24 @@ define amdgpu_kernel void @v2i16_arg(ptr addrspace(1) %out, <2 x i16> %in) { ; ; EG-LABEL: v2i16_arg: ; EG: ; %bb.0: ; %entry -; EG-NEXT: ALU 0, @10, KC0[], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 5, @11, KC0[CB0:0-32], KC1[] -; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1 +; EG-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD -; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_16 T5.X, T4.X, 42, #3 -; EG-NEXT: VTX_READ_16 T4.X, T4.X, 40, #3 -; EG-NEXT: ALU clause starting at 10: -; EG-NEXT: MOV * T4.X, 0.0, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: LSHL T0.W, T5.X, literal.x, -; EG-NEXT: AND_INT * T1.W, T4.X, literal.y, -; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41) -; EG-NEXT: OR_INT T4.X, PV.W, PS, -; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; EG-NEXT: ALU clause starting at 4: +; EG-NEXT: MOV T0.X, KC0[2].Z, +; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) ; ; CM-LABEL: v2i16_arg: ; CM: ; %bb.0: ; %entry -; CM-NEXT: ALU 0, @10, KC0[], KC1[] -; CM-NEXT: TEX 1 @6 -; CM-NEXT: ALU 5, @11, KC0[CB0:0-32], KC1[] -; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T4.X, T5.X +; CM-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X ; CM-NEXT: CF_END ; CM-NEXT: PAD -; CM-NEXT: Fetch clause starting at 6: -; CM-NEXT: VTX_READ_16 T5.X, T4.X, 42, #3 -; CM-NEXT: VTX_READ_16 T4.X, T4.X, 40, #3 -; CM-NEXT: ALU clause starting at 10: -; CM-NEXT: MOV * T4.X, 0.0, -; CM-NEXT: ALU clause starting at 11: -; CM-NEXT: LSHL T0.Z, T5.X, literal.x, -; CM-NEXT: AND_INT * T0.W, T4.X, literal.y, BS:VEC_120/SCL_212 -; CM-NEXT: 16(2.242078e-44), 65535(9.183409e-41) -; CM-NEXT: OR_INT * T4.X, PV.Z, PV.W, -; CM-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; CM-NEXT: ALU clause starting at 4: +; CM-NEXT: MOV * T0.X, KC0[2].Z, +; CM-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; CM-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: store <2 x i16> %in, ptr addrspace(1) %out @@ -1322,68 +1292,24 @@ define amdgpu_kernel void @v4i8_arg(ptr addrspace(1) %out, <4 x i8> %in) { ; ; EG-LABEL: v4i8_arg: ; EG: ; %bb.0: ; %entry -; EG-NEXT: ALU 0, @14, KC0[], KC1[] -; EG-NEXT: TEX 3 @6 -; EG-NEXT: ALU 15, @15, KC0[CB0:0-32], KC1[] -; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1 +; EG-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD -; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_8 T5.X, T4.X, 42, #3 -; EG-NEXT: VTX_READ_8 T6.X, T4.X, 40, #3 -; EG-NEXT: VTX_READ_8 T7.X, T4.X, 43, #3 -; EG-NEXT: VTX_READ_8 T4.X, T4.X, 41, #3 -; EG-NEXT: ALU clause starting at 14: -; EG-NEXT: MOV * T4.X, 0.0, -; EG-NEXT: ALU clause starting at 15: -; EG-NEXT: AND_INT * T0.W, T5.X, literal.x, -; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00) -; EG-NEXT: AND_INT T0.Z, T4.X, literal.x, -; EG-NEXT: LSHL T0.W, PV.W, literal.y, -; EG-NEXT: LSHL * T1.W, T7.X, literal.z, -; EG-NEXT: 255(3.573311e-43), 16(2.242078e-44) -; EG-NEXT: 24(3.363116e-44), 0(0.000000e+00) -; EG-NEXT: OR_INT T0.W, PS, PV.W, -; EG-NEXT: LSHL * T1.W, PV.Z, literal.x, -; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00) -; EG-NEXT: OR_INT T0.W, PV.W, PS, -; EG-NEXT: AND_INT * T1.W, T6.X, literal.x, -; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00) -; EG-NEXT: OR_INT T4.X, PV.W, PS, -; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; EG-NEXT: ALU clause starting at 4: +; EG-NEXT: MOV T0.X, KC0[2].Z, +; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) ; ; CM-LABEL: v4i8_arg: ; CM: ; %bb.0: ; %entry -; CM-NEXT: ALU 0, @14, KC0[], KC1[] -; CM-NEXT: TEX 3 @6 -; CM-NEXT: ALU 15, @15, KC0[CB0:0-32], KC1[] -; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T4.X, T5.X +; CM-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X ; CM-NEXT: CF_END ; CM-NEXT: PAD -; CM-NEXT: Fetch clause starting at 6: -; CM-NEXT: VTX_READ_8 T5.X, T4.X, 42, #3 -; CM-NEXT: VTX_READ_8 T6.X, T4.X, 40, #3 -; CM-NEXT: VTX_READ_8 T7.X, T4.X, 43, #3 -; CM-NEXT: VTX_READ_8 T4.X, T4.X, 41, #3 -; CM-NEXT: ALU clause starting at 14: -; CM-NEXT: MOV * T4.X, 0.0, -; CM-NEXT: ALU clause starting at 15: -; CM-NEXT: AND_INT * T0.W, T5.X, literal.x, -; CM-NEXT: 255(3.573311e-43), 0(0.000000e+00) -; CM-NEXT: AND_INT T0.Y, T4.X, literal.x, -; CM-NEXT: LSHL T0.Z, PV.W, literal.y, -; CM-NEXT: LSHL * T0.W, T7.X, literal.z, BS:VEC_120/SCL_212 -; CM-NEXT: 255(3.573311e-43), 16(2.242078e-44) -; CM-NEXT: 24(3.363116e-44), 0(0.000000e+00) -; CM-NEXT: OR_INT T0.Z, PV.W, PV.Z, -; CM-NEXT: LSHL * T0.W, PV.Y, literal.x, -; CM-NEXT: 8(1.121039e-44), 0(0.000000e+00) -; CM-NEXT: OR_INT T0.Z, PV.Z, PV.W, -; CM-NEXT: AND_INT * T0.W, T6.X, literal.x, -; CM-NEXT: 255(3.573311e-43), 0(0.000000e+00) -; CM-NEXT: OR_INT * T4.X, PV.Z, PV.W, -; CM-NEXT: LSHR * T5.X, KC0[2].Y, literal.x, +; CM-NEXT: ALU clause starting at 4: +; CM-NEXT: MOV * T0.X, KC0[2].Z, +; CM-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; CM-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: store <4 x i8> %in, ptr addrspace(1) %out @@ -1427,115 +1353,27 @@ define amdgpu_kernel void @v4i16_arg(ptr addrspace(1) %out, <4 x i16> %in) { ; ; EG-LABEL: v4i16_arg: ; EG: ; %bb.0: ; %entry -; EG-NEXT: ALU 1, @20, KC0[], KC1[] -; EG-NEXT: TEX 0 @12 -; EG-NEXT: ALU 5, @22, KC0[], KC1[] -; EG-NEXT: TEX 0 @14 -; EG-NEXT: ALU 5, @28, KC0[], KC1[] -; EG-NEXT: TEX 0 @16 -; EG-NEXT: ALU 5, @34, KC0[], KC1[] -; EG-NEXT: TEX 0 @18 -; EG-NEXT: ALU 7, @40, KC0[CB0:0-32], KC1[] -; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T5.XY, T6.X, 1 +; EG-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD -; EG-NEXT: Fetch clause starting at 12: -; EG-NEXT: VTX_READ_16 T6.X, T5.X, 50, #3 -; EG-NEXT: Fetch clause starting at 14: -; EG-NEXT: VTX_READ_16 T6.X, T5.X, 48, #3 -; EG-NEXT: Fetch clause starting at 16: -; EG-NEXT: VTX_READ_16 T6.X, T5.X, 46, #3 -; EG-NEXT: Fetch clause starting at 18: -; EG-NEXT: VTX_READ_16 T5.X, T5.X, 44, #3 -; EG-NEXT: ALU clause starting at 20: -; EG-NEXT: MOV * T0.Y, T3.X, -; EG-NEXT: MOV * T5.X, 0.0, -; EG-NEXT: ALU clause starting at 22: -; EG-NEXT: LSHL T0.W, T6.X, literal.x, -; EG-NEXT: AND_INT * T1.W, T0.Y, literal.y, -; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41) -; EG-NEXT: OR_INT * T0.W, PS, PV.W, -; EG-NEXT: MOV * T3.X, PV.W, -; EG-NEXT: MOV * T0.Y, PV.X, -; EG-NEXT: ALU clause starting at 28: -; EG-NEXT: AND_INT T0.W, T0.Y, literal.x, -; EG-NEXT: AND_INT * T1.W, T6.X, literal.y, -; EG-NEXT: -65536(nan), 65535(9.183409e-41) -; EG-NEXT: OR_INT * T0.W, PV.W, PS, -; EG-NEXT: MOV T3.X, PV.W, -; EG-NEXT: MOV * T0.Y, T2.X, -; EG-NEXT: ALU clause starting at 34: -; EG-NEXT: AND_INT T0.W, T0.Y, literal.x, -; EG-NEXT: LSHL * T1.W, T6.X, literal.y, -; EG-NEXT: 65535(9.183409e-41), 16(2.242078e-44) -; EG-NEXT: OR_INT * T0.W, PV.W, PS, -; EG-NEXT: MOV * T2.X, PV.W, -; EG-NEXT: MOV * T0.Y, PV.X, -; EG-NEXT: ALU clause starting at 40: -; EG-NEXT: LSHR T6.X, KC0[2].Y, literal.x, -; EG-NEXT: AND_INT T0.W, T0.Y, literal.y, -; EG-NEXT: AND_INT * T1.W, T5.X, literal.z, -; EG-NEXT: 2(2.802597e-45), -65536(nan) -; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) -; EG-NEXT: OR_INT * T5.X, PV.W, PS, -; EG-NEXT: MOV T2.X, PV.X, -; EG-NEXT: MOV * T5.Y, T3.X, +; EG-NEXT: ALU clause starting at 4: +; EG-NEXT: MOV * T0.Y, KC0[3].X, +; EG-NEXT: MOV T0.X, KC0[2].W, +; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) ; ; CM-LABEL: v4i16_arg: ; CM: ; %bb.0: ; %entry -; CM-NEXT: ALU 1, @20, KC0[], KC1[] -; CM-NEXT: TEX 0 @12 -; CM-NEXT: ALU 5, @22, KC0[], KC1[] -; CM-NEXT: TEX 0 @14 -; CM-NEXT: ALU 5, @28, KC0[], KC1[] -; CM-NEXT: TEX 0 @16 -; CM-NEXT: ALU 5, @34, KC0[], KC1[] -; CM-NEXT: TEX 0 @18 -; CM-NEXT: ALU 7, @40, KC0[CB0:0-32], KC1[] -; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T5, T6.X +; CM-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; CM-NEXT: MEM_RAT_CACHELESS STORE_DWORD T0, T1.X ; CM-NEXT: CF_END ; CM-NEXT: PAD -; CM-NEXT: Fetch clause starting at 12: -; CM-NEXT: VTX_READ_16 T6.X, T5.X, 50, #3 -; CM-NEXT: Fetch clause starting at 14: -; CM-NEXT: VTX_READ_16 T6.X, T5.X, 48, #3 -; CM-NEXT: Fetch clause starting at 16: -; CM-NEXT: VTX_READ_16 T6.X, T5.X, 46, #3 -; CM-NEXT: Fetch clause starting at 18: -; CM-NEXT: VTX_READ_16 T5.X, T5.X, 44, #3 -; CM-NEXT: ALU clause starting at 20: -; CM-NEXT: MOV * T0.Y, T3.X, -; CM-NEXT: MOV * T5.X, 0.0, -; CM-NEXT: ALU clause starting at 22: -; CM-NEXT: LSHL T0.Z, T6.X, literal.x, -; CM-NEXT: AND_INT * T0.W, T0.Y, literal.y, -; CM-NEXT: 16(2.242078e-44), 65535(9.183409e-41) -; CM-NEXT: OR_INT * T0.W, PV.W, PV.Z, -; CM-NEXT: MOV * T3.X, PV.W, -; CM-NEXT: MOV * T0.Y, PV.X, -; CM-NEXT: ALU clause starting at 28: -; CM-NEXT: AND_INT T0.Z, T0.Y, literal.x, -; CM-NEXT: AND_INT * T0.W, T6.X, literal.y, -; CM-NEXT: -65536(nan), 65535(9.183409e-41) -; CM-NEXT: OR_INT * T0.W, PV.Z, PV.W, -; CM-NEXT: MOV T3.X, PV.W, -; CM-NEXT: MOV * T0.Y, T2.X, -; CM-NEXT: ALU clause starting at 34: -; CM-NEXT: AND_INT T0.Z, T0.Y, literal.x, -; CM-NEXT: LSHL * T0.W, T6.X, literal.y, -; CM-NEXT: 65535(9.183409e-41), 16(2.242078e-44) -; CM-NEXT: OR_INT * T0.W, PV.Z, PV.W, -; CM-NEXT: MOV * T2.X, PV.W, -; CM-NEXT: MOV * T0.Y, PV.X, -; CM-NEXT: ALU clause starting at 40: -; CM-NEXT: LSHR T6.X, KC0[2].Y, literal.x, -; CM-NEXT: AND_INT T0.Z, T0.Y, literal.y, -; CM-NEXT: AND_INT * T0.W, T5.X, literal.z, -; CM-NEXT: 2(2.802597e-45), -65536(nan) -; CM-NEXT: 65535(9.183409e-41), 0(0.000000e+00) -; CM-NEXT: OR_INT * T5.X, PV.Z, PV.W, -; CM-NEXT: MOV T2.X, PV.X, -; CM-NEXT: MOV * T5.Y, T3.X, +; CM-NEXT: ALU clause starting at 4: +; CM-NEXT: MOV * T0.Y, KC0[3].X, +; CM-NEXT: MOV * T0.X, KC0[2].W, +; CM-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; CM-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: store <4 x i16> %in, ptr addrspace(1) %out ret void From 318d5bff0b65aa7d52fc7004d49587416f0fb564 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Mon, 18 Dec 2023 17:21:15 +0100 Subject: [PATCH 140/884] [InstCombine] Favour `m_Poison` in `SimplifyDemandedVectorElts` A miscompilation issue has been addressed with refined checking. --- .../InstCombineSimplifyDemanded.cpp | 4 +- .../Transforms/InstCombine/X86/clmulqdq.ll | 36 ++++-- .../X86/x86-avx512-inseltpoison.ll | 24 ++-- .../Transforms/InstCombine/X86/x86-avx512.ll | 24 ++-- .../Transforms/InstCombine/X86/x86-pshufb.ll | 6 +- llvm/test/Transforms/InstCombine/broadcast.ll | 12 +- .../Transforms/InstCombine/extractelement.ll | 16 +-- .../Transforms/InstCombine/inselt-binop.ll | 108 +++++++++--------- .../InstCombine/insert-extract-shuffle.ll | 8 +- llvm/test/Transforms/InstCombine/pr38984.ll | 2 +- .../Transforms/InstCombine/shuffle_select.ll | 2 +- .../InstCombine/shufflevector-div-rem.ll | 8 +- .../InstCombine/sub-of-negatible.ll | 4 +- .../InstCombine/vec_demanded_elts.ll | 12 +- .../InstCombine/vec_gep_scalar_arg.ll | 2 +- .../Transforms/InstCombine/vec_shuffle.ll | 2 +- .../InstCombine/vector-casts-inseltpoison.ll | 2 +- .../Transforms/InstCombine/vector-casts.ll | 2 +- 18 files changed, 143 insertions(+), 131 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 5dcd7598c2a50..61af2f7c79eff 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1350,8 +1350,8 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return !isa(V) ? PoisonValue::get(V->getType()) : nullptr; } - if (match(V, m_Undef())) { - // If the entire vector is undef or poison, just return this info. + if (match(V, m_Poison())) { + // If the entire vector is poison, just return this info. PoisonElts = EltMask; return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll index 763b79c9e5815..392fbf23d19ea 100644 --- a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll +++ b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll @@ -51,7 +51,8 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_17(<2 x i64> %a0, <2 x i64> %a1) define <2 x i64> @test_demanded_elts_pclmulqdq_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_0( -; CHECK-NEXT: ret <2 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 0) +; CHECK-NEXT: ret <2 x i64> [[TMP1]] ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 0) ret <2 x i64> %1 @@ -59,7 +60,8 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_0() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_1( -; CHECK-NEXT: ret <2 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 1) +; CHECK-NEXT: ret <2 x i64> [[TMP1]] ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 1) ret <2 x i64> %1 @@ -67,7 +69,8 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_1() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_16( -; CHECK-NEXT: ret <2 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 16) +; CHECK-NEXT: ret <2 x i64> [[TMP1]] ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 16) ret <2 x i64> %1 @@ -75,7 +78,8 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_16() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_17( -; CHECK-NEXT: ret <2 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 17) +; CHECK-NEXT: ret <2 x i64> [[TMP1]] ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 17) ret <2 x i64> %1 @@ -135,7 +139,8 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_17(<4 x i64> %a0, <4 x i64> % define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_0( -; CHECK-NEXT: ret <4 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 0) +; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 0) ret <4 x i64> %1 @@ -143,7 +148,8 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_0() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_1( -; CHECK-NEXT: ret <4 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 1) +; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 1) ret <4 x i64> %1 @@ -151,7 +157,8 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_1() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_16( -; CHECK-NEXT: ret <4 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 16) +; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 16) ret <4 x i64> %1 @@ -159,7 +166,8 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_16() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_17( -; CHECK-NEXT: ret <4 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 17) +; CHECK-NEXT: ret <4 x i64> [[TMP1]] ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 17) ret <4 x i64> %1 @@ -235,7 +243,8 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_17(<8 x i64> %a0, <8 x i64> % define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_0( -; CHECK-NEXT: ret <8 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 0) +; CHECK-NEXT: ret <8 x i64> [[TMP1]] ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 0) ret <8 x i64> %1 @@ -243,7 +252,8 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_0() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_1( -; CHECK-NEXT: ret <8 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 1) +; CHECK-NEXT: ret <8 x i64> [[TMP1]] ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 1) ret <8 x i64> %1 @@ -251,7 +261,8 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_1() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_16( -; CHECK-NEXT: ret <8 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 16) +; CHECK-NEXT: ret <8 x i64> [[TMP1]] ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 16) ret <8 x i64> %1 @@ -259,7 +270,8 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_16() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_17( -; CHECK-NEXT: ret <8 x i64> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 17) +; CHECK-NEXT: ret <8 x i64> [[TMP1]] ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 17) ret <8 x i64> %1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll index b27c94667d56d..9b990480709c9 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll @@ -23,7 +23,7 @@ define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_add_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -103,7 +103,7 @@ define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_add_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -175,7 +175,7 @@ define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_sub_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -255,7 +255,7 @@ define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_sub_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -327,7 +327,7 @@ define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_mul_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -407,7 +407,7 @@ define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_mul_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -479,7 +479,7 @@ define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_div_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -559,7 +559,7 @@ define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_div_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -615,7 +615,7 @@ declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_max_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -661,7 +661,7 @@ declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_max_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -699,7 +699,7 @@ declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_min_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -745,7 +745,7 @@ declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_min_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll index ea806a89679c5..c10c922f66432 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll @@ -23,7 +23,7 @@ define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_add_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -103,7 +103,7 @@ define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_add_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -175,7 +175,7 @@ define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_sub_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -255,7 +255,7 @@ define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_sub_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -327,7 +327,7 @@ define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_mul_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -407,7 +407,7 @@ define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_mul_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -479,7 +479,7 @@ define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_div_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -559,7 +559,7 @@ define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_div_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -615,7 +615,7 @@ declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_max_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -661,7 +661,7 @@ declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_max_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -699,7 +699,7 @@ declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_min_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -745,7 +745,7 @@ declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_min_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll index 9fde3237737ec..cd90696eafac6 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll @@ -446,7 +446,7 @@ define <64 x i8> @fold_with_undef_elts_avx512(<64 x i8> %InVec) { define <16 x i8> @fold_with_allundef_elts(<16 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts( -; CHECK-NEXT: ret <16 x i8> undef +; CHECK-NEXT: ret <16 x i8> poison ; %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> undef) ret <16 x i8> %1 @@ -454,7 +454,7 @@ define <16 x i8> @fold_with_allundef_elts(<16 x i8> %InVec) { define <32 x i8> @fold_with_allundef_elts_avx2(<32 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts_avx2( -; CHECK-NEXT: ret <32 x i8> undef +; CHECK-NEXT: ret <32 x i8> poison ; %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> undef) ret <32 x i8> %1 @@ -462,7 +462,7 @@ define <32 x i8> @fold_with_allundef_elts_avx2(<32 x i8> %InVec) { define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts_avx512( -; CHECK-NEXT: ret <64 x i8> undef +; CHECK-NEXT: ret <64 x i8> poison ; %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> undef) ret <64 x i8> %1 diff --git a/llvm/test/Transforms/InstCombine/broadcast.ll b/llvm/test/Transforms/InstCombine/broadcast.ll index c70b975a312ad..ffc5508092b4c 100644 --- a/llvm/test/Transforms/InstCombine/broadcast.ll +++ b/llvm/test/Transforms/InstCombine/broadcast.ll @@ -57,7 +57,7 @@ define <4 x float> @good4(float %arg) { define <4 x float> @good5(float %v) { ; CHECK-LABEL: @good5( -; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> undef, float [[V:%.*]], i64 0 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> , float [[V:%.*]], i64 0 ; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]] ; CHECK-NEXT: [[INS4:%.*]] = shufflevector <4 x float> [[INS1]], <4 x float> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[RES:%.*]] = fadd <4 x float> [[A1]], [[INS4]] @@ -76,7 +76,7 @@ define <4 x float> @good5(float %v) { define <4 x float> @splat_undef1(float %arg) { ; CHECK-LABEL: @splat_undef1( -; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 1 +; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] @@ -92,7 +92,7 @@ define <4 x float> @splat_undef1(float %arg) { define <4 x float> @splat_undef2(float %arg) { ; CHECK-LABEL: @splat_undef2( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] @@ -132,7 +132,7 @@ define <1 x float> @bad4(float %arg) { define <4 x float> @splat_undef3(float %arg) { ; CHECK-LABEL: @splat_undef3( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 @@ -149,7 +149,7 @@ define <4 x float> @splat_undef3(float %arg) { define <4 x float> @bad6(float %arg, i32 %k) { ; CHECK-LABEL: @bad6( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i32 [[K:%.*]] ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 @@ -164,7 +164,7 @@ define <4 x float> @bad6(float %arg, i32 %k) { define <4 x float> @bad7(float %v) { ; CHECK-LABEL: @bad7( -; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> undef, float [[V:%.*]], i64 1 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> , float [[V:%.*]], i64 1 ; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]] ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[INS1]], float [[V]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[V]], i64 2 diff --git a/llvm/test/Transforms/InstCombine/extractelement.ll b/llvm/test/Transforms/InstCombine/extractelement.ll index 3d94106d7f888..bc5dd060a540a 100644 --- a/llvm/test/Transforms/InstCombine/extractelement.ll +++ b/llvm/test/Transforms/InstCombine/extractelement.ll @@ -145,14 +145,14 @@ declare void @use(<8 x i8>) define i8 @bitcasted_inselt_wide_source_uses(i32 %x) { ; ANYLE-LABEL: @bitcasted_inselt_wide_source_uses( -; ANYLE-NEXT: [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i64 0 +; ANYLE-NEXT: [[I:%.*]] = insertelement <2 x i32> , i32 [[X:%.*]], i64 0 ; ANYLE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8> ; ANYLE-NEXT: call void @use(<8 x i8> [[B]]) ; ANYLE-NEXT: [[R:%.*]] = extractelement <8 x i8> [[B]], i64 3 ; ANYLE-NEXT: ret i8 [[R]] ; ; ANYBE-LABEL: @bitcasted_inselt_wide_source_uses( -; ANYBE-NEXT: [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i64 0 +; ANYBE-NEXT: [[I:%.*]] = insertelement <2 x i32> , i32 [[X:%.*]], i64 0 ; ANYBE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8> ; ANYBE-NEXT: call void @use(<8 x i8> [[B]]) ; ANYBE-NEXT: [[R:%.*]] = trunc i32 [[X]] to i8 @@ -188,7 +188,7 @@ declare void @use_v8f32(<8 x float>) define float @bitcasted_inselt_to_FP_uses(i128 %x) { ; ANY-LABEL: @bitcasted_inselt_to_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> , i128 [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2i128(<2 x i128> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float> ; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i64 1 @@ -203,7 +203,7 @@ define float @bitcasted_inselt_to_FP_uses(i128 %x) { define float @bitcasted_inselt_to_FP_uses2(i128 %x) { ; ANY-LABEL: @bitcasted_inselt_to_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> , i128 [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float> ; ANY-NEXT: call void @use_v8f32(<8 x float> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i64 1 @@ -239,7 +239,7 @@ declare void @use_v8i16(<8 x i16>) define i16 @bitcasted_inselt_from_FP_uses(double %x) { ; ANY-LABEL: @bitcasted_inselt_from_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16> ; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i64 1 @@ -254,7 +254,7 @@ define i16 @bitcasted_inselt_from_FP_uses(double %x) { define i16 @bitcasted_inselt_from_FP_uses2(double %x) { ; ANY-LABEL: @bitcasted_inselt_from_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16> ; ANY-NEXT: call void @use_v8i16(<8 x i16> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i64 1 @@ -282,7 +282,7 @@ define float @bitcasted_inselt_to_and_from_FP(double %x) { define float @bitcasted_inselt_to_and_from_FP_uses(double %x) { ; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float> ; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i64 1 @@ -299,7 +299,7 @@ declare void @use_v4f32(<4 x float>) define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) { ; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float> ; ANY-NEXT: call void @use_v4f32(<4 x float> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i64 1 diff --git a/llvm/test/Transforms/InstCombine/inselt-binop.ll b/llvm/test/Transforms/InstCombine/inselt-binop.ll index 6592a59acb906..dc362a0207b37 100644 --- a/llvm/test/Transforms/InstCombine/inselt-binop.ll +++ b/llvm/test/Transforms/InstCombine/inselt-binop.ll @@ -3,7 +3,7 @@ define <2 x i8> @add_constant(i8 %x) { ; CHECK-LABEL: @add_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -14,7 +14,7 @@ define <2 x i8> @add_constant(i8 %x) { define <2 x i8> @add_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @add_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -27,7 +27,7 @@ define <2 x i8> @add_constant_not_undef_lane(i8 %x) { define <2 x i8> @sub_constant_op0(i8 %x) { ; CHECK-LABEL: @sub_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sub nuw nsw <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -38,7 +38,7 @@ define <2 x i8> @sub_constant_op0(i8 %x) { define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sub_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sub nuw <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -49,7 +49,7 @@ define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @sub_constant_op1(i8 %x) { ; CHECK-LABEL: @sub_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -60,7 +60,7 @@ define <2 x i8> @sub_constant_op1(i8 %x) { define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sub_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -71,7 +71,7 @@ define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) { define <3 x i8> @mul_constant(i8 %x) { ; CHECK-LABEL: @mul_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 2 +; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> , i8 [[X:%.*]], i64 2 ; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], ; CHECK-NEXT: ret <3 x i8> [[BO]] ; @@ -82,7 +82,7 @@ define <3 x i8> @mul_constant(i8 %x) { define <3 x i8> @mul_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @mul_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 2 +; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> , i8 [[X:%.*]], i64 2 ; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], ; CHECK-NEXT: ret <3 x i8> [[BO]] ; @@ -93,7 +93,7 @@ define <3 x i8> @mul_constant_not_undef_lane(i8 %x) { define <2 x i8> @shl_constant_op0(i8 %x) { ; CHECK-LABEL: @shl_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -104,7 +104,7 @@ define <2 x i8> @shl_constant_op0(i8 %x) { define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @shl_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -115,7 +115,7 @@ define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @shl_constant_op1(i8 %x) { ; CHECK-LABEL: @shl_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -126,7 +126,7 @@ define <2 x i8> @shl_constant_op1(i8 %x) { define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @shl_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -137,7 +137,7 @@ define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @ashr_constant_op0(i8 %x) { ; CHECK-LABEL: @ashr_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = ashr exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -148,7 +148,7 @@ define <2 x i8> @ashr_constant_op0(i8 %x) { define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @ashr_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -159,7 +159,7 @@ define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @ashr_constant_op1(i8 %x) { ; CHECK-LABEL: @ashr_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -170,7 +170,7 @@ define <2 x i8> @ashr_constant_op1(i8 %x) { define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @ashr_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -181,7 +181,7 @@ define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @lshr_constant_op0(i8 %x) { ; CHECK-LABEL: @lshr_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -192,7 +192,7 @@ define <2 x i8> @lshr_constant_op0(i8 %x) { define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @lshr_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -203,7 +203,7 @@ define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @lshr_constant_op1(i8 %x) { ; CHECK-LABEL: @lshr_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -214,7 +214,7 @@ define <2 x i8> @lshr_constant_op1(i8 %x) { define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @lshr_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -225,7 +225,7 @@ define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @urem_constant_op0(i8 %x) { ; CHECK-LABEL: @urem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -236,7 +236,7 @@ define <2 x i8> @urem_constant_op0(i8 %x) { define <2 x i8> @urem_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @urem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -256,7 +256,7 @@ define <2 x i8> @urem_constant_op1(i8 %x) { define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @urem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -267,7 +267,7 @@ define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @srem_constant_op0(i8 %x) { ; CHECK-LABEL: @srem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -278,7 +278,7 @@ define <2 x i8> @srem_constant_op0(i8 %x) { define <2 x i8> @srem_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @srem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -298,7 +298,7 @@ define <2 x i8> @srem_constant_op1(i8 %x) { define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @srem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -309,7 +309,7 @@ define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @udiv_constant_op0(i8 %x) { ; CHECK-LABEL: @udiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -320,7 +320,7 @@ define <2 x i8> @udiv_constant_op0(i8 %x) { define <2 x i8> @udiv_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @udiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -340,7 +340,7 @@ define <2 x i8> @udiv_constant_op1(i8 %x) { define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @udiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = udiv <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -351,7 +351,7 @@ define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @sdiv_constant_op0(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -362,7 +362,7 @@ define <2 x i8> @sdiv_constant_op0(i8 %x) { define <2 x i8> @sdiv_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -382,7 +382,7 @@ define <2 x i8> @sdiv_constant_op1(i8 %x) { define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sdiv exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -393,7 +393,7 @@ define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @and_constant(i8 %x) { ; CHECK-LABEL: @and_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -404,7 +404,7 @@ define <2 x i8> @and_constant(i8 %x) { define <2 x i8> @and_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @and_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -415,7 +415,7 @@ define <2 x i8> @and_constant_not_undef_lane(i8 %x) { define <2 x i8> @or_constant(i8 %x) { ; CHECK-LABEL: @or_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -426,7 +426,7 @@ define <2 x i8> @or_constant(i8 %x) { define <2 x i8> @or_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @or_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -437,7 +437,7 @@ define <2 x i8> @or_constant_not_undef_lane(i8 %x) { define <2 x i8> @xor_constant(i8 %x) { ; CHECK-LABEL: @xor_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -448,7 +448,7 @@ define <2 x i8> @xor_constant(i8 %x) { define <2 x i8> @xor_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @xor_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -459,7 +459,7 @@ define <2 x i8> @xor_constant_not_undef_lane(i8 %x) { define <2 x float> @fadd_constant(float %x) { ; CHECK-LABEL: @fadd_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -470,7 +470,7 @@ define <2 x float> @fadd_constant(float %x) { define <2 x float> @fadd_constant_not_undef_lane(float %x) { ; CHECK-LABEL: @fadd_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -481,7 +481,7 @@ define <2 x float> @fadd_constant_not_undef_lane(float %x) { define <2 x float> @fsub_constant_op0(float %x) { ; CHECK-LABEL: @fsub_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fsub fast <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -492,7 +492,7 @@ define <2 x float> @fsub_constant_op0(float %x) { define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @fsub_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fsub nsz <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -503,7 +503,7 @@ define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) { define <2 x float> @fsub_constant_op1(float %x) { ; CHECK-LABEL: @fsub_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -514,7 +514,7 @@ define <2 x float> @fsub_constant_op1(float %x) { define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @fsub_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -525,7 +525,7 @@ define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) { define <2 x float> @fmul_constant(float %x) { ; CHECK-LABEL: @fmul_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fmul reassoc <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -536,7 +536,7 @@ define <2 x float> @fmul_constant(float %x) { define <2 x float> @fmul_constant_not_undef_lane(float %x) { ; CHECK-LABEL: @fmul_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fmul <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -547,7 +547,7 @@ define <2 x float> @fmul_constant_not_undef_lane(float %x) { define <2 x float> @fdiv_constant_op0(float %x) { ; CHECK-LABEL: @fdiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fdiv nnan <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -558,7 +558,7 @@ define <2 x float> @fdiv_constant_op0(float %x) { define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @fdiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv ninf <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -569,7 +569,7 @@ define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) { define <2 x float> @fdiv_constant_op1(float %x) { ; CHECK-LABEL: @fdiv_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -580,7 +580,7 @@ define <2 x float> @fdiv_constant_op1(float %x) { define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @fdiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -591,7 +591,7 @@ define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) { define <2 x float> @frem_constant_op0(float %x) { ; CHECK-LABEL: @frem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = frem fast <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -602,7 +602,7 @@ define <2 x float> @frem_constant_op0(float %x) { define <2 x float> @frem_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @frem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = frem <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -613,7 +613,7 @@ define <2 x float> @frem_constant_op0_not_undef_lane(float %x) { define <2 x float> @frem_constant_op1(float %x) { ; CHECK-LABEL: @frem_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = frem ninf <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -624,7 +624,7 @@ define <2 x float> @frem_constant_op1(float %x) { define <2 x float> @frem_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @frem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = frem nnan <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll index 5034f44b6a6f9..c87e2e8596c62 100644 --- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll +++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll @@ -290,7 +290,7 @@ define <4 x float> @collectShuffleElts(<2 x float> %x, float %y) { ; CHECK-LABEL: @collectShuffleElts( ; CHECK-NEXT: [[X0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 ; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x float> [[X]], i64 1 -; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> undef, float [[X0]], i64 1 +; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> , float [[X0]], i64 1 ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[X1]], i64 2 ; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Y:%.*]], i64 3 ; CHECK-NEXT: ret <4 x float> [[V3]] @@ -462,7 +462,7 @@ define <5 x i7> @insert_nonzero_index_splat_widen(i7 %x) { define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) { ; CHECK-LABEL: @insert_nonzero_index_splat_extra_use( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 2 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i64 2 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] @@ -490,7 +490,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) { ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 [[INDEX:%.*]] +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i32 [[INDEX:%.*]] ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] ; @@ -513,7 +513,7 @@ define <4 x float> @insert_in_splat(float %x) { define <4 x float> @insert_in_splat_extra_uses(float %x) { ; CHECK-LABEL: @insert_in_splat_extra_uses( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i64 0 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x float> [[SPLAT]]) diff --git a/llvm/test/Transforms/InstCombine/pr38984.ll b/llvm/test/Transforms/InstCombine/pr38984.ll index af1d05a4654f7..c148765fce59f 100644 --- a/llvm/test/Transforms/InstCombine/pr38984.ll +++ b/llvm/test/Transforms/InstCombine/pr38984.ll @@ -25,7 +25,7 @@ define <4 x i1> @PR38984_2() { ; CHECK-LABEL: @PR38984_2( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @offsets, align 2 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> undef, i16 [[TMP0]], i64 3 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> , i16 [[TMP0]], i64 3 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds ([21 x i16], ptr @a, i16 1, i16 0), <4 x i16> [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr null, <4 x i16> [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x ptr> [[TMP2]], [[TMP3]] diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll index 12bf09f8aeb76..a1b0d782b554f 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll @@ -1521,7 +1521,7 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) { define <4 x i32> @PR41419(<4 x i32> %v) { ; CHECK-LABEL: @PR41419( -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> , <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[S]] ; %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll index 457300a25e769..1699418dcc28b 100644 --- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll +++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll @@ -26,7 +26,7 @@ define i16 @test_srem_orig(i16 %a, i1 %cmp) { ; "evaluateInDifferentElementOrder". define <2 x i16> @test_srem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_srem( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -41,7 +41,7 @@ define <2 x i16> @test_srem(i16 %a, i1 %cmp) { define <2 x i16> @test_urem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_urem( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -56,7 +56,7 @@ define <2 x i16> @test_urem(i16 %a, i1 %cmp) { define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_sdiv( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -71,7 +71,7 @@ define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) { define <2 x i16> @test_udiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_udiv( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll index 790d551e5b1de..f2a28c0dd02b3 100644 --- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll +++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll @@ -843,8 +843,8 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef( ; CHECK-NEXT: [[T0_NEG:%.*]] = shl <2 x i4> , [[X:%.*]] -; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> -; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]] +; CHECK-NEXT: [[T11_NEG:%.*]] = insertelement <2 x i4> [[T0_NEG]], i4 undef, i64 1 +; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T11_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i4> [[T2]] ; %t0 = shl <2 x i4> , %x diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll index 576af85b5ee27..c84d39c5aa9dd 100644 --- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll +++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll @@ -148,7 +148,7 @@ define <2 x i64> @PR24922(<2 x i64> %v) { define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) { ; CHECK-LABEL: @inselt_shuf_no_demand( -; CHECK-NEXT: ret <4 x float> undef +; CHECK-NEXT: ret <4 x float> ; %out1 = insertelement <4 x float> undef, float %a1, i32 1 %out12 = insertelement <4 x float> %out1, float %a2, i32 2 @@ -161,7 +161,7 @@ define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) { define <4 x float> @inselt_shuf_no_demand_commute(float %a1, float %a2, float %a3) { ; CHECK-LABEL: @inselt_shuf_no_demand_commute( -; CHECK-NEXT: ret <4 x float> undef +; CHECK-NEXT: ret <4 x float> ; %out1 = insertelement <4 x float> undef, float %a1, i32 1 %out12 = insertelement <4 x float> %out1, float %a2, i32 2 @@ -192,7 +192,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) { ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain( -; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]] +; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> , float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]] ; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SHUFFLE]] ; @@ -617,8 +617,8 @@ define ptr @gep_splat_both(ptr %base, i64 %idx) { define <2 x ptr> @gep_all_lanes_undef(ptr %base, i64 %idx) {; ; CHECK-LABEL: @gep_all_lanes_undef( -; CHECK-NEXT: [[BASEVEC:%.*]] = insertelement <2 x ptr> undef, ptr [[BASE:%.*]], i64 0 -; CHECK-NEXT: [[IDXVEC:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i64 1 +; CHECK-NEXT: [[BASEVEC:%.*]] = insertelement <2 x ptr> , ptr [[BASE:%.*]], i64 0 +; CHECK-NEXT: [[IDXVEC:%.*]] = insertelement <2 x i64> , i64 [[IDX:%.*]], i64 1 ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC]], <2 x i64> [[IDXVEC]] ; CHECK-NEXT: ret <2 x ptr> [[GEP]] ; @@ -780,7 +780,7 @@ define <4 x float> @ins_of_ext_twice(<4 x float> %x, float %y) { define <4 x float> @ins_of_ext_wrong_demand(<4 x float> %x, float %y) { ; CHECK-LABEL: @ins_of_ext_wrong_demand( ; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i64 0 -; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> undef, float [[E0]], i64 0 +; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> , float [[E0]], i64 0 ; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[Y:%.*]], i64 1 ; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i64 2 ; CHECK-NEXT: ret <4 x float> [[I2]] diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll index 69149720c9335..4e4fa7defa8cf 100644 --- a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll +++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll @@ -4,7 +4,7 @@ define <4 x ptr> @PR41270(ptr %x) { ; CHECK-LABEL: @PR41270( ; CHECK-NEXT: [[T3:%.*]] = getelementptr inbounds [4 x i16], ptr [[X:%.*]], i64 0, i64 3 -; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x ptr> undef, ptr [[T3]], i64 0 +; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x ptr> , ptr [[T3]], i64 0 ; CHECK-NEXT: ret <4 x ptr> [[INS2]] ; %ins = insertelement <4 x ptr> undef, ptr %x, i32 0 diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index 8eff837d6e1a3..250a175ad0ebe 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -81,7 +81,7 @@ define float @testvscale6( %X) { define <4 x float> @test7(<4 x float> %x) { ; CHECK-LABEL: @test7( -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> , <4 x i32> ; CHECK-NEXT: ret <4 x float> [[R]] ; %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 > diff --git a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll index 67ef387637537..cf1b72fbcf3e1 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll @@ -294,7 +294,7 @@ define <8 x i32> @pr24458(<8 x float> %n) { define <3 x i16> @trunc_inselt_undef(i32 %x) { ; CHECK-LABEL: @trunc_inselt_undef( ; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16 -; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i64 1 +; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> , i16 [[TMP1]], i64 1 ; CHECK-NEXT: ret <3 x i16> [[TRUNC]] ; %vec = insertelement <3 x i32> poison, i32 %x, i32 1 diff --git a/llvm/test/Transforms/InstCombine/vector-casts.ll b/llvm/test/Transforms/InstCombine/vector-casts.ll index 21e0818fa001e..281fc5f6011ea 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts.ll @@ -294,7 +294,7 @@ define <8 x i32> @pr24458(<8 x float> %n) { define <3 x i16> @trunc_inselt_undef(i32 %x) { ; CHECK-LABEL: @trunc_inselt_undef( ; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16 -; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i64 1 +; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> , i16 [[TMP1]], i64 1 ; CHECK-NEXT: ret <3 x i16> [[TRUNC]] ; %vec = insertelement <3 x i32> undef, i32 %x, i32 1 From f888e739d3247d09e287e4642ef85c1f9bb1ce43 Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Mon, 18 Dec 2023 16:38:36 +0000 Subject: [PATCH 141/884] [Clang][SME2] Add multi-vector zip & unzip builtins (#74841) Adds the following SME2 builtins: - svzip (x2 & x4) - svzipq (x2 & x4) - svuzp (x2 & x4) - svuzpq (x2 & x4) See https://github.com/ARM-software/acle/pull/217/files Patch by David Sherwood --- clang/include/clang/Basic/arm_sve.td | 16 + .../acle_sme2_vector_uzpx2.c | 651 ++++++++++++ .../acle_sme2_vector_uzpx4.c | 939 ++++++++++++++++++ .../acle_sme2_vector_zipx2.c | 651 ++++++++++++ .../acle_sme2_vector_zipx4.c | 938 +++++++++++++++++ 5 files changed, 3195 insertions(+) create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx2.c create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx4.c create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx2.c create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx4.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index cbc2af73d6052..741b799da2efd 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -2308,6 +2308,22 @@ let TargetGuard = "sme2" in { def SVQCVTN_U16_S64_X4 : SInst<"svqcvtn_u16[_{d}_x4]", "b4.d", "l", MergeNone, "aarch64_sve_sqcvtun_x4", [IsStreaming], []>; } +// +// Multi-vector zip/unzip +// + +let TargetGuard = "sme2" in { + def SVZIP_X2 : SInst<"svzip[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x2", [IsStreaming], []>; + def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>; + def SVZIP_X4 : SInst<"svzip[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x4", [IsStreaming], []>; + def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>; + + def SVUZP_X2 : SInst<"svuzp[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x2", [IsStreaming], []>; + def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>; + def SVUZP_X4 : SInst<"svuzp[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x4", [IsStreaming], []>; + def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>; +} + // // Multi-vector unpack // diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx2.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx2.c new file mode 100644 index 0000000000000..118b73972575f --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx2.c @@ -0,0 +1,651 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1,A2) A1##A2 +#endif + +// 8-bit UZPs + +// CHECK-LABEL: @test_svuzp_s8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z16test_svuzp_s8_x210svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint8x2_t test_svuzp_s8_x2(svint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s8_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_u8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z16test_svuzp_u8_x211svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint8x2_t test_svuzp_u8_x2(svuint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u8_x2)(zn); +} + +// 16-bit UZPs + +// CHECK-LABEL: @test_svuzp_s16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s16_x211svint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint16x2_t test_svuzp_s16_x2(svint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_u16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u16_x212svuint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint16x2_t test_svuzp_u16_x2(svuint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_f16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f16_x213svfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat16x2_t test_svuzp_f16_x2(svfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_bf16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzp_bf16_x214svbfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svbfloat16x2_t test_svuzp_bf16_x2(svbfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_bf16_x2)(zn); +} + +// 32-bit UZPs + +// CHECK-LABEL: @test_svuzp_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint32x2_t test_svuzp_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s32_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_u32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u32_x212svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint32x2_t test_svuzp_u32_x2(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u32_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat32x2_t test_svuzp_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f32_x2)(zn); +} + +// 64-bit UZPs + +// CHECK-LABEL: @test_svuzp_s64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s64_x211svint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint64x2_t test_svuzp_s64_x2(svint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s64_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_u64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u64_x212svuint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint64x2_t test_svuzp_u64_x2(svuint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u64_x2)(zn); +} + +// CHECK-LABEL: @test_svuzp_f64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f64_x213svfloat64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzp.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat64x2_t test_svuzp_f64_x2(svfloat64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f64_x2)(zn); +} + +// 128-bit UZPs + +// CHECK-LABEL: @test_svuzpq_s8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzpq_s8_x210svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint8x2_t test_svuzpq_s8_x2(svint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s8_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzpq_u8_x211svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint8x2_t test_svuzpq_u8_x2(svuint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u8_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s16_x211svint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint16x2_t test_svuzpq_s16_x2(svint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u16_x212svuint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint16x2_t test_svuzpq_u16_x2(svuint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f16_x213svfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat16x2_t test_svuzpq_f16_x2(svfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_bf16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z19test_svuzpq_bf16_x214svbfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svbfloat16x2_t test_svuzpq_bf16_x2(svbfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_bf16_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint32x2_t test_svuzpq_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s32_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u32_x212svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint32x2_t test_svuzpq_u32_x2(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u32_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat32x2_t test_svuzpq_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s64_x211svint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint64x2_t test_svuzpq_s64_x2(svint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s64_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u64_x212svuint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint64x2_t test_svuzpq_u64_x2(svuint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u64_x2)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f64_x213svfloat64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.uzpq.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat64x2_t test_svuzpq_f64_x2(svfloat64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f64_x2)(zn); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx4.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx4.c new file mode 100644 index 0000000000000..9dcd995445c25 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_uzpx4.c @@ -0,0 +1,939 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1,A2) A1##A2 +#endif + +// 8-bit UZPs + +// CHECK-LABEL: @test_svuzp_s8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z16test_svuzp_s8_x410svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint8x4_t test_svuzp_s8_x4(svint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s8_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_u8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z16test_svuzp_u8_x411svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint8x4_t test_svuzp_u8_x4(svuint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u8_x4)(zn); +} + +// 16-bit UZPs + +// CHECK-LABEL: @test_svuzp_s16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s16_x411svint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint16x4_t test_svuzp_s16_x4(svint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_u16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u16_x412svuint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint16x4_t test_svuzp_u16_x4(svuint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_f16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f16_x413svfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat16x4_t test_svuzp_f16_x4(svfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_bf16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzp_bf16_x414svbfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svbfloat16x4_t test_svuzp_bf16_x4(svbfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_bf16_x4)(zn); +} + +// 32-bit UZPs + +// CHECK-LABEL: @test_svuzp_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint32x4_t test_svuzp_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s32_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint32x4_t test_svuzp_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u32_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat32x4_t test_svuzp_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f32_x4)(zn); +} + +// 64-bit UZPs + +// CHECK-LABEL: @test_svuzp_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint64x4_t test_svuzp_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_s64_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint64x4_t test_svuzp_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_u64_x4)(zn); +} + +// CHECK-LABEL: @test_svuzp_f64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzp_f64_x413svfloat64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzp.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat64x4_t test_svuzp_f64_x4(svfloat64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzp,_f64_x4)(zn); +} + +// 128-bit UZPs + +// CHECK-LABEL: @test_svuzpq_s8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzpq_s8_x410svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint8x4_t test_svuzpq_s8_x4(svint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s8_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svuzpq_u8_x411svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint8x4_t test_svuzpq_u8_x4(svuint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u8_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s16_x411svint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint16x4_t test_svuzpq_s16_x4(svint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u16_x412svuint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint16x4_t test_svuzpq_u16_x4(svuint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f16_x413svfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat16x4_t test_svuzpq_f16_x4(svfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_bf16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z19test_svuzpq_bf16_x414svbfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svbfloat16x4_t test_svuzpq_bf16_x4(svbfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_bf16_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint32x4_t test_svuzpq_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s32_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint32x4_t test_svuzpq_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u32_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat32x4_t test_svuzpq_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f32_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint64x4_t test_svuzpq_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_s64_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint64x4_t test_svuzpq_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_u64_x4)(zn); +} + +// CHECK-LABEL: @test_svuzpq_f64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svuzpq_f64_x413svfloat64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.uzpq.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat64x4_t test_svuzpq_f64_x4(svfloat64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svuzpq,_f64_x4)(zn); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx2.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx2.c new file mode 100644 index 0000000000000..17555dc7683f4 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx2.c @@ -0,0 +1,651 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1,A2) A1##A2 +#endif + +// 8-bit ZIPs + +// CHECK-LABEL: @test_svzip_s8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z16test_svzip_s8_x210svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint8x2_t test_svzip_s8_x2(svint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s8_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_u8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z16test_svzip_u8_x211svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint8x2_t test_svzip_u8_x2(svuint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u8_x2)(zn); +} + +// 16-bit ZIPs + +// CHECK-LABEL: @test_svzip_s16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s16_x211svint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint16x2_t test_svzip_s16_x2(svint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s16_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_u16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u16_x212svuint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint16x2_t test_svzip_u16_x2(svuint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u16_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_f16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f16_x213svfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat16x2_t test_svzip_f16_x2(svfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f16_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_bf16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzip_bf16_x214svbfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svbfloat16x2_t test_svzip_bf16_x2(svbfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_bf16_x2)(zn); +} + +// 32-bit ZIPs + +// CHECK-LABEL: @test_svzip_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint32x2_t test_svzip_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s32_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_u32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u32_x212svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint32x2_t test_svzip_u32_x2(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u32_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat32x2_t test_svzip_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f32_x2)(zn); +} + +// 64-bit ZIPs + +// CHECK-LABEL: @test_svzip_s64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s64_x211svint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint64x2_t test_svzip_s64_x2(svint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s64_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_u64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u64_x212svuint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint64x2_t test_svzip_u64_x2(svuint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u64_x2)(zn); +} + +// CHECK-LABEL: @test_svzip_f64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f64_x213svfloat64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zip.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat64x2_t test_svzip_f64_x2(svfloat64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f64_x2)(zn); +} + +// 128-bit ZIPs + +// CHECK-LABEL: @test_svzipq_s8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzipq_s8_x210svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint8x2_t test_svzipq_s8_x2(svint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s8_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_u8_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z17test_svzipq_u8_x211svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv16i8( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i8.nxv16i8( [[TMP4]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint8x2_t test_svzipq_u8_x2(svuint8x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u8_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_s16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s16_x211svint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint16x2_t test_svzipq_s16_x2(svint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s16_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_u16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u16_x212svuint16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv16i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8i16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i16.nxv8i16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint16x2_t test_svzipq_u16_x2(svuint16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u16_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_f16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f16_x213svfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv16f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8f16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f16.nxv8f16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat16x2_t test_svzipq_f16_x2(svfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f16_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_bf16_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z19test_svzipq_bf16_x214svbfloat16x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv16bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv8bf16( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16bf16.nxv8bf16( [[TMP4]], [[TMP5]], i64 8) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svbfloat16x2_t test_svzipq_bf16_x2(svbfloat16x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_bf16_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint32x2_t test_svzipq_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s32_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_u32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u32_x212svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4i32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i32.nxv4i32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint32x2_t test_svzipq_u32_x2(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u32_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv8f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv4f32( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f32.nxv4f32( [[TMP4]], [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat32x2_t test_svzipq_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_s64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s64_x211svint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svint64x2_t test_svzipq_s64_x2(svint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s64_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_u64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u64_x212svuint64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv4i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2i64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4i64.nxv2i64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svuint64x2_t test_svzipq_u64_x2(svuint64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u64_x2)(zn); +} + +// CHECK-LABEL: @test_svzipq_f64_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f64_x213svfloat64x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv4f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { , } @llvm.aarch64.sve.zipq.x2.nxv2f64( [[TMP0]], [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( poison, [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv4f64.nxv2f64( [[TMP4]], [[TMP5]], i64 2) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svfloat64x2_t test_svzipq_f64_x2(svfloat64x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f64_x2)(zn); +} + diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx4.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx4.c new file mode 100644 index 0000000000000..ef88f724011bf --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_zipx4.c @@ -0,0 +1,938 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1,A2) A1##A2 +#endif + +// 8-bit ZIPs + +// CHECK-LABEL: @test_svzip_s8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z16test_svzip_s8_x410svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint8x4_t test_svzip_s8_x4(svint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s8_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_u8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z16test_svzip_u8_x411svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint8x4_t test_svzip_u8_x4(svuint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u8_x4)(zn); +} + +// 16-bit ZIPs + +// CHECK-LABEL: @test_svzip_s16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s16_x411svint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint16x4_t test_svzip_s16_x4(svint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s16_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_u16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u16_x412svuint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint16x4_t test_svzip_u16_x4(svuint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u16_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_f16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f16_x413svfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat16x4_t test_svzip_f16_x4(svfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f16_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_bf16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzip_bf16_x414svbfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svbfloat16x4_t test_svzip_bf16_x4(svbfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_bf16_x4)(zn); +} + +// 32-bit ZIPs + +// CHECK-LABEL: @test_svzip_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint32x4_t test_svzip_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s32_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint32x4_t test_svzip_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u32_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat32x4_t test_svzip_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f32_x4)(zn); +} + +// 64-bit ZIPs + +// CHECK-LABEL: @test_svzip_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint64x4_t test_svzip_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_s64_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint64x4_t test_svzip_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_u64_x4)(zn); +} + +// CHECK-LABEL: @test_svzip_f64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzip_f64_x413svfloat64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zip.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat64x4_t test_svzip_f64_x4(svfloat64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzip,_f64_x4)(zn); +} + +// 128-bit ZIPs + +// CHECK-LABEL: @test_svzipq_s8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzipq_s8_x410svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint8x4_t test_svzipq_s8_x4(svint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s8_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_u8_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z17test_svzipq_u8_x411svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv64i8( [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv16i8( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP6]], [[TMP7]], i64 16) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP8]], [[TMP9]], i64 32) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv64i8.nxv16i8( [[TMP10]], [[TMP11]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint8x4_t test_svzipq_u8_x4(svuint8x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u8_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_s16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s16_x411svint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint16x4_t test_svzipq_s16_x4(svint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s16_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_u16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u16_x412svuint16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8i16.nxv32i16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8i16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32i16.nxv8i16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint16x4_t test_svzipq_u16_x4(svuint16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u16_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_f16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f16_x413svfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8f16.nxv32f16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8f16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32f16.nxv8f16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat16x4_t test_svzipq_f16_x4(svfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f16_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_bf16_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z19test_svzipq_bf16_x414svbfloat16x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv8bf16.nxv32bf16( [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv8bf16( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP6]], [[TMP7]], i64 8) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP8]], [[TMP9]], i64 16) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv32bf16.nxv8bf16( [[TMP10]], [[TMP11]], i64 24) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svbfloat16x4_t test_svzipq_bf16_x4(svbfloat16x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_bf16_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint32x4_t test_svzipq_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s32_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16i32.nxv4i32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint32x4_t test_svzipq_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u32_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4f32.nxv16f32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv4f32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP6]], [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP8]], [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv16f32.nxv4f32( [[TMP10]], [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat32x4_t test_svzipq_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f32_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svint64x4_t test_svzipq_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_s64_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8i64.nxv2i64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svuint64x4_t test_svzipq_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_u64_x4)(zn); +} + +// CHECK-LABEL: @test_svzipq_f64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CHECK-NEXT: ret [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z18test_svzipq_f64_x413svfloat64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2f64.nxv8f64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { , , , } @llvm.aarch64.sve.zipq.x4.nxv2f64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( poison, [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP6]], [[TMP7]], i64 2) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { , , , } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP8]], [[TMP9]], i64 4) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { , , , } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call @llvm.vector.insert.nxv8f64.nxv2f64( [[TMP10]], [[TMP11]], i64 6) +// CPP-CHECK-NEXT: ret [[TMP12]] +// +svfloat64x4_t test_svzipq_f64_x4(svfloat64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svzipq,_f64_x4)(zn); +} From 3b0705827dbe711788c6b6bec3afa94205db1ce8 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Mon, 18 Dec 2023 17:49:11 +0100 Subject: [PATCH 142/884] [libc++][modules] Adds CMake 3.28 support. (#75700) This is a preparation to start using CMake 3.28 in the CI. --- libcxx/modules/CMakeLists.txt.in | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libcxx/modules/CMakeLists.txt.in b/libcxx/modules/CMakeLists.txt.in index b02b68915b8f4..98168673ebfe9 100644 --- a/libcxx/modules/CMakeLists.txt.in +++ b/libcxx/modules/CMakeLists.txt.in @@ -3,12 +3,16 @@ cmake_minimum_required(VERSION 3.26) project(libc++-modules LANGUAGES CXX) # Enable CMake's module support -if(CMAKE_VERSION VERSION_LESS "3.27.0") - set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "2182bf5c-ef0d-489a-91da-49dbc3090d2a") +if(CMAKE_VERSION VERSION_LESS "3.28.0") + if(CMAKE_VERSION VERSION_LESS "3.27.0") + set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "2182bf5c-ef0d-489a-91da-49dbc3090d2a") + else() + set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "aa1f7df0-828a-4fcd-9afc-2dc80491aca7") + endif() + set(CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP 1) else() - set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "aa1f7df0-828a-4fcd-9afc-2dc80491aca7") + cmake_policy(VERSION 3.28) endif() -set(CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP 1) # Default to C++ extensions being off. Libc++'s modules support have trouble # with extensions right now. From 35a77fc13ce62f4bc21b65c130801cb5f42fb228 Mon Sep 17 00:00:00 2001 From: Momchil Velikov Date: Mon, 18 Dec 2023 16:49:59 +0000 Subject: [PATCH 143/884] [Clang][SVE2.1] Update names of the `svwhileXX` builtins with predicate-as-counter (#75200) The `_s64`/`_u64` part can be omitted now and the name variants do not include unsigned comparison mnemonics. Both are inferred from the argument types. --- clang/include/clang/Basic/arm_sve.td | 18 +-- .../acle_sve2p1_while_pn.c | 139 ++++++++++-------- .../acle_sve2p1_imm.cpp | 103 ++++++------- 3 files changed, 135 insertions(+), 125 deletions(-) diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 741b799da2efd..9dc22933c49b6 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1979,17 +1979,15 @@ let TargetGuard = "sve2p1|sme2" in { //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; -} -let TargetGuard = "sve2p1" in { -def SVWHILEGE_COUNT : SInst<"svwhilege_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELE_COUNT : SInst<"svwhilele_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELT_COUNT : SInst<"svwhilelt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELO_COUNT : SInst<"svwhilelo_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELS_COUNT : SInst<"svwhilels_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHI_COUNT : SInst<"svwhilehi_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHS_COUNT : SInst<"svwhilehs_{d}", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVLD1B_X2 : MInst<"svld1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; def SVLD1H_X2 : MInst<"svld1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c index 3dbb38582b676..143a43b4a9219 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c @@ -1,12 +1,21 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s // REQUIRES: aarch64-registered-target #include +#ifdef SVE_OVERLOADED_FORMS +#define SVE_ACLE_FUNC(A1, A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1, A2) A1##A2 +#endif + // WHILEGE @@ -22,7 +31,7 @@ // svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) { - return svwhilege_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilege_c8_vl4( @@ -37,7 +46,7 @@ svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) { - return svwhilege_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilege_c16_vl2( @@ -52,7 +61,7 @@ svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) { - return svwhilege_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilege_c16_vl4( @@ -67,7 +76,7 @@ svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) { - return svwhilege_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilege_c32_vl2( @@ -82,7 +91,7 @@ svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) { - return svwhilege_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilege_c32_vl4( @@ -97,7 +106,7 @@ svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) { - return svwhilege_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilege_c64_vl2( @@ -112,7 +121,7 @@ svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) { - return svwhilege_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilege_c64_vl4( @@ -127,7 +136,7 @@ svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) { - return svwhilege_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 4); } // WHILEGT @@ -145,7 +154,7 @@ svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) { - return svwhilegt_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilegt_c8_vl4( @@ -160,7 +169,7 @@ svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) { - return svwhilegt_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilegt_c16_vl2( @@ -175,7 +184,7 @@ svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) { - return svwhilegt_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilegt_c16_vl4( @@ -190,7 +199,7 @@ svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) { - return svwhilegt_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilegt_c32_vl2( @@ -205,7 +214,7 @@ svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) { - return svwhilegt_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilegt_c32_vl4( @@ -220,7 +229,7 @@ svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) { - return svwhilegt_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilegt_c64_vl2( @@ -235,7 +244,7 @@ svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) { - return svwhilegt_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilegt_c64_vl4( @@ -250,7 +259,7 @@ svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) { - return svwhilegt_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 4); } @@ -268,7 +277,7 @@ svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) { - return svwhilehi_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehi_c8_vl4( @@ -283,7 +292,7 @@ svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) { - return svwhilehi_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehi_c16_vl2( @@ -298,7 +307,7 @@ svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) { - return svwhilehi_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehi_c16_vl4( @@ -313,7 +322,7 @@ svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) { - return svwhilehi_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehi_c32_vl2( @@ -328,7 +337,7 @@ svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) { - return svwhilehi_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehi_c32_vl4( @@ -343,7 +352,7 @@ svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) { - return svwhilehi_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehi_c64_vl2( @@ -358,7 +367,7 @@ svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) { - return svwhilehi_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehi_c64_vl4( @@ -373,7 +382,7 @@ svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) { - return svwhilehi_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 4); } @@ -391,7 +400,7 @@ svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) { - return svwhilehs_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehs_c8_vl4( @@ -406,7 +415,7 @@ svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) { - return svwhilehs_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehs_c16_vl2( @@ -421,7 +430,7 @@ svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) { - return svwhilehs_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehs_c16_vl4( @@ -436,7 +445,7 @@ svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) { - return svwhilehs_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehs_c32_vl2( @@ -451,7 +460,7 @@ svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) { - return svwhilehs_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehs_c32_vl4( @@ -466,7 +475,7 @@ svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) { - return svwhilehs_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilehs_c64_vl2( @@ -481,7 +490,7 @@ svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) { - return svwhilehs_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilehs_c64_vl4( @@ -496,7 +505,7 @@ svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) { - return svwhilehs_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 4); } @@ -514,7 +523,7 @@ svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) { - return svwhilele_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilele_c8_vl4( @@ -529,7 +538,7 @@ svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) { - return svwhilele_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilele_c16_vl2( @@ -544,7 +553,7 @@ svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) { - return svwhilele_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilele_c16_vl4( @@ -559,7 +568,7 @@ svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) { - return svwhilele_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilele_c32_vl2( @@ -574,7 +583,7 @@ svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) { - return svwhilele_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilele_c32_vl4( @@ -589,7 +598,7 @@ svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) { - return svwhilele_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilele_c64_vl2( @@ -604,7 +613,7 @@ svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) { - return svwhilele_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilele_c64_vl4( @@ -619,7 +628,7 @@ svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) { - return svwhilele_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 4); } @@ -637,7 +646,7 @@ svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) { - return svwhilelo_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelo_c8_vl4( @@ -652,7 +661,7 @@ svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) { - return svwhilelo_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelo_c16_vl2( @@ -667,7 +676,7 @@ svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) { - return svwhilelo_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelo_c16_vl4( @@ -682,7 +691,7 @@ svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) { - return svwhilelo_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelo_c32_vl2( @@ -697,7 +706,7 @@ svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) { - return svwhilelo_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelo_c32_vl4( @@ -712,7 +721,7 @@ svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) { - return svwhilelo_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelo_c64_vl2( @@ -727,7 +736,7 @@ svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) { - return svwhilelo_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelo_c64_vl4( @@ -742,7 +751,7 @@ svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) { - return svwhilelo_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 4); } @@ -760,7 +769,7 @@ svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) { - return svwhilels_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilels_c8_vl4( @@ -775,7 +784,7 @@ svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) { - return svwhilels_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilels_c16_vl2( @@ -790,7 +799,7 @@ svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) { - return svwhilels_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilels_c16_vl4( @@ -805,7 +814,7 @@ svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) { - return svwhilels_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilels_c32_vl2( @@ -820,7 +829,7 @@ svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) { - return svwhilels_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilels_c32_vl4( @@ -835,7 +844,7 @@ svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) { - return svwhilels_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilels_c64_vl2( @@ -850,7 +859,7 @@ svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) { - return svwhilels_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilels_c64_vl4( @@ -865,7 +874,7 @@ svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) // svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) { - return svwhilels_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 4); } @@ -883,7 +892,7 @@ svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) // svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) { - return svwhilelt_c8(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelt_c8_vl4( @@ -898,7 +907,7 @@ svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) { - return svwhilelt_c8(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelt_c16_vl2( @@ -913,7 +922,7 @@ svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) { - return svwhilelt_c16(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelt_c16_vl4( @@ -928,7 +937,7 @@ svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) { - return svwhilelt_c16(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelt_c32_vl2( @@ -943,7 +952,7 @@ svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) { - return svwhilelt_c32(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelt_c32_vl4( @@ -958,7 +967,7 @@ svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) { - return svwhilelt_c32(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 4); } // CHECK-LABEL: @test_svwhilelt_c64_vl2( @@ -973,7 +982,7 @@ svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) { - return svwhilelt_c64(op1, op2, 2); + return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 2); } // CHECK-LABEL: @test_svwhilelt_c64_vl4( @@ -988,5 +997,5 @@ svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) // svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) { - return svwhilelt_c64(op1, op2, 4); + return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 4); } diff --git a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp index f90ae0fb50e5f..a3ec4c5b8b1bc 100644 --- a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp +++ b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp @@ -1,6 +1,6 @@ -// RUN: %clang_cc1 -triple aarch14-none-linux-gnu -target-feature +sve2p1 -fsyntax-only -verify %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -fsyntax-only -verify %s -// REQUIRES: aarch14-registered-target +// REQUIRES: aarch64-registered-target #include void test_svpext_lane_imm_0_3(svcount_t c) { @@ -27,7 +27,43 @@ void test_svpext_lane_x2_imm_0_1(svcount_t c) { svpext_lane_c64_x2(c, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} } -svcount_t test_svwhile_pn(int64_t op1, int64_t op2) { +svcount_t test_svwhile_pn_signed(int64_t op1, int64_t op2) { + svwhilege_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilege_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilege_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilege_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilegt_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilegt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilegt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilegt_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilele_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilele_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilele_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilele_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilelt_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilelt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilelt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + svwhilelt_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} + + svwhilege_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilege_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilege_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilege_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilegt_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilegt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilegt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilegt_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilele_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilele_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilele_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilele_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilelt_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilelt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilelt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} + svwhilelt_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} +} + +svcount_t test_svwhile_pn_unsigned(uint64_t op1, uint64_t op2) { svwhilege_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilege_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilege_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} @@ -36,26 +72,10 @@ svcount_t test_svwhile_pn(int64_t op1, int64_t op2) { svwhilegt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilegt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilegt_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehi_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehi_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehi_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehi_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehs_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehs_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehs_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilehs_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilele_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilele_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilele_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilele_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilelo_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilelo_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilelo_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilelo_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilels_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilels_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilels_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} - svwhilels_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilelt_c8(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilelt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} svwhilelt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}} @@ -69,26 +89,10 @@ svcount_t test_svwhile_pn(int64_t op1, int64_t op2) { svwhilegt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilegt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilegt_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehi_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehi_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehi_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehi_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehs_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehs_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehs_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilehs_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilele_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilele_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilele_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilele_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilelo_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilelo_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilelo_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilelo_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilels_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilels_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilels_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} - svwhilels_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilelt_c8(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilelt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} svwhilelt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}} @@ -97,21 +101,19 @@ svcount_t test_svwhile_pn(int64_t op1, int64_t op2) { void test_cntp(svcount_t c) { svcntp_c8(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} - svcntp_c11(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} + svcntp_c16(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} svcntp_c32(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} - svcntp_c14(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} + svcntp_c64(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}} svcntp_c8(c, 3); // expected-error {{argument should be a multiple of 2}} - svcntp_c11(c, 3); // expected-error {{argument should be a multiple of 2}} + svcntp_c16(c, 3); // expected-error {{argument should be a multiple of 2}} svcntp_c32(c, 3); // expected-error {{argument should be a multiple of 2}} - svcntp_c14(c, 3); // expected-error {{argument should be a multiple of 2}} + svcntp_c64(c, 3); // expected-error {{argument should be a multiple of 2}} } + void test_svdot_lane_2way(svint32_t s32, svuint32_t u32, svint16_t s16, svuint16_t u16, svfloat32_t f32, svfloat16_t f16) { - svdot_lane_s32_s16_s16(s32, s16, s16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}} - svdot_lane_u32_u16_u16(u32, u16, u16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}} - svdot_lane_f32_f16_f16(f32, f16, f16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}} svdot_lane_s32_s16_s16(s32, s16, s16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} svdot_lane_u32_u16_u16(u32, u16, u16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} svdot_lane_f32_f16_f16(f32, f16, f16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} @@ -124,8 +126,8 @@ void test_svbfml_lane(svbfloat16_t zda, svbfloat16_t zn, svbfloat16_t zm, uint64 svmla_lane_bf16(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svmls_lane_bf16(zda, zn, zm, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}} svmls_lane_bf16(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} - svmla_lane_bf16(zda, zn, zm, idx); // expected-errcor {{argument to 'svmla_lane_bf16' must be a constant integer}} - svmls_lane_bf16(zda, zn, zm, idx); // expected-error {{argument to 'svmla_lane_bf16' must be a constant integer}} + svmla_lane_bf16(zda, zn, zm, idx); // expected-error {{argument to 'svmla_lane_bf16' must be a constant integer}} + svmls_lane_bf16(zda, zn, zm, idx); // expected-error {{argument to 'svmls_lane_bf16' must be a constant integer}} } __attribute__((target("+sve2p1+b16b16"))) @@ -133,6 +135,7 @@ void test_svbfmul_lane(svbfloat16_t zn, svbfloat16_t zm, uint64_t idx){ svmul_lane_bf16(zn, zm, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}} svmul_lane_bf16(zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svmul_lane_bf16(zn, zm, idx); // expected-error {{argument to 'svmul_lane_bf16' must be a constant integer}} +} __attribute__((target("+sve2p1"))) void test_svextq_lane(svint16_t zn_i16, svint16_t zm_i16, svfloat16_t zn_f16, svfloat16_t zm_f16){ @@ -148,10 +151,10 @@ void test_svpmov_lane(){ svuint64_t zn_u64; svbool_t pn; - svpmov_lane_u8(zn_u8, -1); // expected-error {{argument value -1 is outside the valid range [0, 0]}} - svpmov_lane_u16(zn_u16, -1); // expected-error {{argument value -1 is outside the valid range [0, 1]}} - svpmov_lane_u32(zn_u32, -1); // expected-error {{argument value -1 is outside the valid range [0, 3]}} - svpmov_lane_u64(zn_u64, -1); // expected-error {{argument value -1 is outside the valid range [0, 7]}} + svpmov_lane_u8(zn_u8, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 0]}} + svpmov_lane_u16(zn_u16, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}} + svpmov_lane_u32(zn_u32, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}} + svpmov_lane_u64(zn_u64, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}} svpmov_lane_u8(zn_u8, 1); // expected-error {{argument value 1 is outside the valid range [0, 0]}} svpmov_lane_u16(zn_u16, 3); // expected-error {{argument value 3 is outside the valid range [0, 1]}} @@ -180,8 +183,8 @@ void test_svget_svset_b(uint64_t idx, svboolx2_t tuple2, svboolx4_t tuple4, svbo svget4(tuple4, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}} svget4(tuple4, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} - svset2(tuple2, idx, res); // expected-error {{argument to 'svste2' must be a constant integer}} - svset4(tupl4, idx, res); // expected-error {{argument to 'svset4' must be a constant integer}} + svset2(tuple2, idx, res); // expected-error {{argument to 'svset2' must be a constant integer}} + svset4(tuple4, idx, res); // expected-error {{argument to 'svset4' must be a constant integer}} svget2(tuple2, idx); // expected-error {{argument to 'svget2' must be a constant integer}} svget4(tuple4, idx); // expected-error {{argument to 'svget4' must be a constant integer}} } From fade67565e1a3d7447335f9b5dfbf85dccb5773b Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Mon, 18 Dec 2023 16:51:10 +0000 Subject: [PATCH 144/884] [AArch64][SME2] Enable bfm builtins for sme2 (#71927) This patch enables the following builtins for SME2 svbfmlslb_f32 svbfmlslb_lane_f32 svbfmlslt_f32 svbfmlslt_lane_f32 Patch by: Kerry McLaughlin --------- Co-authored-by: Matthew Devereau --- clang/include/clang/Basic/arm_sve.td | 16 ++++++++++------ .../acle_sve2p1_bfmlsl.c | 17 +++++++++++++---- .../aarch64-sme2-intrinsics/acle_sme2_imm.cpp | 5 +++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 9dc22933c49b6..98d7028eb2830 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -2067,12 +2067,6 @@ def SVDOT_X2_F : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "f", MergeNone, "aarch64_ def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "i", MergeNone, "aarch64_sve_sdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; - -def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone], []>; -def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone], []>; - -def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; -def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>; } let TargetGuard = "sve2p1|sme" in { @@ -2332,3 +2326,13 @@ let TargetGuard = "sme2" in { def SVSUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "sil", MergeNone, "aarch64_sve_sunpk_x4", [IsStreaming], []>; def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x4", [IsStreaming], []>; } + +let TargetGuard = "sve2p1|sme2" in { +// == BFloat16 multiply-subtract == +// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added + def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>; + def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>; + + def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c index 7e1763a63ec4e..c1d14e16ad17b 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c @@ -2,13 +2,22 @@ // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -target-feature -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming_compatible +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 @@ -29,7 +38,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.bfmlslb( [[ZDA:%.*]], [[ZN:%.*]], [[ZM:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_bfmlslb(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) +svfloat32_t test_bfmlslb(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) ATTR { return SVE_ACLE_FUNC(svbfmlslb,_f32,,)(zda, zn, zm); } @@ -45,7 +54,7 @@ svfloat32_t test_bfmlslb(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.bfmlslb.lane( [[ZDA:%.*]], [[ZN:%.*]], [[ZM:%.*]], i32 7) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_bfmlslb_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) +svfloat32_t test_bfmlslb_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) ATTR { return SVE_ACLE_FUNC(svbfmlslb_lane,_f32,,)(zda, zn, zm, 7); } @@ -63,7 +72,7 @@ svfloat32_t test_bfmlslb_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.bfmlslt( [[ZDA:%.*]], [[ZN:%.*]], [[ZM:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_bfmlslt(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) +svfloat32_t test_bfmlslt(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) ATTR { return SVE_ACLE_FUNC(svbfmlslt,_f32,,)(zda, zn, zm); } @@ -79,7 +88,7 @@ svfloat32_t test_bfmlslt(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.bfmlslt.lane( [[ZDA:%.*]], [[ZN:%.*]], [[ZM:%.*]], i32 7) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_bfmlslt_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) +svfloat32_t test_bfmlslt_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) ATTR { return SVE_ACLE_FUNC(svbfmlslt_lane,_f32,,)(zda, zn, zm, 7); } diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 3750045fd8dff..41cd3c7597638 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -185,3 +185,8 @@ void test_svluti4_lane_zt_x2(svuint8_t zn_u8) __arm_streaming __arm_shared_za __ // Test index value range svluti4_lane_zt_f32_x2(0, zn_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} } + +void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming_compatible { + svbfmlslb_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svbfmlslt_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} +} From 913622d012f72edb5ac3a501cef8639d0ebe471b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 18 Dec 2023 11:01:02 -0600 Subject: [PATCH 145/884] [Libomptarget] Remove remaining global constructors in plugins (#75814) Summary: This patch fixes the remaining global constructor in the plguins after addressing the ones in the JIT interface. This struct was mistakenly using global constructors as not all the members were being initialized properly. This was almost certainly being optimized out because it's trivial, but would still be present in debug builds and prevented us from compiling with `-Werror=global-constructors`. We will want to do that once offloading is moved to a runtimes only build. --- .../common/src/PluginInterface.cpp | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 3c7d1ca899878..1d96468340a08 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -49,15 +49,15 @@ struct RecordReplayTy { private: // Memory pointers for recording, replaying memory. - void *MemoryStart; - void *MemoryPtr; - size_t MemorySize; - size_t TotalSize; - GenericDeviceTy *Device; + void *MemoryStart = nullptr; + void *MemoryPtr = nullptr; + size_t MemorySize = 0; + size_t TotalSize = 0; + GenericDeviceTy *Device = nullptr; std::mutex AllocationLock; - RRStatusTy Status; - bool ReplaySaveOutput; + RRStatusTy Status = RRDeactivated; + bool ReplaySaveOutput = false; bool UsedVAMap = false; uintptr_t MemoryOffset = 0; @@ -190,9 +190,6 @@ struct RecordReplayTy { void setStatus(RRStatusTy Status) { this->Status = Status; } bool isSaveOutputEnabled() const { return ReplaySaveOutput; } - RecordReplayTy() - : Status(RRStatusTy::RRDeactivated), ReplaySaveOutput(false) {} - void saveImage(const char *Name, const DeviceImageTy &Image) { SmallString<128> ImageName = {Name, ".image"}; std::error_code EC; @@ -352,8 +349,9 @@ struct RecordReplayTy { Device->free(MemoryStart); } } +}; -} RecordReplay; +static RecordReplayTy RecordReplay; // Extract the mapping of host function pointers to device function pointers // from the entry table. Functions marked as 'indirect' in OpenMP will have From e400c59beb5f853e109ada06eb5c855c3eab6a31 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 18 Dec 2023 18:08:57 +0100 Subject: [PATCH 146/884] Revert "[InstCombine] Favour `m_Poison` in `SimplifyDemandedVectorElts`" This reverts commit 318d5bff0b65aa7d52fc7004d49587416f0fb564. Has incomplete test updates. --- .../InstCombineSimplifyDemanded.cpp | 4 +- .../Transforms/InstCombine/X86/clmulqdq.ll | 36 ++---- .../X86/x86-avx512-inseltpoison.ll | 24 ++-- .../Transforms/InstCombine/X86/x86-avx512.ll | 24 ++-- .../Transforms/InstCombine/X86/x86-pshufb.ll | 6 +- llvm/test/Transforms/InstCombine/broadcast.ll | 12 +- .../Transforms/InstCombine/extractelement.ll | 16 +-- .../Transforms/InstCombine/inselt-binop.ll | 108 +++++++++--------- .../InstCombine/insert-extract-shuffle.ll | 8 +- llvm/test/Transforms/InstCombine/pr38984.ll | 2 +- .../Transforms/InstCombine/shuffle_select.ll | 2 +- .../InstCombine/shufflevector-div-rem.ll | 8 +- .../InstCombine/sub-of-negatible.ll | 4 +- .../InstCombine/vec_demanded_elts.ll | 12 +- .../InstCombine/vec_gep_scalar_arg.ll | 2 +- .../Transforms/InstCombine/vec_shuffle.ll | 2 +- .../InstCombine/vector-casts-inseltpoison.ll | 2 +- .../Transforms/InstCombine/vector-casts.ll | 2 +- 18 files changed, 131 insertions(+), 143 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 61af2f7c79eff..5dcd7598c2a50 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1350,8 +1350,8 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return !isa(V) ? PoisonValue::get(V->getType()) : nullptr; } - if (match(V, m_Poison())) { - // If the entire vector is poison, just return this info. + if (match(V, m_Undef())) { + // If the entire vector is undef or poison, just return this info. PoisonElts = EltMask; return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll index 392fbf23d19ea..763b79c9e5815 100644 --- a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll +++ b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll @@ -51,8 +51,7 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_17(<2 x i64> %a0, <2 x i64> %a1) define <2 x i64> @test_demanded_elts_pclmulqdq_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_0( -; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 0) -; CHECK-NEXT: ret <2 x i64> [[TMP1]] +; CHECK-NEXT: ret <2 x i64> zeroinitializer ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 0) ret <2 x i64> %1 @@ -60,8 +59,7 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_0() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_1( -; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 1) -; CHECK-NEXT: ret <2 x i64> [[TMP1]] +; CHECK-NEXT: ret <2 x i64> zeroinitializer ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 1) ret <2 x i64> %1 @@ -69,8 +67,7 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_1() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_16( -; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 16) -; CHECK-NEXT: ret <2 x i64> [[TMP1]] +; CHECK-NEXT: ret <2 x i64> zeroinitializer ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 16) ret <2 x i64> %1 @@ -78,8 +75,7 @@ define <2 x i64> @test_demanded_elts_pclmulqdq_undef_16() { define <2 x i64> @test_demanded_elts_pclmulqdq_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_17( -; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 17) -; CHECK-NEXT: ret <2 x i64> [[TMP1]] +; CHECK-NEXT: ret <2 x i64> zeroinitializer ; %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> , <2 x i64> , i8 17) ret <2 x i64> %1 @@ -139,8 +135,7 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_17(<4 x i64> %a0, <4 x i64> % define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_0( -; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 0) -; CHECK-NEXT: ret <4 x i64> [[TMP1]] +; CHECK-NEXT: ret <4 x i64> zeroinitializer ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 0) ret <4 x i64> %1 @@ -148,8 +143,7 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_0() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_1( -; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 1) -; CHECK-NEXT: ret <4 x i64> [[TMP1]] +; CHECK-NEXT: ret <4 x i64> zeroinitializer ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 1) ret <4 x i64> %1 @@ -157,8 +151,7 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_1() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_16( -; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 16) -; CHECK-NEXT: ret <4 x i64> [[TMP1]] +; CHECK-NEXT: ret <4 x i64> zeroinitializer ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 16) ret <4 x i64> %1 @@ -166,8 +159,7 @@ define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_16() { define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_17( -; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 17) -; CHECK-NEXT: ret <4 x i64> [[TMP1]] +; CHECK-NEXT: ret <4 x i64> zeroinitializer ; %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> , <4 x i64> , i8 17) ret <4 x i64> %1 @@ -243,8 +235,7 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_17(<8 x i64> %a0, <8 x i64> % define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_0() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_0( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 0) -; CHECK-NEXT: ret <8 x i64> [[TMP1]] +; CHECK-NEXT: ret <8 x i64> zeroinitializer ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 0) ret <8 x i64> %1 @@ -252,8 +243,7 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_0() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_1() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_1( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 1) -; CHECK-NEXT: ret <8 x i64> [[TMP1]] +; CHECK-NEXT: ret <8 x i64> zeroinitializer ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 1) ret <8 x i64> %1 @@ -261,8 +251,7 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_1() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_16() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_16( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 16) -; CHECK-NEXT: ret <8 x i64> [[TMP1]] +; CHECK-NEXT: ret <8 x i64> zeroinitializer ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 16) ret <8 x i64> %1 @@ -270,8 +259,7 @@ define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_16() { define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_17() { ; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_17( -; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 17) -; CHECK-NEXT: ret <8 x i64> [[TMP1]] +; CHECK-NEXT: ret <8 x i64> zeroinitializer ; %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> , <8 x i64> , i8 17) ret <8 x i64> %1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll index 9b990480709c9..b27c94667d56d 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll @@ -23,7 +23,7 @@ define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_add_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -103,7 +103,7 @@ define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_add_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -175,7 +175,7 @@ define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_sub_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -255,7 +255,7 @@ define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_sub_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -327,7 +327,7 @@ define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_mul_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -407,7 +407,7 @@ define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_mul_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -479,7 +479,7 @@ define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_div_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -559,7 +559,7 @@ define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_div_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -615,7 +615,7 @@ declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_max_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -661,7 +661,7 @@ declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_max_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -699,7 +699,7 @@ declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_min_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -745,7 +745,7 @@ declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_min_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll index c10c922f66432..ea806a89679c5 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll @@ -23,7 +23,7 @@ define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_add_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -103,7 +103,7 @@ define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_add_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -175,7 +175,7 @@ define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_sub_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -255,7 +255,7 @@ define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_sub_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -327,7 +327,7 @@ define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_mul_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -407,7 +407,7 @@ define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_mul_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -479,7 +479,7 @@ define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) { define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_div_ss_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -559,7 +559,7 @@ define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) { define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_div_sd_round( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 8) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -615,7 +615,7 @@ declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_max_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -661,7 +661,7 @@ declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_max_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 @@ -699,7 +699,7 @@ declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) { ; ; CHECK-LABEL: @test_min_ss( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1 @@ -745,7 +745,7 @@ declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x doubl define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) { ; ; CHECK-LABEL: @test_min_sd( -; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> , i8 -1, i32 4) +; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1 diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll index cd90696eafac6..9fde3237737ec 100644 --- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll +++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll @@ -446,7 +446,7 @@ define <64 x i8> @fold_with_undef_elts_avx512(<64 x i8> %InVec) { define <16 x i8> @fold_with_allundef_elts(<16 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts( -; CHECK-NEXT: ret <16 x i8> poison +; CHECK-NEXT: ret <16 x i8> undef ; %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> undef) ret <16 x i8> %1 @@ -454,7 +454,7 @@ define <16 x i8> @fold_with_allundef_elts(<16 x i8> %InVec) { define <32 x i8> @fold_with_allundef_elts_avx2(<32 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts_avx2( -; CHECK-NEXT: ret <32 x i8> poison +; CHECK-NEXT: ret <32 x i8> undef ; %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> undef) ret <32 x i8> %1 @@ -462,7 +462,7 @@ define <32 x i8> @fold_with_allundef_elts_avx2(<32 x i8> %InVec) { define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) { ; CHECK-LABEL: @fold_with_allundef_elts_avx512( -; CHECK-NEXT: ret <64 x i8> poison +; CHECK-NEXT: ret <64 x i8> undef ; %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> undef) ret <64 x i8> %1 diff --git a/llvm/test/Transforms/InstCombine/broadcast.ll b/llvm/test/Transforms/InstCombine/broadcast.ll index ffc5508092b4c..c70b975a312ad 100644 --- a/llvm/test/Transforms/InstCombine/broadcast.ll +++ b/llvm/test/Transforms/InstCombine/broadcast.ll @@ -57,7 +57,7 @@ define <4 x float> @good4(float %arg) { define <4 x float> @good5(float %v) { ; CHECK-LABEL: @good5( -; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> , float [[V:%.*]], i64 0 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> undef, float [[V:%.*]], i64 0 ; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]] ; CHECK-NEXT: [[INS4:%.*]] = shufflevector <4 x float> [[INS1]], <4 x float> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[RES:%.*]] = fadd <4 x float> [[A1]], [[INS4]] @@ -76,7 +76,7 @@ define <4 x float> @good5(float %v) { define <4 x float> @splat_undef1(float %arg) { ; CHECK-LABEL: @splat_undef1( -; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 1 +; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] @@ -92,7 +92,7 @@ define <4 x float> @splat_undef1(float %arg) { define <4 x float> @splat_undef2(float %arg) { ; CHECK-LABEL: @splat_undef2( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 ; CHECK-NEXT: ret <4 x float> [[T6]] @@ -132,7 +132,7 @@ define <1 x float> @bad4(float %arg) { define <4 x float> @splat_undef3(float %arg) { ; CHECK-LABEL: @splat_undef3( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i64 2 ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 @@ -149,7 +149,7 @@ define <4 x float> @splat_undef3(float %arg) { define <4 x float> @bad6(float %arg, i32 %k) { ; CHECK-LABEL: @bad6( -; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> , float [[ARG:%.*]], i64 0 +; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0 ; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i64 1 ; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i32 [[K:%.*]] ; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i64 3 @@ -164,7 +164,7 @@ define <4 x float> @bad6(float %arg, i32 %k) { define <4 x float> @bad7(float %v) { ; CHECK-LABEL: @bad7( -; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> , float [[V:%.*]], i64 1 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> undef, float [[V:%.*]], i64 1 ; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]] ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[INS1]], float [[V]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[V]], i64 2 diff --git a/llvm/test/Transforms/InstCombine/extractelement.ll b/llvm/test/Transforms/InstCombine/extractelement.ll index bc5dd060a540a..3d94106d7f888 100644 --- a/llvm/test/Transforms/InstCombine/extractelement.ll +++ b/llvm/test/Transforms/InstCombine/extractelement.ll @@ -145,14 +145,14 @@ declare void @use(<8 x i8>) define i8 @bitcasted_inselt_wide_source_uses(i32 %x) { ; ANYLE-LABEL: @bitcasted_inselt_wide_source_uses( -; ANYLE-NEXT: [[I:%.*]] = insertelement <2 x i32> , i32 [[X:%.*]], i64 0 +; ANYLE-NEXT: [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i64 0 ; ANYLE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8> ; ANYLE-NEXT: call void @use(<8 x i8> [[B]]) ; ANYLE-NEXT: [[R:%.*]] = extractelement <8 x i8> [[B]], i64 3 ; ANYLE-NEXT: ret i8 [[R]] ; ; ANYBE-LABEL: @bitcasted_inselt_wide_source_uses( -; ANYBE-NEXT: [[I:%.*]] = insertelement <2 x i32> , i32 [[X:%.*]], i64 0 +; ANYBE-NEXT: [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i64 0 ; ANYBE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8> ; ANYBE-NEXT: call void @use(<8 x i8> [[B]]) ; ANYBE-NEXT: [[R:%.*]] = trunc i32 [[X]] to i8 @@ -188,7 +188,7 @@ declare void @use_v8f32(<8 x float>) define float @bitcasted_inselt_to_FP_uses(i128 %x) { ; ANY-LABEL: @bitcasted_inselt_to_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> , i128 [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2i128(<2 x i128> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float> ; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i64 1 @@ -203,7 +203,7 @@ define float @bitcasted_inselt_to_FP_uses(i128 %x) { define float @bitcasted_inselt_to_FP_uses2(i128 %x) { ; ANY-LABEL: @bitcasted_inselt_to_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> , i128 [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float> ; ANY-NEXT: call void @use_v8f32(<8 x float> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i64 1 @@ -239,7 +239,7 @@ declare void @use_v8i16(<8 x i16>) define i16 @bitcasted_inselt_from_FP_uses(double %x) { ; ANY-LABEL: @bitcasted_inselt_from_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16> ; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i64 1 @@ -254,7 +254,7 @@ define i16 @bitcasted_inselt_from_FP_uses(double %x) { define i16 @bitcasted_inselt_from_FP_uses2(double %x) { ; ANY-LABEL: @bitcasted_inselt_from_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16> ; ANY-NEXT: call void @use_v8i16(<8 x i16> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i64 1 @@ -282,7 +282,7 @@ define float @bitcasted_inselt_to_and_from_FP(double %x) { define float @bitcasted_inselt_to_and_from_FP_uses(double %x) { ; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 ; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]]) ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float> ; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i64 1 @@ -299,7 +299,7 @@ declare void @use_v4f32(<4 x float>) define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) { ; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses2( -; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> , double [[X:%.*]], i64 0 +; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i64 0 ; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float> ; ANY-NEXT: call void @use_v4f32(<4 x float> [[B]]) ; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i64 1 diff --git a/llvm/test/Transforms/InstCombine/inselt-binop.ll b/llvm/test/Transforms/InstCombine/inselt-binop.ll index dc362a0207b37..6592a59acb906 100644 --- a/llvm/test/Transforms/InstCombine/inselt-binop.ll +++ b/llvm/test/Transforms/InstCombine/inselt-binop.ll @@ -3,7 +3,7 @@ define <2 x i8> @add_constant(i8 %x) { ; CHECK-LABEL: @add_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -14,7 +14,7 @@ define <2 x i8> @add_constant(i8 %x) { define <2 x i8> @add_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @add_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -27,7 +27,7 @@ define <2 x i8> @add_constant_not_undef_lane(i8 %x) { define <2 x i8> @sub_constant_op0(i8 %x) { ; CHECK-LABEL: @sub_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sub nuw nsw <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -38,7 +38,7 @@ define <2 x i8> @sub_constant_op0(i8 %x) { define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sub_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sub nuw <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -49,7 +49,7 @@ define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @sub_constant_op1(i8 %x) { ; CHECK-LABEL: @sub_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -60,7 +60,7 @@ define <2 x i8> @sub_constant_op1(i8 %x) { define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sub_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -71,7 +71,7 @@ define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) { define <3 x i8> @mul_constant(i8 %x) { ; CHECK-LABEL: @mul_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> , i8 [[X:%.*]], i64 2 +; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 2 ; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], ; CHECK-NEXT: ret <3 x i8> [[BO]] ; @@ -82,7 +82,7 @@ define <3 x i8> @mul_constant(i8 %x) { define <3 x i8> @mul_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @mul_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> , i8 [[X:%.*]], i64 2 +; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i64 2 ; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], ; CHECK-NEXT: ret <3 x i8> [[BO]] ; @@ -93,7 +93,7 @@ define <3 x i8> @mul_constant_not_undef_lane(i8 %x) { define <2 x i8> @shl_constant_op0(i8 %x) { ; CHECK-LABEL: @shl_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -104,7 +104,7 @@ define <2 x i8> @shl_constant_op0(i8 %x) { define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @shl_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -115,7 +115,7 @@ define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @shl_constant_op1(i8 %x) { ; CHECK-LABEL: @shl_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -126,7 +126,7 @@ define <2 x i8> @shl_constant_op1(i8 %x) { define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @shl_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -137,7 +137,7 @@ define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @ashr_constant_op0(i8 %x) { ; CHECK-LABEL: @ashr_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = ashr exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -148,7 +148,7 @@ define <2 x i8> @ashr_constant_op0(i8 %x) { define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @ashr_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -159,7 +159,7 @@ define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @ashr_constant_op1(i8 %x) { ; CHECK-LABEL: @ashr_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -170,7 +170,7 @@ define <2 x i8> @ashr_constant_op1(i8 %x) { define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @ashr_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -181,7 +181,7 @@ define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @lshr_constant_op0(i8 %x) { ; CHECK-LABEL: @lshr_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -192,7 +192,7 @@ define <2 x i8> @lshr_constant_op0(i8 %x) { define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @lshr_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -203,7 +203,7 @@ define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) { define <2 x i8> @lshr_constant_op1(i8 %x) { ; CHECK-LABEL: @lshr_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -214,7 +214,7 @@ define <2 x i8> @lshr_constant_op1(i8 %x) { define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @lshr_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -225,7 +225,7 @@ define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @urem_constant_op0(i8 %x) { ; CHECK-LABEL: @urem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -236,7 +236,7 @@ define <2 x i8> @urem_constant_op0(i8 %x) { define <2 x i8> @urem_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @urem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -256,7 +256,7 @@ define <2 x i8> @urem_constant_op1(i8 %x) { define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @urem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -267,7 +267,7 @@ define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @srem_constant_op0(i8 %x) { ; CHECK-LABEL: @srem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -278,7 +278,7 @@ define <2 x i8> @srem_constant_op0(i8 %x) { define <2 x i8> @srem_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @srem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -298,7 +298,7 @@ define <2 x i8> @srem_constant_op1(i8 %x) { define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @srem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -309,7 +309,7 @@ define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @udiv_constant_op0(i8 %x) { ; CHECK-LABEL: @udiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -320,7 +320,7 @@ define <2 x i8> @udiv_constant_op0(i8 %x) { define <2 x i8> @udiv_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @udiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -340,7 +340,7 @@ define <2 x i8> @udiv_constant_op1(i8 %x) { define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @udiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = udiv <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -351,7 +351,7 @@ define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @sdiv_constant_op0(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -362,7 +362,7 @@ define <2 x i8> @sdiv_constant_op0(i8 %x) { define <2 x i8> @sdiv_constant_op0_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> , [[INS]] ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -382,7 +382,7 @@ define <2 x i8> @sdiv_constant_op1(i8 %x) { define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) { ; CHECK-LABEL: @sdiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = sdiv exact <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -393,7 +393,7 @@ define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) { define <2 x i8> @and_constant(i8 %x) { ; CHECK-LABEL: @and_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -404,7 +404,7 @@ define <2 x i8> @and_constant(i8 %x) { define <2 x i8> @and_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @and_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -415,7 +415,7 @@ define <2 x i8> @and_constant_not_undef_lane(i8 %x) { define <2 x i8> @or_constant(i8 %x) { ; CHECK-LABEL: @or_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -426,7 +426,7 @@ define <2 x i8> @or_constant(i8 %x) { define <2 x i8> @or_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @or_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -437,7 +437,7 @@ define <2 x i8> @or_constant_not_undef_lane(i8 %x) { define <2 x i8> @xor_constant(i8 %x) { ; CHECK-LABEL: @xor_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -448,7 +448,7 @@ define <2 x i8> @xor_constant(i8 %x) { define <2 x i8> @xor_constant_not_undef_lane(i8 %x) { ; CHECK-LABEL: @xor_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> , i8 [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], ; CHECK-NEXT: ret <2 x i8> [[BO]] ; @@ -459,7 +459,7 @@ define <2 x i8> @xor_constant_not_undef_lane(i8 %x) { define <2 x float> @fadd_constant(float %x) { ; CHECK-LABEL: @fadd_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -470,7 +470,7 @@ define <2 x float> @fadd_constant(float %x) { define <2 x float> @fadd_constant_not_undef_lane(float %x) { ; CHECK-LABEL: @fadd_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -481,7 +481,7 @@ define <2 x float> @fadd_constant_not_undef_lane(float %x) { define <2 x float> @fsub_constant_op0(float %x) { ; CHECK-LABEL: @fsub_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fsub fast <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -492,7 +492,7 @@ define <2 x float> @fsub_constant_op0(float %x) { define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @fsub_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fsub nsz <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -503,7 +503,7 @@ define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) { define <2 x float> @fsub_constant_op1(float %x) { ; CHECK-LABEL: @fsub_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -514,7 +514,7 @@ define <2 x float> @fsub_constant_op1(float %x) { define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @fsub_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -525,7 +525,7 @@ define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) { define <2 x float> @fmul_constant(float %x) { ; CHECK-LABEL: @fmul_constant( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fmul reassoc <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -536,7 +536,7 @@ define <2 x float> @fmul_constant(float %x) { define <2 x float> @fmul_constant_not_undef_lane(float %x) { ; CHECK-LABEL: @fmul_constant_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fmul <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -547,7 +547,7 @@ define <2 x float> @fmul_constant_not_undef_lane(float %x) { define <2 x float> @fdiv_constant_op0(float %x) { ; CHECK-LABEL: @fdiv_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = fdiv nnan <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -558,7 +558,7 @@ define <2 x float> @fdiv_constant_op0(float %x) { define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @fdiv_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv ninf <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -569,7 +569,7 @@ define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) { define <2 x float> @fdiv_constant_op1(float %x) { ; CHECK-LABEL: @fdiv_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -580,7 +580,7 @@ define <2 x float> @fdiv_constant_op1(float %x) { define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @fdiv_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -591,7 +591,7 @@ define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) { define <2 x float> @frem_constant_op0(float %x) { ; CHECK-LABEL: @frem_constant_op0( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = frem fast <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -602,7 +602,7 @@ define <2 x float> @frem_constant_op0(float %x) { define <2 x float> @frem_constant_op0_not_undef_lane(float %x) { ; CHECK-LABEL: @frem_constant_op0_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = frem <2 x float> , [[INS]] ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -613,7 +613,7 @@ define <2 x float> @frem_constant_op0_not_undef_lane(float %x) { define <2 x float> @frem_constant_op1(float %x) { ; CHECK-LABEL: @frem_constant_op1( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 1 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 1 ; CHECK-NEXT: [[BO:%.*]] = frem ninf <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; @@ -624,7 +624,7 @@ define <2 x float> @frem_constant_op1(float %x) { define <2 x float> @frem_constant_op1_not_undef_lane(float %x) { ; CHECK-LABEL: @frem_constant_op1_not_undef_lane( -; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: [[BO:%.*]] = frem nnan <2 x float> [[INS]], ; CHECK-NEXT: ret <2 x float> [[BO]] ; diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll index c87e2e8596c62..5034f44b6a6f9 100644 --- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll +++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll @@ -290,7 +290,7 @@ define <4 x float> @collectShuffleElts(<2 x float> %x, float %y) { ; CHECK-LABEL: @collectShuffleElts( ; CHECK-NEXT: [[X0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0 ; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x float> [[X]], i64 1 -; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> , float [[X0]], i64 1 +; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> undef, float [[X0]], i64 1 ; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[X1]], i64 2 ; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Y:%.*]], i64 3 ; CHECK-NEXT: ret <4 x float> [[V3]] @@ -462,7 +462,7 @@ define <5 x i7> @insert_nonzero_index_splat_widen(i7 %x) { define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) { ; CHECK-LABEL: @insert_nonzero_index_splat_extra_use( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i64 2 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 2 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] @@ -490,7 +490,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) { ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i32 [[INDEX:%.*]] +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 [[INDEX:%.*]] ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SPLAT]] ; @@ -513,7 +513,7 @@ define <4 x float> @insert_in_splat(float %x) { define <4 x float> @insert_in_splat_extra_uses(float %x) { ; CHECK-LABEL: @insert_in_splat_extra_uses( -; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> , float [[X:%.*]], i64 0 +; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0 ; CHECK-NEXT: call void @use(<4 x float> [[XV]]) ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: call void @use(<4 x float> [[SPLAT]]) diff --git a/llvm/test/Transforms/InstCombine/pr38984.ll b/llvm/test/Transforms/InstCombine/pr38984.ll index c148765fce59f..af1d05a4654f7 100644 --- a/llvm/test/Transforms/InstCombine/pr38984.ll +++ b/llvm/test/Transforms/InstCombine/pr38984.ll @@ -25,7 +25,7 @@ define <4 x i1> @PR38984_2() { ; CHECK-LABEL: @PR38984_2( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @offsets, align 2 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> , i16 [[TMP0]], i64 3 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> undef, i16 [[TMP0]], i64 3 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr getelementptr inbounds ([21 x i16], ptr @a, i16 1, i16 0), <4 x i16> [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr null, <4 x i16> [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x ptr> [[TMP2]], [[TMP3]] diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll index a1b0d782b554f..12bf09f8aeb76 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll @@ -1521,7 +1521,7 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) { define <4 x i32> @PR41419(<4 x i32> %v) { ; CHECK-LABEL: @PR41419( -; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> , <4 x i32> +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: ret <4 x i32> [[S]] ; %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll index 1699418dcc28b..457300a25e769 100644 --- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll +++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll @@ -26,7 +26,7 @@ define i16 @test_srem_orig(i16 %a, i1 %cmp) { ; "evaluateInDifferentElementOrder". define <2 x i16> @test_srem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_srem( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -41,7 +41,7 @@ define <2 x i16> @test_srem(i16 %a, i1 %cmp) { define <2 x i16> @test_urem(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_urem( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -56,7 +56,7 @@ define <2 x i16> @test_urem(i16 %a, i1 %cmp) { define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_sdiv( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] @@ -71,7 +71,7 @@ define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) { define <2 x i16> @test_udiv(i16 %a, i1 %cmp) { ; CHECK-LABEL: @test_udiv( -; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> , i16 [[A:%.*]], i64 0 +; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0 ; CHECK-NEXT: [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], ; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> ; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> , <2 x i16> [[SPLAT_OP]] diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll index f2a28c0dd02b3..790d551e5b1de 100644 --- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll +++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll @@ -843,8 +843,8 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) { ; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef( ; CHECK-NEXT: [[T0_NEG:%.*]] = shl <2 x i4> , [[X:%.*]] -; CHECK-NEXT: [[T11_NEG:%.*]] = insertelement <2 x i4> [[T0_NEG]], i4 undef, i64 1 -; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T11_NEG]], [[Y:%.*]] +; CHECK-NEXT: [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> +; CHECK-NEXT: [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x i4> [[T2]] ; %t0 = shl <2 x i4> , %x diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll index c84d39c5aa9dd..576af85b5ee27 100644 --- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll +++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll @@ -148,7 +148,7 @@ define <2 x i64> @PR24922(<2 x i64> %v) { define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) { ; CHECK-LABEL: @inselt_shuf_no_demand( -; CHECK-NEXT: ret <4 x float> +; CHECK-NEXT: ret <4 x float> undef ; %out1 = insertelement <4 x float> undef, float %a1, i32 1 %out12 = insertelement <4 x float> %out1, float %a2, i32 2 @@ -161,7 +161,7 @@ define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) { define <4 x float> @inselt_shuf_no_demand_commute(float %a1, float %a2, float %a3) { ; CHECK-LABEL: @inselt_shuf_no_demand_commute( -; CHECK-NEXT: ret <4 x float> +; CHECK-NEXT: ret <4 x float> undef ; %out1 = insertelement <4 x float> undef, float %a1, i32 1 %out12 = insertelement <4 x float> %out1, float %a2, i32 2 @@ -192,7 +192,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) { ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain( -; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> , float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]] +; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]] ; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[SHUFFLE]] ; @@ -617,8 +617,8 @@ define ptr @gep_splat_both(ptr %base, i64 %idx) { define <2 x ptr> @gep_all_lanes_undef(ptr %base, i64 %idx) {; ; CHECK-LABEL: @gep_all_lanes_undef( -; CHECK-NEXT: [[BASEVEC:%.*]] = insertelement <2 x ptr> , ptr [[BASE:%.*]], i64 0 -; CHECK-NEXT: [[IDXVEC:%.*]] = insertelement <2 x i64> , i64 [[IDX:%.*]], i64 1 +; CHECK-NEXT: [[BASEVEC:%.*]] = insertelement <2 x ptr> undef, ptr [[BASE:%.*]], i64 0 +; CHECK-NEXT: [[IDXVEC:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i64 1 ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC]], <2 x i64> [[IDXVEC]] ; CHECK-NEXT: ret <2 x ptr> [[GEP]] ; @@ -780,7 +780,7 @@ define <4 x float> @ins_of_ext_twice(<4 x float> %x, float %y) { define <4 x float> @ins_of_ext_wrong_demand(<4 x float> %x, float %y) { ; CHECK-LABEL: @ins_of_ext_wrong_demand( ; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i64 0 -; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> , float [[E0]], i64 0 +; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> undef, float [[E0]], i64 0 ; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[Y:%.*]], i64 1 ; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i64 2 ; CHECK-NEXT: ret <4 x float> [[I2]] diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll index 4e4fa7defa8cf..69149720c9335 100644 --- a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll +++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll @@ -4,7 +4,7 @@ define <4 x ptr> @PR41270(ptr %x) { ; CHECK-LABEL: @PR41270( ; CHECK-NEXT: [[T3:%.*]] = getelementptr inbounds [4 x i16], ptr [[X:%.*]], i64 0, i64 3 -; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x ptr> , ptr [[T3]], i64 0 +; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x ptr> undef, ptr [[T3]], i64 0 ; CHECK-NEXT: ret <4 x ptr> [[INS2]] ; %ins = insertelement <4 x ptr> undef, ptr %x, i32 0 diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll index 250a175ad0ebe..8eff837d6e1a3 100644 --- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll @@ -81,7 +81,7 @@ define float @testvscale6( %X) { define <4 x float> @test7(<4 x float> %x) { ; CHECK-LABEL: @test7( -; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> , <4 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: ret <4 x float> [[R]] ; %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 > diff --git a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll index cf1b72fbcf3e1..67ef387637537 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll @@ -294,7 +294,7 @@ define <8 x i32> @pr24458(<8 x float> %n) { define <3 x i16> @trunc_inselt_undef(i32 %x) { ; CHECK-LABEL: @trunc_inselt_undef( ; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16 -; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> , i16 [[TMP1]], i64 1 +; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i64 1 ; CHECK-NEXT: ret <3 x i16> [[TRUNC]] ; %vec = insertelement <3 x i32> poison, i32 %x, i32 1 diff --git a/llvm/test/Transforms/InstCombine/vector-casts.ll b/llvm/test/Transforms/InstCombine/vector-casts.ll index 281fc5f6011ea..21e0818fa001e 100644 --- a/llvm/test/Transforms/InstCombine/vector-casts.ll +++ b/llvm/test/Transforms/InstCombine/vector-casts.ll @@ -294,7 +294,7 @@ define <8 x i32> @pr24458(<8 x float> %n) { define <3 x i16> @trunc_inselt_undef(i32 %x) { ; CHECK-LABEL: @trunc_inselt_undef( ; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16 -; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> , i16 [[TMP1]], i64 1 +; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i64 1 ; CHECK-NEXT: ret <3 x i16> [[TRUNC]] ; %vec = insertelement <3 x i32> undef, i32 %x, i32 1 From 245cddae705aa8c8c8c88f80afb128d46b911695 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 18 Dec 2023 09:10:39 -0800 Subject: [PATCH 147/884] [PGO][GlobalValue][LTO]In GlobalValues::getGlobalIdentifier, use semicolon as delimiter for local-linkage varibles. (#74008) Commit fe05193 (phab D156569), IRPGO names uses format `[;]` while prior format is `[:`. The format change would break the use case demonstrated in (updated) `llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll` and `compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp` This patch changes `GlobalValues::getGlobalIdentifer` to use the semicolon. To elaborate on the scenario how things break without this PR 1. IRPGO raw profiles stores (compressed) IRPGO names of functions in one section, and per-function profile data in another section. The [NameRef](https://github.com/llvm/llvm-project/blob/fc715e4cd942612a091097339841733757b53824/compiler-rt/include/profile/InstrProfData.inc#L72) field in per-function profile data is the MD5 hash of IRPGO names. 2. When raw profiles are converted to indexed format profiles, the profiled address is [mapped](https://github.com/llvm/llvm-project/blob/fc715e4cd942612a091097339841733757b53824/llvm/lib/ProfileData/InstrProf.cpp#L876-L885) to the MD5 hash of the callee. 3. In `pgo-instr-use` thin-lto prelink pipeline, MD5 hash of IRPGO names will be [annotated](https://github.com/llvm/llvm-project/blob/fc715e4cd942612a091097339841733757b53824/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp#L1707) as value profiles, and used to import indirect-call-prom candidates. If the annotated MD5 hash is computed from the new format while import uses the prior format, the callee cannot be imported. *`compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp` is added to have an end-to-end test. * `llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll` is updated to have better test coverage from another aspect (as runtime tests are more sensitive to the environment and may be skipped by some contributors) --- ...trprof-thinlto-indirect-call-promotion.cpp | 115 ++++++++++++++++++ llvm/include/llvm/IR/GlobalValue.h | 4 + llvm/include/llvm/ProfileData/InstrProf.h | 26 ++-- llvm/lib/IR/Globals.cpp | 12 +- llvm/lib/ProfileData/InstrProf.cpp | 38 ++++-- llvm/lib/ProfileData/InstrProfReader.cpp | 11 +- .../thinlto-function-summary-originalnames.ll | 10 +- llvm/test/ThinLTO/X86/memprof-basic.ll | 26 ++-- .../X86/memprof-duplicate-context-ids.ll | 10 +- .../ThinLTO/X86/memprof-funcassigncloning.ll | 6 +- llvm/test/ThinLTO/X86/memprof-indirectcall.ll | 32 ++--- llvm/test/ThinLTO/X86/memprof-inlined.ll | 14 +-- .../Inputs/thinlto_indirect_call_promotion.ll | 16 --- .../thinlto_indirect_call_promotion.profraw | Bin 0 -> 528 bytes ..._thinlto_indirect_call_promotion_inputs.sh | 62 ++++++++++ .../thinlto_indirect_call_promotion.ll | 105 +++++++++++----- llvm/unittests/ProfileData/InstrProfTest.cpp | 4 +- 17 files changed, 363 insertions(+), 128 deletions(-) create mode 100644 compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp delete mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw create mode 100755 llvm/test/Transforms/PGOProfile/Inputs/update_thinlto_indirect_call_promotion_inputs.sh diff --git a/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp b/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp new file mode 100644 index 0000000000000..82ca1cd7d0a56 --- /dev/null +++ b/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp @@ -0,0 +1,115 @@ +// This is a regression test for ThinLTO indirect-call-promotion when candidate +// callees need to be imported from another IR module. In the C++ test case, +// `main` calls `global_func` which is defined in another module. `global_func` +// has two indirect callees, one has external linkage and one has local linkage. +// All three functions should be imported into the IR module of main. + +// What the test does: +// - Generate raw profiles from executables and convert it to indexed profiles. +// During the conversion, a profiled callee address in raw profiles will be +// converted to function hash in indexed profiles. +// - Run IRPGO profile use and ThinTLO prelink pipeline and get LLVM bitcodes +// for both cpp files in the C++ test case. +// - Generate ThinLTO summary file with LLVM bitcodes, and run `function-import` pass. +// - Run `pgo-icall-prom` pass for the IR module which needs to import callees. + +// Use lld as linker for more robust test. We need to REQUIRE LLVMgold.so for +// LTO if default linker is GNU ld or gold anyway. +// REQUIRES: lld-available + +// Test should fail where linkage-name and mangled-name diverges, see issue https://github.com/llvm/llvm-project/issues/74565). +// Currently, this name divergence happens on Mach-O object file format, or on +// many (but not all) 32-bit Windows systems. +// +// XFAIL: system-darwin +// +// Mark 32-bit Windows as UNSUPPORTED for now as opposed to XFAIL. This test +// should fail on many (but not all) 32-bit Windows systems and succeed on the +// rest. The flexibility in triple string parsing makes it tricky to capture +// both sets accurately. i[3-9]86 specifies arch as Triple::ArchType::x86, (win32|windows) +// specifies OS as Triple::OS::Win32 +// +// UNSUPPORTED: target={{i.86.*windows.*}} + +// RUN: rm -rf %t && split-file %s %t && cd %t + +// Do setup work for all below tests. +// Generate raw profiles from real programs and convert it into indexed profiles. +// Use clangxx_pgogen for IR level instrumentation for C++. +// RUN: %clangxx_pgogen -fuse-ld=lld -O2 lib.cpp main.cpp -o main +// RUN: env LLVM_PROFILE_FILE=main.profraw %run ./main +// RUN: llvm-profdata merge main.profraw -o main.profdata + +// Use profile on lib and get bitcode, test that local function callee0 has +// expected !PGOFuncName metadata and external function callee1 doesn't have +// !PGOFuncName metadata. Explicitly skip ICP pass to test ICP happens as +// expected in the IR module that imports functions from lib. +// RUN: %clang -mllvm -disable-icp -fprofile-use=main.profdata -flto=thin -O2 -c lib.cpp -o lib.bc +// RUN: llvm-dis lib.bc -o - | FileCheck %s --check-prefix=PGOName + +// Use profile on main and get bitcode. +// RUN: %clang -fprofile-use=main.profdata -flto=thin -O2 -c main.cpp -o main.bc + +// Run llvm-lto to get summary file. +// RUN: llvm-lto -thinlto -o summary main.bc lib.bc + +// Test the imports of functions. Default import thresholds would work but do +// explicit override to be more futureproof. Note all functions have one basic +// block with a function-entry-count of one, so they are actually hot functions +// per default profile summary hotness cutoff. +// RUN: opt -passes=function-import -import-instr-limit=100 -import-cold-multiplier=1 -summary-file summary.thinlto.bc main.bc -o main.import.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS +// Test that '_Z11global_funcv' has indirect calls annotated with value profiles. +// RUN: llvm-dis main.import.bc -o - | FileCheck %s --check-prefix=IR + +// Test that both candidates are ICP'ed and there is no `!VP` in the IR. +// RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefixes=ICP-IR,ICP-REMARK --implicit-check-not="!VP" + +// IMPORTS: main.cpp: Import _Z7callee1v +// IMPORTS: main.cpp: Import _ZL7callee0v.llvm.[[#]] +// IMPORTS: main.cpp: Import _Z11global_funcv + +// PGOName: define {{(dso_local )?}}void @_Z7callee1v() #[[#]] !prof ![[#]] { +// PGOName: define internal void @_ZL7callee0v() #[[#]] !prof ![[#]] !PGOFuncName ![[#MD:]] { +// PGOName: ![[#MD]] = !{!"{{.*}}lib.cpp;_ZL7callee0v"} + +// IR-LABEL: define available_externally {{.*}} void @_Z11global_funcv() {{.*}} !prof ![[#]] { +// IR-NEXT: entry: +// IR-NEXT: %0 = load ptr, ptr @calleeAddrs +// IR-NEXT: tail call void %0(), !prof ![[#PROF1:]] +// IR-NEXT: %1 = load ptr, ptr getelementptr inbounds ([2 x ptr], ptr @calleeAddrs, +// IR-NEXT: tail call void %1(), !prof ![[#PROF2:]] + +// The GUID of indirect callee is the MD5 hash of `/path/to/lib.cpp;_ZL7callee0v` +// that depends on the directory. Use [[#]] for its MD5 hash. +// Use {{.*}} for integer types so the test works on 32-bit and 64-bit systems. +// IR: ![[#PROF1]] = !{!"VP", i32 0, {{.*}} 1, {{.*}} [[#]], {{.*}} 1} +// IR: ![[#PROF2]] = !{!"VP", i32 0, {{.*}} 1, {{.*}} -3993653843325621743, {{.*}} 1} + +// ICP-REMARK: Promote indirect call to _ZL7callee0v.llvm.[[#]] with count 1 out of 1 +// ICP-REMARK: Promote indirect call to _Z7callee1v with count 1 out of 1 + +// ICP-IR: br i1 %[[#]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof ![[#BRANCH_WEIGHT1:]] +// ICP-IR: br i1 %[[#]], label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof ![[#BRANCH_WEIGHT1]] +// ICP-IR: ![[#BRANCH_WEIGHT1]] = !{!"branch_weights", i32 1, i32 0} + +//--- lib.h +void global_func(); + +//--- lib.cpp +#include "lib.h" +static void callee0() {} +void callee1() {} +typedef void (*FPT)(); +FPT calleeAddrs[] = {callee0, callee1}; +// `global_func`` might call one of two indirect callees. callee0 has internal +// linkage and callee1 has external linkage. +void global_func() { + FPT fp = calleeAddrs[0]; + fp(); + fp = calleeAddrs[1]; + fp(); +} + +//--- main.cpp +#include "lib.h" +int main() { global_func(); } diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index d1891c157099d..e97a7f2b96360 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -41,6 +41,10 @@ namespace Intrinsic { typedef unsigned ID; } // end namespace Intrinsic +// Choose ';' as the delimiter. ':' was used once but it doesn't work well for +// Objective-C functions which commonly have :'s in their names. +inline constexpr char kGlobalIdentifierDelimiter = ';'; + class GlobalValue : public Constant { public: /// An enumeration for the kinds of linkage for global values. diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 288dc71d756ae..36be2e7d869e7 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -171,6 +171,8 @@ inline StringRef getInstrProfCounterBiasVarName() { /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } +/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is +/// for front-end (Clang, etc) instrumentation. /// Return the modified name for function \c F suitable to be /// used the key for profile lookup. Variable \c InLTO indicates if this /// is called in LTO optimization passes. @@ -196,20 +198,22 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO = false); std::pair getParsedIRPGOFuncName(StringRef IRPGOFuncName); /// Return the name of the global variable used to store a function -/// name in PGO instrumentation. \c FuncName is the name of the function -/// returned by the \c getPGOFuncName call. +/// name in PGO instrumentation. \c FuncName is the IRPGO function name +/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO +/// function name (returned by \c getPGOFuncName) for front-end instrumentation. std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage); /// Create and return the global variable for function name used in PGO -/// instrumentation. \c FuncName is the name of the function returned -/// by \c getPGOFuncName call. +/// instrumentation. \c FuncName is the IRPGO function name (returned by +/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name +/// (returned by \c getPGOFuncName) for front-end instrumentation. GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); /// Create and return the global variable for function name used in PGO -/// instrumentation. /// \c FuncName is the name of the function -/// returned by \c getPGOFuncName call, \c M is the owning module, -/// and \c Linkage is the linkage of the instrumented function. +/// instrumentation. \c FuncName is the IRPGO function name (returned by +/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name +/// (returned by \c getPGOFuncName) for front-end instrumentation. GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, StringRef PGOFuncName); @@ -417,11 +421,11 @@ uint64_t ComputeHash(StringRef K); } // end namespace IndexedInstrProf -/// A symbol table used for function PGO name look-up with keys +/// A symbol table used for function [IR]PGO name look-up with keys /// (such as pointers, md5hash values) to the function. A function's -/// PGO name or name's md5hash are used in retrieving the profile -/// data of the function. See \c getPGOFuncName() method for details -/// on how PGO name is formed. +/// [IR]PGO name or name's md5hash are used in retrieving the profile +/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName +/// methods for details how [IR]PGO name is formed. class InstrProfSymtab { public: using AddrHashMap = std::vector>; diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index 51bdbeb0abf2c..239acd2181e85 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -144,25 +144,27 @@ void GlobalObject::copyAttributesFrom(const GlobalObject *Src) { std::string GlobalValue::getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName) { - // Value names may be prefixed with a binary '1' to indicate // that the backend should not modify the symbols due to any platform // naming convention. Do not include that '1' in the PGO profile name. if (Name[0] == '\1') Name = Name.substr(1); - std::string NewName = std::string(Name); + std::string GlobalName; if (llvm::GlobalValue::isLocalLinkage(Linkage)) { // For local symbols, prepend the main file name to distinguish them. // Do not include the full path in the file name since there's no guarantee // that it will stay the same, e.g., if the files are checked out from // version control in different locations. if (FileName.empty()) - NewName = NewName.insert(0, ":"); + GlobalName += ""; else - NewName = NewName.insert(0, FileName.str() + ":"); + GlobalName += FileName; + + GlobalName += kGlobalIdentifierDelimiter; } - return NewName; + GlobalName += Name; + return GlobalName; } std::string GlobalValue::getGlobalIdentifier() const { diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 649d814cfd9de..f5fde84eb6bd0 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -246,11 +246,27 @@ std::string InstrProfError::message() const { char InstrProfError::ID = 0; -std::string getPGOFuncName(StringRef RawFuncName, - GlobalValue::LinkageTypes Linkage, +std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName, uint64_t Version LLVM_ATTRIBUTE_UNUSED) { - return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); + // Value names may be prefixed with a binary '1' to indicate + // that the backend should not modify the symbols due to any platform + // naming convention. Do not include that '1' in the PGO profile name. + if (Name[0] == '\1') + Name = Name.substr(1); + + std::string NewName = std::string(Name); + if (llvm::GlobalValue::isLocalLinkage(Linkage)) { + // For local symbols, prepend the main file name to distinguish them. + // Do not include the full path in the file name since there's no guarantee + // that it will stay the same, e.g., if the files are checked out from + // version control in different locations. + if (FileName.empty()) + NewName = NewName.insert(0, ":"); + else + NewName = NewName.insert(0, FileName.str() + ":"); + } + return NewName; } // Strip NumPrefix level of directory name from PathNameStr. If the number of @@ -300,12 +316,10 @@ getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { SmallString<64> Name; - if (llvm::GlobalValue::isLocalLinkage(Linkage)) { - Name.append(FileName.empty() ? "" : FileName); - Name.append(";"); - } + // FIXME: Mangler's handling is kept outside of `getGlobalIdentifier` for now. + // For more details please check issue #74565. Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); - return Name.str().str(); + return GlobalValue::getGlobalIdentifier(Name, Linkage, FileName); } static std::optional lookupPGONameFromMetadata(MDNode *MD) { @@ -352,6 +366,9 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO) { return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F)); } +// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is +// for front-end (Clang, etc) instrumentation. +// The implementation is kept for profile matching from older profiles. // This is similar to `getIRPGOFuncName` except that this function calls // 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls // 'getIRPGONameForGlobalObject'. See the difference between two callees in the @@ -384,8 +401,9 @@ getParsedIRPGOFuncName(StringRef IRPGOFuncName) { StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { if (FileName.empty()) return PGOFuncName; - // Drop the file name including ':'. See also getPGOFuncName. - if (PGOFuncName.starts_with(FileName)) + // Drop the file name including ':' or ';'. See getIRPGONameForGlobalObject as + // well. + if (PGOFuncName.startswith(FileName)) PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); return PGOFuncName; } diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 068922d421f8b..7f8eb0ce747fe 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1008,13 +1008,14 @@ class llvm::InstrProfReaderItaniumRemapper /// Extract the original function name from a PGO function name. static StringRef extractName(StringRef Name) { - // We can have multiple :-separated pieces; there can be pieces both - // before and after the mangled name. Find the first part that starts - // with '_Z'; we'll assume that's the mangled name we want. + // We can have multiple pieces separated by kGlobalIdentifierDelimiter ( + // semicolon now and colon in older profiles); there can be pieces both + // before and after the mangled name. Find the first part that starts with + // '_Z'; we'll assume that's the mangled name we want. std::pair Parts = {StringRef(), Name}; while (true) { - Parts = Parts.second.split(':'); - if (Parts.first.starts_with("_Z")) + Parts = Parts.second.split(kGlobalIdentifierDelimiter); + if (Parts.first.startswith("_Z")) return Parts.first; if (Parts.second.empty()) return Name; diff --git a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll index 7cc9654c8c7b1..0139f00b4aa3f 100644 --- a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll @@ -6,13 +6,13 @@ ; COMBINED: -; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-DAG: -; COMBINED-DAG: +; COMBINED-DAG: ; COMBINED-DAG: ; COMBINED-NEXT: diff --git a/llvm/test/ThinLTO/X86/memprof-basic.ll b/llvm/test/ThinLTO/X86/memprof-basic.ll index 0d466830ba57d..54e01e5fcdf95 100644 --- a/llvm/test/ThinLTO/X86/memprof-basic.ll +++ b/llvm/test/ThinLTO/X86/memprof-basic.ll @@ -148,7 +148,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 ; DUMP: Node [[BAZ]] -; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -157,7 +157,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 ; DUMP: Node [[FOO]] -; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -167,7 +167,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -175,7 +175,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -197,7 +197,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAR2:0x[a-z0-9]+]] ; DUMP: Node [[BAZ]] -; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -207,7 +207,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAZ2:0x[a-z0-9]+]] ; DUMP: Node [[FOO]] -; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -217,7 +217,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO2:0x[a-z0-9]+]] ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -225,7 +225,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -233,7 +233,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[FOO2]] -; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -243,7 +243,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clone of [[FOO]] ; DUMP: Node [[BAZ2]] -; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -344,7 +344,7 @@ attributes #0 = { noinline optnone } ; DOTCLONED: } -; DISTRIB: ^[[BAZ:[0-9]+]] = gv: (guid: 5878270615442837395, {{.*}} callsites: ((callee: ^[[BAR:[0-9]+]], clones: (0, 1) -; DISTRIB: ^[[FOO:[0-9]+]] = gv: (guid: 6731117468105397038, {{.*}} callsites: ((callee: ^[[BAZ]], clones: (0, 1) -; DISTRIB: ^[[BAR]] = gv: (guid: 9832687305761716512, {{.*}} allocs: ((versions: (notcold, cold) +; DISTRIB: ^[[BAZ:[0-9]+]] = gv: (guid: 1807954217441101578, {{.*}} callsites: ((callee: ^[[BAR:[0-9]+]], clones: (0, 1) +; DISTRIB: ^[[FOO:[0-9]+]] = gv: (guid: 8107868197919466657, {{.*}} callsites: ((callee: ^[[BAZ]], clones: (0, 1) +; DISTRIB: ^[[BAR]] = gv: (guid: 11481133863268513686, {{.*}} allocs: ((versions: (notcold, cold) ; DISTRIB: ^[[MAIN:[0-9]+]] = gv: (guid: 15822663052811949562, {{.*}} callsites: ((callee: ^[[FOO]], clones: (0), {{.*}} (callee: ^[[FOO]], clones: (1) diff --git a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll index f7ba0d27dca78..65d794e9cba87 100644 --- a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll +++ b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll @@ -260,8 +260,8 @@ attributes #0 = { noinline optnone} ; STATS-BE: 1 memprof-context-disambiguation - Number of original (not cloned) allocations with memprof profiles during ThinLTO backend -; DISTRIB: ^[[C:[0-9]+]] = gv: (guid: 1643923691937891493, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) -; DISTRIB: ^[[D]] = gv: (guid: 4881081444663423788, {{.*}} allocs: ((versions: (notcold, cold) -; DISTRIB: ^[[B:[0-9]+]] = gv: (guid: 14590037969532473829, {{.*}} callsites: ((callee: ^[[D]], clones: (1) -; DISTRIB: ^[[F:[0-9]+]] = gv: (guid: 17035303613541779335, {{.*}} callsites: ((callee: ^[[D]], clones: (0) -; DISTRIB: ^[[E:[0-9]+]] = gv: (guid: 17820708772846654376, {{.*}} callsites: ((callee: ^[[D]], clones: (1) +; DISTRIB: ^[[E:[0-9]+]] = gv: (guid: 331966645857188136, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) +; DISTRIB: ^[[D]] = gv: (guid: 11079124245221721799, {{.*}} allocs: ((versions: (notcold, cold) +; DISTRIB: ^[[F:[0-9]+]] = gv: (guid: 11254287701717398916, {{.*}} callsites: ((callee: ^[[D]], clones: (0) +; DISTRIB: ^[[B:[0-9]+]] = gv: (guid: 13579056193435805313, {{.*}} callsites: ((callee: ^[[D]], clones: (1) +; DISTRIB: ^[[C:[0-9]+]] = gv: (guid: 15101436305866936160, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) diff --git a/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll b/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll index 9a72ae43b2f1e..f1a494d077fef 100644 --- a/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll +++ b/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll @@ -176,7 +176,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[ENEW1CLONE:0x[a-z0-9]+]] ; DUMP: Node [[D:0x[a-z0-9]+]] -; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 6 ; DUMP: CalleeEdges: @@ -185,7 +185,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[C]] -; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 2 5 ; DUMP: CalleeEdges: @@ -194,7 +194,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[B]] -; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 4 ; DUMP: CalleeEdges: diff --git a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll index 76273959f4f4a..07a52f441ca27 100644 --- a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll +++ b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll @@ -202,7 +202,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 6 ; DUMP: Node [[AX]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -225,7 +225,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 5 ; DUMP: Node [[MAIN3]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -233,7 +233,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN4]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -241,7 +241,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -249,7 +249,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[BX]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 4 5 ; DUMP: CalleeEdges: @@ -258,7 +258,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 ; DUMP: Node [[MAIN5]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: @@ -266,7 +266,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN6]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 5 ; DUMP: CalleeEdges: @@ -274,7 +274,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 6 ; DUMP: CalleeEdges: @@ -302,7 +302,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO2:0x[a-z0-9]+]] ; DUMP: Node [[AX]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -324,7 +324,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6]] AllocTypes: NotCold ContextIds: 5 ; DUMP: Node [[MAIN3]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -332,7 +332,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN4]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -340,7 +340,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -348,7 +348,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[BX]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 4 5 ; DUMP: CalleeEdges: @@ -357,7 +357,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 ; DUMP: Node [[MAIN5]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: @@ -365,7 +365,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN6]] -; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 5 ; DUMP: CalleeEdges: @@ -373,7 +373,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 6 ; DUMP: CalleeEdges: diff --git a/llvm/test/ThinLTO/X86/memprof-inlined.ll b/llvm/test/ThinLTO/X86/memprof-inlined.ll index feb9c94344223..89df345b22042 100644 --- a/llvm/test/ThinLTO/X86/memprof-inlined.ll +++ b/llvm/test/ThinLTO/X86/memprof-inlined.ll @@ -170,7 +170,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 3 ; DUMP: CalleeEdges: @@ -179,7 +179,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 4 ; DUMP: CalleeEdges: @@ -201,7 +201,7 @@ attributes #0 = { noinline optnone } ;; This is the node synthesized for the call to bar in foo that was created ;; by inlining baz into foo. ; DUMP: Node [[FOO]] -; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 3 4 ; DUMP: CalleeEdges: @@ -234,7 +234,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 3 ; DUMP: CalleeEdges: @@ -243,7 +243,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 4 ; DUMP: CalleeEdges: @@ -264,7 +264,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAR2:0x[a-z0-9]+]] ; DUMP: Node [[FOO]] -; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -274,7 +274,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO3]] ; DUMP: Node [[FOO3]] -; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll deleted file mode 100644 index 7412120bb52cf..0000000000000 --- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll +++ /dev/null @@ -1,16 +0,0 @@ -target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-unknown-linux-gnu" - -source_filename = "thinlto_indirect_call_promotion.c" - -define void @a() { -entry: - ret void -} - -define internal void @c() !PGOFuncName !1 { -entry: - ret void -} - -!1 = !{!"thinlto_indirect_call_promotion.c:c"} diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw new file mode 100644 index 0000000000000000000000000000000000000000..5efda10bb98a941c04b6846db05d3691bc36aac0 GIT binary patch literal 528 zcmZoHO3N=Q$obF700xW@ih+Rz#(>i3d^BkWXQ;q~{}ABueD@>eRsa8Q&U2Q%6Ux8< zGg$D|W`(~SCZ7PRdViVAmkXvIW}d)Ih0eiSpPQvjy$#d`cE1Ih{ssc}!~BDb zHZZQZ)!lF8?|0^;r?06_@|lx9m%RP9_4R*h?2?>fc+6c=YT5HA>Z;F lib.h << EOF +void global_func(); +EOF + +# Creates lib.cc. `global_func` might call one of two indirect callees. One +# callee has internal linkage and the other has external linkage. +cat > lib.cc << EOF +#include "lib.h" +static void callee0() {} +void callee1() {} +typedef void (*FPT)(); +FPT calleeAddrs[] = {callee0, callee1}; +void global_func() { + FPT fp = nullptr; + fp = calleeAddrs[0]; + fp(); + fp = calleeAddrs[1]; + fp(); +} +EOF + +# Create main.cc. Function `main` calls `global_func`. +cat > main.cc << EOF +#include "lib.h" +int main() { + global_func(); +} +EOF + +# Clean up temporary files on exit and return to original directory. +cleanup() { + rm -f $OUTDIR/lib.h + rm -f $OUTDIR/lib.cc + rm -f $OUTDIR/main.cc + rm -f $OUTDIR/lib.h.pch + rm -f $OUTDIR/a.out + cd $CURDIR +} +trap cleanup EXIT + +# Generate instrumented binary +${CLANG} -fuse-ld=lld -O2 -fprofile-generate=. lib.h lib.cc main.cc +# Create raw profiles +env LLVM_PROFILE_FILE=thinlto_indirect_call_promotion.profraw ./a.out diff --git a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll index 173296f223e56..63e2a7a904a1b 100644 --- a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll +++ b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll @@ -1,39 +1,84 @@ -; Do setup work for all below tests: generate bitcode and combined index -; RUN: opt -module-summary %s -o %t.bc -; RUN: opt -module-summary %p/Inputs/thinlto_indirect_call_promotion.ll -o %t2.bc -; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc +; The raw profiles (and reduced IR if needed) could be re-generated (e.g., when +; there is a profile version bump) from script +; Inputs/update_thinlto_indirect_call_promotion_inputs.sh +; +; The script generates raw profiles. This regression test will convert it to +; indexed profiles. This way the test exercises code path where a profiled +; callee address in raw profiles is converted to function hash in index profiles. -; RUN: opt -passes=function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS -; IMPORTS-DAG: Import a -; IMPORTS-DAG: Import c +; The raw profiles storesd compressed function names, so profile reader should +; be built with zlib support to decompress them. +; REQUIRES: zlib -; RUN: opt %t4.bc -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt %t4.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK -; PASS-REMARK: Promote indirect call to a with count 1 out of 1 -; PASS-REMARK: Promote indirect call to c.llvm.0 with count 1 out of 1 +; RUN: rm -rf %t && split-file %s %t && cd %t -target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-unknown-linux-gnu" +; Do setup work for all below tests: convert raw profiles to indexed profiles, +; run profile-use pass, generate bitcode and combined ThinLTO index. +; Note `pgo-instr-use` pass runs without `pgo-icall-prom` pass. As a result ICP +; transformation won't happen at test setup time. +; RUN: llvm-profdata merge %p/Inputs/thinlto_indirect_call_promotion.profraw -o icp.profdata +; RUN: opt -passes=pgo-instr-use -pgo-test-profile-file=icp.profdata -module-summary main.ll -o main.bc +; RUN: opt -passes=pgo-instr-use -pgo-test-profile-file=icp.profdata -module-summary lib.ll -o lib.bc +; RUN: llvm-lto -thinlto -o summary main.bc lib.bc -@foo = external local_unnamed_addr global ptr, align 8 -@bar = external local_unnamed_addr global ptr, align 8 +; Test that callee with local linkage has `PGOFuncName` metadata while callee with external doesn't have it. +; RUN: llvm-dis lib.bc -o - | FileCheck %s --check-prefix=PGOName +; PGOName: define void @_Z7callee1v() {{.*}} !prof ![[#]] { +; PGOName: define internal void @_ZL7callee0v() {{.*}} !prof ![[#]] !PGOFuncName ![[#MD:]] { +; The source filename of `lib.ll` is specified as "lib.cc" (i.e., the name does +; not change with the directory), so match the full name here. +; PGOName: ![[#MD]] = !{!"lib.cc;_ZL7callee0v"} -define i32 @main() local_unnamed_addr { -entry: - %0 = load ptr, ptr @foo, align 8 -; ICALL-PROM: br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] - tail call void %0(), !prof !1 - %1 = load ptr, ptr @bar, align 8 -; ICALL-PROM: br i1 %{{[0-9]+}}, label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof [[BRANCH_WEIGHT:![0-9]+]] - tail call void %1(), !prof !2 +; Tests that both external and internal callees are correctly imported. +; RUN: opt -passes=function-import -summary-file summary.thinlto.bc main.bc -o main.import.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS +; IMPORTS: Import _Z7callee1v +; IMPORTS: Import _ZL7callee0v.llvm.[[#]] +; IMPORTS: Import _Z11global_funcv + +; Tests that ICP transformations happen. +; Both candidates are ICP'ed, check there is no `!VP` in the IR. +; RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM --implicit-check-not="!VP" +; RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK + +; PASS-REMARK: Promote indirect call to _ZL7callee0v.llvm.[[#]] with count 1 out of 1 +; PASS-REMARK: Promote indirect call to _Z7callee1v with count 1 out of 1 + +; ICALL-PROM: br i1 %[[#]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof ![[#BRANCH_WEIGHT1:]] +; ICALL-PROM: br i1 %[[#]], label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof ![[#BRANCH_WEIGHT1]] + +; ICALL-PROM: ![[#BRANCH_WEIGHT1]] = !{!"branch_weights", i32 1, i32 0} + +;--- main.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @main() { + call void @_Z11global_funcv() ret i32 0 } -!1 = !{!"VP", i32 0, i64 1, i64 -6289574019528802036, i64 1} -!2 = !{!"VP", i32 0, i64 1, i64 591260329866125152, i64 1} +declare void @_Z11global_funcv() -; Should not have a VP annotation on new indirect call (check before and after -; branch_weights annotation). -; ICALL-PROM-NOT: !"VP" -; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 0} -; ICALL-PROM-NOT: !"VP" +;--- lib.ll +source_filename = "lib.cc" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@calleeAddrs = global [2 x ptr] [ptr @_ZL7callee0v, ptr @_Z7callee1v] + +define void @_Z7callee1v() { + ret void +} + +define internal void @_ZL7callee0v() { + ret void +} + +define void @_Z11global_funcv() { +entry: + %0 = load ptr, ptr @calleeAddrs + call void %0() + %1 = load ptr, ptr getelementptr inbounds ([2 x ptr], ptr @calleeAddrs, i64 0, i64 1) + call void %1() + ret void +} diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index e6613a90dc7c5..6a71a975fbb12 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -1379,7 +1379,7 @@ TEST(SymtabTest, instr_prof_symtab_compression_test) { TEST_P(MaybeSparseInstrProfTest, remapping_test) { Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err); - Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err); + Writer.addRecord({"file;_Z3barf", 0x567, {5, 6, 7}}, Err); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"( type i l @@ -1397,7 +1397,7 @@ TEST_P(MaybeSparseInstrProfTest, remapping_test) { EXPECT_EQ(4u, Counts[3]); } - for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) { + for (StringRef BarName : {"file;_Z3barf", "file;_Z4quuxf"}) { EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts), Succeeded()); ASSERT_EQ(3u, Counts.size()); From 8e2cc19e2e60d348859c0967a2707bcc8cb9936a Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 18 Dec 2023 11:16:49 -0600 Subject: [PATCH 148/884] [LinkerWrapper] Forward more arguments to the CPU offloading linker (#75757) Summary: The CPU target currently inherits all the libraries from the normal link job to ensure that it has access to the same envrionment that the host does. However, this previously was not respecting argument libraries that are passed by name rather than `-l` as well as the whole archive flags. This patch fixes this to allow the CPU linker to correctly pick up the libraries associated with things like address sanitizers. Fixes: https://github.com/llvm/llvm-project/issues/75651 --- clang/test/Driver/linker-wrapper.c | 6 ++-- .../ClangLinkerWrapper.cpp | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c index b763a003452ba..e51c5ea381d31 100644 --- a/clang/test/Driver/linker-wrapper.c +++ b/clang/test/Driver/linker-wrapper.c @@ -49,10 +49,12 @@ // RUN: --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu \ // RUN: --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out +// RUN: llvm-ar rcs %t.a %t.o // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld.lld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CPU-LINK +// RUN: --linker-path=/usr/bin/ld.lld -- --whole-archive %t.a --no-whole-archive \ +// RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CPU-LINK -// CPU-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu -march=native -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o -Wl,-Bsymbolic -shared +// CPU-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu -march=native -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o -Wl,-Bsymbolic -shared -Wl,--whole-archive {{.*}}.a -Wl,--no-whole-archive // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -mllvm -openmp-opt-disable \ diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index bebe76355eb46..122ba1998eb83 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -396,11 +396,31 @@ Expected clang(ArrayRef InputFiles, const ArgList &Args) { CmdArgs.push_back("-Wl,-Bsymbolic"); CmdArgs.push_back("-shared"); ArgStringList LinkerArgs; - for (const opt::Arg *Arg : Args.filtered(OPT_library, OPT_library_path)) - Arg->render(Args, LinkerArgs); - for (const opt::Arg *Arg : Args.filtered(OPT_rpath)) - LinkerArgs.push_back( - Args.MakeArgString("-Wl,-rpath," + StringRef(Arg->getValue()))); + for (const opt::Arg *Arg : + Args.filtered(OPT_INPUT, OPT_library, OPT_library_path, OPT_rpath, + OPT_whole_archive, OPT_no_whole_archive)) { + // Sometimes needed libraries are passed by name, such as when using + // sanitizers. We need to check the file magic for any libraries. + if (Arg->getOption().matches(OPT_INPUT)) { + if (!sys::fs::exists(Arg->getValue()) || + sys::fs::is_directory(Arg->getValue())) + continue; + + file_magic Magic; + if (auto EC = identify_magic(Arg->getValue(), Magic)) + return createStringError(inconvertibleErrorCode(), + "Failed to open %s", Arg->getValue()); + if (Magic != file_magic::archive && + Magic != file_magic::elf_shared_object) + continue; + } + if (Arg->getOption().matches(OPT_whole_archive)) + LinkerArgs.push_back(Args.MakeArgString("-Wl,--whole-archive")); + else if (Arg->getOption().matches(OPT_no_whole_archive)) + LinkerArgs.push_back(Args.MakeArgString("-Wl,--no-whole-archive")); + else + Arg->render(Args, LinkerArgs); + } llvm::copy(LinkerArgs, std::back_inserter(CmdArgs)); } From 3768039913be32666a316a2b5c12739c423dbc61 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Mon, 18 Dec 2023 12:26:18 -0500 Subject: [PATCH 149/884] [OpenMP] Directly use user's grid and block size in kernel language mode (#70612) In kernel language mode, use user's grid and blocks size directly. No validity check, which means if user's values are too large, the launch will fail, similar to what CUDA and HIP are doing right now. --- .../common/include/PluginInterface.h | 3 ++ .../common/src/PluginInterface.cpp | 8 ++++ .../libomptarget/test/offloading/ompx_bare.c | 38 +++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 openmp/libomptarget/test/offloading/ompx_bare.c diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h index 716b0ad784331..28484ae4d5f5e 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h @@ -397,6 +397,9 @@ struct GenericKernelTy { /// The prototype kernel launch environment. KernelLaunchEnvironmentTy KernelLaunchEnvironment; + + /// If the kernel is a bare kernel. + bool IsBareKernel = false; }; /// Class representing a map of host pinned allocations. We track these pinned diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 1d96468340a08..1c9777dba7a9a 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -436,6 +436,7 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice, Name, ErrStr.data()); assert(KernelEnvironment.Configuration.ReductionDataSize == 0 && "Default initialization failed."); + IsBareKernel = true; } // Max = Config.Max > 0 ? min(Config.Max, Device.Max) : Device.Max; @@ -594,6 +595,10 @@ uint32_t GenericKernelTy::getNumThreads(GenericDeviceTy &GenericDevice, uint32_t ThreadLimitClause[3]) const { assert(ThreadLimitClause[1] == 0 && ThreadLimitClause[2] == 0 && "Multi dimensional launch not supported yet."); + + if (IsBareKernel && ThreadLimitClause[0] > 0) + return ThreadLimitClause[0]; + if (ThreadLimitClause[0] > 0 && isGenericMode()) ThreadLimitClause[0] += GenericDevice.getWarpSize(); @@ -610,6 +615,9 @@ uint64_t GenericKernelTy::getNumBlocks(GenericDeviceTy &GenericDevice, assert(NumTeamsClause[1] == 0 && NumTeamsClause[2] == 0 && "Multi dimensional launch not supported yet."); + if (IsBareKernel && NumTeamsClause[0] > 0) + return NumTeamsClause[0]; + if (NumTeamsClause[0] > 0) { // TODO: We need to honor any value and consequently allow more than the // block limit. For this we might need to start multiple kernels or let the diff --git a/openmp/libomptarget/test/offloading/ompx_bare.c b/openmp/libomptarget/test/offloading/ompx_bare.c new file mode 100644 index 0000000000000..fb3810bd1df12 --- /dev/null +++ b/openmp/libomptarget/test/offloading/ompx_bare.c @@ -0,0 +1,38 @@ +// RUN: %libomptarget-compile-generic +// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | %fcheck-generic +// +// UNSUPPORTED: x86_64-pc-linux-gnu +// UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: aarch64-unknown-linux-gnu +// UNSUPPORTED: aarch64-unknown-linux-gnu-LTO + +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + const int num_blocks = 64; + const int block_size = 64; + const int N = num_blocks * block_size; + int *data = (int *)malloc(N * sizeof(int)); + + // CHECK: "PluginInterface" device 0 info: Launching kernel __omp_offloading_{{.*}} with 64 blocks and 64 threads in SPMD mode + +#pragma omp target teams ompx_bare num_teams(num_blocks) thread_limit(block_size) map(from: data[0:N]) + { + int bid = ompx_block_id_x(); + int bdim = ompx_block_dim_x(); + int tid = ompx_thread_id_x(); + int idx = bid * bdim + tid; + data[idx] = idx; + } + + for (int i = 0; i < N; ++i) + assert(data[i] == i); + + // CHECK: PASS + printf("PASS\n"); + + return 0; +} From 3aa5d71127ae31f13cc3383ea461fd5bf86bf055 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 18 Dec 2023 09:39:55 -0800 Subject: [PATCH 150/884] Revert "[PGO][GlobalValue][LTO]In GlobalValues::getGlobalIdentifier, use semicolon as delimiter for local-linkage varibles." (#75835) Reverts llvm/llvm-project#74008 The compiler-rt test failed due to `llvm-dis` not found (https://lab.llvm.org/buildbot/#/builders/127/builds/59884) Will revert and investigate how to require the proper dependency. --- ...trprof-thinlto-indirect-call-promotion.cpp | 115 ------------------ llvm/include/llvm/IR/GlobalValue.h | 4 - llvm/include/llvm/ProfileData/InstrProf.h | 26 ++-- llvm/lib/IR/Globals.cpp | 12 +- llvm/lib/ProfileData/InstrProf.cpp | 38 ++---- llvm/lib/ProfileData/InstrProfReader.cpp | 11 +- .../thinlto-function-summary-originalnames.ll | 10 +- llvm/test/ThinLTO/X86/memprof-basic.ll | 26 ++-- .../X86/memprof-duplicate-context-ids.ll | 10 +- .../ThinLTO/X86/memprof-funcassigncloning.ll | 6 +- llvm/test/ThinLTO/X86/memprof-indirectcall.ll | 32 ++--- llvm/test/ThinLTO/X86/memprof-inlined.ll | 14 +-- .../Inputs/thinlto_indirect_call_promotion.ll | 16 +++ .../thinlto_indirect_call_promotion.profraw | Bin 528 -> 0 bytes ..._thinlto_indirect_call_promotion_inputs.sh | 62 ---------- .../thinlto_indirect_call_promotion.ll | 105 +++++----------- llvm/unittests/ProfileData/InstrProfTest.cpp | 4 +- 17 files changed, 128 insertions(+), 363 deletions(-) delete mode 100644 compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll delete mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw delete mode 100755 llvm/test/Transforms/PGOProfile/Inputs/update_thinlto_indirect_call_promotion_inputs.sh diff --git a/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp b/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp deleted file mode 100644 index 82ca1cd7d0a56..0000000000000 --- a/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// This is a regression test for ThinLTO indirect-call-promotion when candidate -// callees need to be imported from another IR module. In the C++ test case, -// `main` calls `global_func` which is defined in another module. `global_func` -// has two indirect callees, one has external linkage and one has local linkage. -// All three functions should be imported into the IR module of main. - -// What the test does: -// - Generate raw profiles from executables and convert it to indexed profiles. -// During the conversion, a profiled callee address in raw profiles will be -// converted to function hash in indexed profiles. -// - Run IRPGO profile use and ThinTLO prelink pipeline and get LLVM bitcodes -// for both cpp files in the C++ test case. -// - Generate ThinLTO summary file with LLVM bitcodes, and run `function-import` pass. -// - Run `pgo-icall-prom` pass for the IR module which needs to import callees. - -// Use lld as linker for more robust test. We need to REQUIRE LLVMgold.so for -// LTO if default linker is GNU ld or gold anyway. -// REQUIRES: lld-available - -// Test should fail where linkage-name and mangled-name diverges, see issue https://github.com/llvm/llvm-project/issues/74565). -// Currently, this name divergence happens on Mach-O object file format, or on -// many (but not all) 32-bit Windows systems. -// -// XFAIL: system-darwin -// -// Mark 32-bit Windows as UNSUPPORTED for now as opposed to XFAIL. This test -// should fail on many (but not all) 32-bit Windows systems and succeed on the -// rest. The flexibility in triple string parsing makes it tricky to capture -// both sets accurately. i[3-9]86 specifies arch as Triple::ArchType::x86, (win32|windows) -// specifies OS as Triple::OS::Win32 -// -// UNSUPPORTED: target={{i.86.*windows.*}} - -// RUN: rm -rf %t && split-file %s %t && cd %t - -// Do setup work for all below tests. -// Generate raw profiles from real programs and convert it into indexed profiles. -// Use clangxx_pgogen for IR level instrumentation for C++. -// RUN: %clangxx_pgogen -fuse-ld=lld -O2 lib.cpp main.cpp -o main -// RUN: env LLVM_PROFILE_FILE=main.profraw %run ./main -// RUN: llvm-profdata merge main.profraw -o main.profdata - -// Use profile on lib and get bitcode, test that local function callee0 has -// expected !PGOFuncName metadata and external function callee1 doesn't have -// !PGOFuncName metadata. Explicitly skip ICP pass to test ICP happens as -// expected in the IR module that imports functions from lib. -// RUN: %clang -mllvm -disable-icp -fprofile-use=main.profdata -flto=thin -O2 -c lib.cpp -o lib.bc -// RUN: llvm-dis lib.bc -o - | FileCheck %s --check-prefix=PGOName - -// Use profile on main and get bitcode. -// RUN: %clang -fprofile-use=main.profdata -flto=thin -O2 -c main.cpp -o main.bc - -// Run llvm-lto to get summary file. -// RUN: llvm-lto -thinlto -o summary main.bc lib.bc - -// Test the imports of functions. Default import thresholds would work but do -// explicit override to be more futureproof. Note all functions have one basic -// block with a function-entry-count of one, so they are actually hot functions -// per default profile summary hotness cutoff. -// RUN: opt -passes=function-import -import-instr-limit=100 -import-cold-multiplier=1 -summary-file summary.thinlto.bc main.bc -o main.import.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS -// Test that '_Z11global_funcv' has indirect calls annotated with value profiles. -// RUN: llvm-dis main.import.bc -o - | FileCheck %s --check-prefix=IR - -// Test that both candidates are ICP'ed and there is no `!VP` in the IR. -// RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefixes=ICP-IR,ICP-REMARK --implicit-check-not="!VP" - -// IMPORTS: main.cpp: Import _Z7callee1v -// IMPORTS: main.cpp: Import _ZL7callee0v.llvm.[[#]] -// IMPORTS: main.cpp: Import _Z11global_funcv - -// PGOName: define {{(dso_local )?}}void @_Z7callee1v() #[[#]] !prof ![[#]] { -// PGOName: define internal void @_ZL7callee0v() #[[#]] !prof ![[#]] !PGOFuncName ![[#MD:]] { -// PGOName: ![[#MD]] = !{!"{{.*}}lib.cpp;_ZL7callee0v"} - -// IR-LABEL: define available_externally {{.*}} void @_Z11global_funcv() {{.*}} !prof ![[#]] { -// IR-NEXT: entry: -// IR-NEXT: %0 = load ptr, ptr @calleeAddrs -// IR-NEXT: tail call void %0(), !prof ![[#PROF1:]] -// IR-NEXT: %1 = load ptr, ptr getelementptr inbounds ([2 x ptr], ptr @calleeAddrs, -// IR-NEXT: tail call void %1(), !prof ![[#PROF2:]] - -// The GUID of indirect callee is the MD5 hash of `/path/to/lib.cpp;_ZL7callee0v` -// that depends on the directory. Use [[#]] for its MD5 hash. -// Use {{.*}} for integer types so the test works on 32-bit and 64-bit systems. -// IR: ![[#PROF1]] = !{!"VP", i32 0, {{.*}} 1, {{.*}} [[#]], {{.*}} 1} -// IR: ![[#PROF2]] = !{!"VP", i32 0, {{.*}} 1, {{.*}} -3993653843325621743, {{.*}} 1} - -// ICP-REMARK: Promote indirect call to _ZL7callee0v.llvm.[[#]] with count 1 out of 1 -// ICP-REMARK: Promote indirect call to _Z7callee1v with count 1 out of 1 - -// ICP-IR: br i1 %[[#]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof ![[#BRANCH_WEIGHT1:]] -// ICP-IR: br i1 %[[#]], label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof ![[#BRANCH_WEIGHT1]] -// ICP-IR: ![[#BRANCH_WEIGHT1]] = !{!"branch_weights", i32 1, i32 0} - -//--- lib.h -void global_func(); - -//--- lib.cpp -#include "lib.h" -static void callee0() {} -void callee1() {} -typedef void (*FPT)(); -FPT calleeAddrs[] = {callee0, callee1}; -// `global_func`` might call one of two indirect callees. callee0 has internal -// linkage and callee1 has external linkage. -void global_func() { - FPT fp = calleeAddrs[0]; - fp(); - fp = calleeAddrs[1]; - fp(); -} - -//--- main.cpp -#include "lib.h" -int main() { global_func(); } diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index e97a7f2b96360..d1891c157099d 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -41,10 +41,6 @@ namespace Intrinsic { typedef unsigned ID; } // end namespace Intrinsic -// Choose ';' as the delimiter. ':' was used once but it doesn't work well for -// Objective-C functions which commonly have :'s in their names. -inline constexpr char kGlobalIdentifierDelimiter = ';'; - class GlobalValue : public Constant { public: /// An enumeration for the kinds of linkage for global values. diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 36be2e7d869e7..288dc71d756ae 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -171,8 +171,6 @@ inline StringRef getInstrProfCounterBiasVarName() { /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } -/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is -/// for front-end (Clang, etc) instrumentation. /// Return the modified name for function \c F suitable to be /// used the key for profile lookup. Variable \c InLTO indicates if this /// is called in LTO optimization passes. @@ -198,22 +196,20 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO = false); std::pair getParsedIRPGOFuncName(StringRef IRPGOFuncName); /// Return the name of the global variable used to store a function -/// name in PGO instrumentation. \c FuncName is the IRPGO function name -/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO -/// function name (returned by \c getPGOFuncName) for front-end instrumentation. +/// name in PGO instrumentation. \c FuncName is the name of the function +/// returned by the \c getPGOFuncName call. std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage); /// Create and return the global variable for function name used in PGO -/// instrumentation. \c FuncName is the IRPGO function name (returned by -/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name -/// (returned by \c getPGOFuncName) for front-end instrumentation. +/// instrumentation. \c FuncName is the name of the function returned +/// by \c getPGOFuncName call. GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); /// Create and return the global variable for function name used in PGO -/// instrumentation. \c FuncName is the IRPGO function name (returned by -/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name -/// (returned by \c getPGOFuncName) for front-end instrumentation. +/// instrumentation. /// \c FuncName is the name of the function +/// returned by \c getPGOFuncName call, \c M is the owning module, +/// and \c Linkage is the linkage of the instrumented function. GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, StringRef PGOFuncName); @@ -421,11 +417,11 @@ uint64_t ComputeHash(StringRef K); } // end namespace IndexedInstrProf -/// A symbol table used for function [IR]PGO name look-up with keys +/// A symbol table used for function PGO name look-up with keys /// (such as pointers, md5hash values) to the function. A function's -/// [IR]PGO name or name's md5hash are used in retrieving the profile -/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName -/// methods for details how [IR]PGO name is formed. +/// PGO name or name's md5hash are used in retrieving the profile +/// data of the function. See \c getPGOFuncName() method for details +/// on how PGO name is formed. class InstrProfSymtab { public: using AddrHashMap = std::vector>; diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index 239acd2181e85..51bdbeb0abf2c 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -144,27 +144,25 @@ void GlobalObject::copyAttributesFrom(const GlobalObject *Src) { std::string GlobalValue::getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName) { + // Value names may be prefixed with a binary '1' to indicate // that the backend should not modify the symbols due to any platform // naming convention. Do not include that '1' in the PGO profile name. if (Name[0] == '\1') Name = Name.substr(1); - std::string GlobalName; + std::string NewName = std::string(Name); if (llvm::GlobalValue::isLocalLinkage(Linkage)) { // For local symbols, prepend the main file name to distinguish them. // Do not include the full path in the file name since there's no guarantee // that it will stay the same, e.g., if the files are checked out from // version control in different locations. if (FileName.empty()) - GlobalName += ""; + NewName = NewName.insert(0, ":"); else - GlobalName += FileName; - - GlobalName += kGlobalIdentifierDelimiter; + NewName = NewName.insert(0, FileName.str() + ":"); } - GlobalName += Name; - return GlobalName; + return NewName; } std::string GlobalValue::getGlobalIdentifier() const { diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index f5fde84eb6bd0..649d814cfd9de 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -246,27 +246,11 @@ std::string InstrProfError::message() const { char InstrProfError::ID = 0; -std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage, +std::string getPGOFuncName(StringRef RawFuncName, + GlobalValue::LinkageTypes Linkage, StringRef FileName, uint64_t Version LLVM_ATTRIBUTE_UNUSED) { - // Value names may be prefixed with a binary '1' to indicate - // that the backend should not modify the symbols due to any platform - // naming convention. Do not include that '1' in the PGO profile name. - if (Name[0] == '\1') - Name = Name.substr(1); - - std::string NewName = std::string(Name); - if (llvm::GlobalValue::isLocalLinkage(Linkage)) { - // For local symbols, prepend the main file name to distinguish them. - // Do not include the full path in the file name since there's no guarantee - // that it will stay the same, e.g., if the files are checked out from - // version control in different locations. - if (FileName.empty()) - NewName = NewName.insert(0, ":"); - else - NewName = NewName.insert(0, FileName.str() + ":"); - } - return NewName; + return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); } // Strip NumPrefix level of directory name from PathNameStr. If the number of @@ -316,10 +300,12 @@ getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { SmallString<64> Name; - // FIXME: Mangler's handling is kept outside of `getGlobalIdentifier` for now. - // For more details please check issue #74565. + if (llvm::GlobalValue::isLocalLinkage(Linkage)) { + Name.append(FileName.empty() ? "" : FileName); + Name.append(";"); + } Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); - return GlobalValue::getGlobalIdentifier(Name, Linkage, FileName); + return Name.str().str(); } static std::optional lookupPGONameFromMetadata(MDNode *MD) { @@ -366,9 +352,6 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO) { return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F)); } -// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is -// for front-end (Clang, etc) instrumentation. -// The implementation is kept for profile matching from older profiles. // This is similar to `getIRPGOFuncName` except that this function calls // 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls // 'getIRPGONameForGlobalObject'. See the difference between two callees in the @@ -401,9 +384,8 @@ getParsedIRPGOFuncName(StringRef IRPGOFuncName) { StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { if (FileName.empty()) return PGOFuncName; - // Drop the file name including ':' or ';'. See getIRPGONameForGlobalObject as - // well. - if (PGOFuncName.startswith(FileName)) + // Drop the file name including ':'. See also getPGOFuncName. + if (PGOFuncName.starts_with(FileName)) PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); return PGOFuncName; } diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 7f8eb0ce747fe..068922d421f8b 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1008,14 +1008,13 @@ class llvm::InstrProfReaderItaniumRemapper /// Extract the original function name from a PGO function name. static StringRef extractName(StringRef Name) { - // We can have multiple pieces separated by kGlobalIdentifierDelimiter ( - // semicolon now and colon in older profiles); there can be pieces both - // before and after the mangled name. Find the first part that starts with - // '_Z'; we'll assume that's the mangled name we want. + // We can have multiple :-separated pieces; there can be pieces both + // before and after the mangled name. Find the first part that starts + // with '_Z'; we'll assume that's the mangled name we want. std::pair Parts = {StringRef(), Name}; while (true) { - Parts = Parts.second.split(kGlobalIdentifierDelimiter); - if (Parts.first.startswith("_Z")) + Parts = Parts.second.split(':'); + if (Parts.first.starts_with("_Z")) return Parts.first; if (Parts.second.empty()) return Name; diff --git a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll index 0139f00b4aa3f..7cc9654c8c7b1 100644 --- a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll @@ -6,13 +6,13 @@ ; COMBINED: -; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-DAG: -; COMBINED-DAG: +; COMBINED-DAG: ; COMBINED-DAG: ; COMBINED-NEXT: diff --git a/llvm/test/ThinLTO/X86/memprof-basic.ll b/llvm/test/ThinLTO/X86/memprof-basic.ll index 54e01e5fcdf95..0d466830ba57d 100644 --- a/llvm/test/ThinLTO/X86/memprof-basic.ll +++ b/llvm/test/ThinLTO/X86/memprof-basic.ll @@ -148,7 +148,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 ; DUMP: Node [[BAZ]] -; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -157,7 +157,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 ; DUMP: Node [[FOO]] -; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -167,7 +167,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -175,7 +175,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -197,7 +197,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAR2:0x[a-z0-9]+]] ; DUMP: Node [[BAZ]] -; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -207,7 +207,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAZ2:0x[a-z0-9]+]] ; DUMP: Node [[FOO]] -; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -217,7 +217,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO2:0x[a-z0-9]+]] ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -225,7 +225,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 8107868197919466657 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 6731117468105397038 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -233,7 +233,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[FOO2]] -; DUMP: Callee: 1807954217441101578 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 5878270615442837395 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -243,7 +243,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clone of [[FOO]] ; DUMP: Node [[BAZ2]] -; DUMP: Callee: 11481133863268513686 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 9832687305761716512 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -344,7 +344,7 @@ attributes #0 = { noinline optnone } ; DOTCLONED: } -; DISTRIB: ^[[BAZ:[0-9]+]] = gv: (guid: 1807954217441101578, {{.*}} callsites: ((callee: ^[[BAR:[0-9]+]], clones: (0, 1) -; DISTRIB: ^[[FOO:[0-9]+]] = gv: (guid: 8107868197919466657, {{.*}} callsites: ((callee: ^[[BAZ]], clones: (0, 1) -; DISTRIB: ^[[BAR]] = gv: (guid: 11481133863268513686, {{.*}} allocs: ((versions: (notcold, cold) +; DISTRIB: ^[[BAZ:[0-9]+]] = gv: (guid: 5878270615442837395, {{.*}} callsites: ((callee: ^[[BAR:[0-9]+]], clones: (0, 1) +; DISTRIB: ^[[FOO:[0-9]+]] = gv: (guid: 6731117468105397038, {{.*}} callsites: ((callee: ^[[BAZ]], clones: (0, 1) +; DISTRIB: ^[[BAR]] = gv: (guid: 9832687305761716512, {{.*}} allocs: ((versions: (notcold, cold) ; DISTRIB: ^[[MAIN:[0-9]+]] = gv: (guid: 15822663052811949562, {{.*}} callsites: ((callee: ^[[FOO]], clones: (0), {{.*}} (callee: ^[[FOO]], clones: (1) diff --git a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll index 65d794e9cba87..f7ba0d27dca78 100644 --- a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll +++ b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll @@ -260,8 +260,8 @@ attributes #0 = { noinline optnone} ; STATS-BE: 1 memprof-context-disambiguation - Number of original (not cloned) allocations with memprof profiles during ThinLTO backend -; DISTRIB: ^[[E:[0-9]+]] = gv: (guid: 331966645857188136, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) -; DISTRIB: ^[[D]] = gv: (guid: 11079124245221721799, {{.*}} allocs: ((versions: (notcold, cold) -; DISTRIB: ^[[F:[0-9]+]] = gv: (guid: 11254287701717398916, {{.*}} callsites: ((callee: ^[[D]], clones: (0) -; DISTRIB: ^[[B:[0-9]+]] = gv: (guid: 13579056193435805313, {{.*}} callsites: ((callee: ^[[D]], clones: (1) -; DISTRIB: ^[[C:[0-9]+]] = gv: (guid: 15101436305866936160, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) +; DISTRIB: ^[[C:[0-9]+]] = gv: (guid: 1643923691937891493, {{.*}} callsites: ((callee: ^[[D:[0-9]+]], clones: (1) +; DISTRIB: ^[[D]] = gv: (guid: 4881081444663423788, {{.*}} allocs: ((versions: (notcold, cold) +; DISTRIB: ^[[B:[0-9]+]] = gv: (guid: 14590037969532473829, {{.*}} callsites: ((callee: ^[[D]], clones: (1) +; DISTRIB: ^[[F:[0-9]+]] = gv: (guid: 17035303613541779335, {{.*}} callsites: ((callee: ^[[D]], clones: (0) +; DISTRIB: ^[[E:[0-9]+]] = gv: (guid: 17820708772846654376, {{.*}} callsites: ((callee: ^[[D]], clones: (1) diff --git a/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll b/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll index f1a494d077fef..9a72ae43b2f1e 100644 --- a/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll +++ b/llvm/test/ThinLTO/X86/memprof-funcassigncloning.ll @@ -176,7 +176,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[ENEW1CLONE:0x[a-z0-9]+]] ; DUMP: Node [[D:0x[a-z0-9]+]] -; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 6 ; DUMP: CalleeEdges: @@ -185,7 +185,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[C]] -; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 2 5 ; DUMP: CalleeEdges: @@ -194,7 +194,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[B]] -; DUMP: Callee: 16147627620923572899 (_Z1EPPcS0_) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 10758063066234039248 (_Z1EPPcS0_) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 4 ; DUMP: CalleeEdges: diff --git a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll index 07a52f441ca27..76273959f4f4a 100644 --- a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll +++ b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll @@ -202,7 +202,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 6 ; DUMP: Node [[AX]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -225,7 +225,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 5 ; DUMP: Node [[MAIN3]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -233,7 +233,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN4]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -241,7 +241,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -249,7 +249,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[BX]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 4 5 ; DUMP: CalleeEdges: @@ -258,7 +258,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 ; DUMP: Node [[MAIN5]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: @@ -266,7 +266,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN6]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 5 ; DUMP: CalleeEdges: @@ -274,7 +274,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 6 ; DUMP: CalleeEdges: @@ -302,7 +302,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO2:0x[a-z0-9]+]] ; DUMP: Node [[AX]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 1 2 ; DUMP: CalleeEdges: @@ -324,7 +324,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6]] AllocTypes: NotCold ContextIds: 5 ; DUMP: Node [[MAIN3]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 ; DUMP: CalleeEdges: @@ -332,7 +332,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN4]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 ; DUMP: CalleeEdges: @@ -340,7 +340,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -348,7 +348,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[BX]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 4 5 ; DUMP: CalleeEdges: @@ -357,7 +357,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 ; DUMP: Node [[MAIN5]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: @@ -365,7 +365,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN6]] -; DUMP: Callee: 2040285415115148168 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 5 ; DUMP: CalleeEdges: @@ -373,7 +373,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 15844184524768596045 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 6 ; DUMP: CalleeEdges: diff --git a/llvm/test/ThinLTO/X86/memprof-inlined.ll b/llvm/test/ThinLTO/X86/memprof-inlined.ll index 89df345b22042..feb9c94344223 100644 --- a/llvm/test/ThinLTO/X86/memprof-inlined.ll +++ b/llvm/test/ThinLTO/X86/memprof-inlined.ll @@ -170,7 +170,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 3 ; DUMP: CalleeEdges: @@ -179,7 +179,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 4 ; DUMP: CalleeEdges: @@ -201,7 +201,7 @@ attributes #0 = { noinline optnone } ;; This is the node synthesized for the call to bar in foo that was created ;; by inlining baz into foo. ; DUMP: Node [[FOO]] -; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: NotColdCold ; DUMP: ContextIds: 3 4 ; DUMP: CalleeEdges: @@ -234,7 +234,7 @@ attributes #0 = { noinline optnone } ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 ; DUMP: Node [[MAIN1]] -; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 1 3 ; DUMP: CalleeEdges: @@ -243,7 +243,7 @@ attributes #0 = { noinline optnone } ; DUMP: CallerEdges: ; DUMP: Node [[MAIN2]] -; DUMP: Callee: 644169328058379925 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 2 4 ; DUMP: CalleeEdges: @@ -264,7 +264,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[BAR2:0x[a-z0-9]+]] ; DUMP: Node [[FOO]] -; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: NotCold ; DUMP: ContextIds: 3 ; DUMP: CalleeEdges: @@ -274,7 +274,7 @@ attributes #0 = { noinline optnone } ; DUMP: Clones: [[FOO3]] ; DUMP: Node [[FOO3]] -; DUMP: Callee: 10349908617508457487 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) ; DUMP: AllocTypes: Cold ; DUMP: ContextIds: 4 ; DUMP: CalleeEdges: diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll new file mode 100644 index 0000000000000..7412120bb52cf --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll @@ -0,0 +1,16 @@ +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +source_filename = "thinlto_indirect_call_promotion.c" + +define void @a() { +entry: + ret void +} + +define internal void @c() !PGOFuncName !1 { +entry: + ret void +} + +!1 = !{!"thinlto_indirect_call_promotion.c:c"} diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw deleted file mode 100644 index 5efda10bb98a941c04b6846db05d3691bc36aac0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 528 zcmZoHO3N=Q$obF700xW@ih+Rz#(>i3d^BkWXQ;q~{}ABueD@>eRsa8Q&U2Q%6Ux8< zGg$D|W`(~SCZ7PRdViVAmkXvIW}d)Ih0eiSpPQvjy$#d`cE1Ih{ssc}!~BDb zHZZQZ)!lF8?|0^;r?06_@|lx9m%RP9_4R*h?2?>fc+6c=YT5HA>Z;F lib.h << EOF -void global_func(); -EOF - -# Creates lib.cc. `global_func` might call one of two indirect callees. One -# callee has internal linkage and the other has external linkage. -cat > lib.cc << EOF -#include "lib.h" -static void callee0() {} -void callee1() {} -typedef void (*FPT)(); -FPT calleeAddrs[] = {callee0, callee1}; -void global_func() { - FPT fp = nullptr; - fp = calleeAddrs[0]; - fp(); - fp = calleeAddrs[1]; - fp(); -} -EOF - -# Create main.cc. Function `main` calls `global_func`. -cat > main.cc << EOF -#include "lib.h" -int main() { - global_func(); -} -EOF - -# Clean up temporary files on exit and return to original directory. -cleanup() { - rm -f $OUTDIR/lib.h - rm -f $OUTDIR/lib.cc - rm -f $OUTDIR/main.cc - rm -f $OUTDIR/lib.h.pch - rm -f $OUTDIR/a.out - cd $CURDIR -} -trap cleanup EXIT - -# Generate instrumented binary -${CLANG} -fuse-ld=lld -O2 -fprofile-generate=. lib.h lib.cc main.cc -# Create raw profiles -env LLVM_PROFILE_FILE=thinlto_indirect_call_promotion.profraw ./a.out diff --git a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll index 63e2a7a904a1b..173296f223e56 100644 --- a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll +++ b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll @@ -1,84 +1,39 @@ -; The raw profiles (and reduced IR if needed) could be re-generated (e.g., when -; there is a profile version bump) from script -; Inputs/update_thinlto_indirect_call_promotion_inputs.sh -; -; The script generates raw profiles. This regression test will convert it to -; indexed profiles. This way the test exercises code path where a profiled -; callee address in raw profiles is converted to function hash in index profiles. +; Do setup work for all below tests: generate bitcode and combined index +; RUN: opt -module-summary %s -o %t.bc +; RUN: opt -module-summary %p/Inputs/thinlto_indirect_call_promotion.ll -o %t2.bc +; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc -; The raw profiles storesd compressed function names, so profile reader should -; be built with zlib support to decompress them. -; REQUIRES: zlib +; RUN: opt -passes=function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS +; IMPORTS-DAG: Import a +; IMPORTS-DAG: Import c -; RUN: rm -rf %t && split-file %s %t && cd %t +; RUN: opt %t4.bc -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt %t4.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; PASS-REMARK: Promote indirect call to a with count 1 out of 1 +; PASS-REMARK: Promote indirect call to c.llvm.0 with count 1 out of 1 -; Do setup work for all below tests: convert raw profiles to indexed profiles, -; run profile-use pass, generate bitcode and combined ThinLTO index. -; Note `pgo-instr-use` pass runs without `pgo-icall-prom` pass. As a result ICP -; transformation won't happen at test setup time. -; RUN: llvm-profdata merge %p/Inputs/thinlto_indirect_call_promotion.profraw -o icp.profdata -; RUN: opt -passes=pgo-instr-use -pgo-test-profile-file=icp.profdata -module-summary main.ll -o main.bc -; RUN: opt -passes=pgo-instr-use -pgo-test-profile-file=icp.profdata -module-summary lib.ll -o lib.bc -; RUN: llvm-lto -thinlto -o summary main.bc lib.bc - -; Test that callee with local linkage has `PGOFuncName` metadata while callee with external doesn't have it. -; RUN: llvm-dis lib.bc -o - | FileCheck %s --check-prefix=PGOName -; PGOName: define void @_Z7callee1v() {{.*}} !prof ![[#]] { -; PGOName: define internal void @_ZL7callee0v() {{.*}} !prof ![[#]] !PGOFuncName ![[#MD:]] { -; The source filename of `lib.ll` is specified as "lib.cc" (i.e., the name does -; not change with the directory), so match the full name here. -; PGOName: ![[#MD]] = !{!"lib.cc;_ZL7callee0v"} - -; Tests that both external and internal callees are correctly imported. -; RUN: opt -passes=function-import -summary-file summary.thinlto.bc main.bc -o main.import.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS -; IMPORTS: Import _Z7callee1v -; IMPORTS: Import _ZL7callee0v.llvm.[[#]] -; IMPORTS: Import _Z11global_funcv - -; Tests that ICP transformations happen. -; Both candidates are ICP'ed, check there is no `!VP` in the IR. -; RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM --implicit-check-not="!VP" -; RUN: opt main.import.bc -icp-lto -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK - -; PASS-REMARK: Promote indirect call to _ZL7callee0v.llvm.[[#]] with count 1 out of 1 -; PASS-REMARK: Promote indirect call to _Z7callee1v with count 1 out of 1 - -; ICALL-PROM: br i1 %[[#]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof ![[#BRANCH_WEIGHT1:]] -; ICALL-PROM: br i1 %[[#]], label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof ![[#BRANCH_WEIGHT1]] - -; ICALL-PROM: ![[#BRANCH_WEIGHT1]] = !{!"branch_weights", i32 1, i32 0} - -;--- main.ll -target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" -target triple = "x86_64-unknown-linux-gnu" - -define i32 @main() { - call void @_Z11global_funcv() - ret i32 0 -} - -declare void @_Z11global_funcv() - -;--- lib.ll -source_filename = "lib.cc" -target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -@calleeAddrs = global [2 x ptr] [ptr @_ZL7callee0v, ptr @_Z7callee1v] +@foo = external local_unnamed_addr global ptr, align 8 +@bar = external local_unnamed_addr global ptr, align 8 -define void @_Z7callee1v() { - ret void +define i32 @main() local_unnamed_addr { +entry: + %0 = load ptr, ptr @foo, align 8 +; ICALL-PROM: br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] + tail call void %0(), !prof !1 + %1 = load ptr, ptr @bar, align 8 +; ICALL-PROM: br i1 %{{[0-9]+}}, label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof [[BRANCH_WEIGHT:![0-9]+]] + tail call void %1(), !prof !2 + ret i32 0 } -define internal void @_ZL7callee0v() { - ret void -} +!1 = !{!"VP", i32 0, i64 1, i64 -6289574019528802036, i64 1} +!2 = !{!"VP", i32 0, i64 1, i64 591260329866125152, i64 1} -define void @_Z11global_funcv() { -entry: - %0 = load ptr, ptr @calleeAddrs - call void %0() - %1 = load ptr, ptr getelementptr inbounds ([2 x ptr], ptr @calleeAddrs, i64 0, i64 1) - call void %1() - ret void -} +; Should not have a VP annotation on new indirect call (check before and after +; branch_weights annotation). +; ICALL-PROM-NOT: !"VP" +; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 0} +; ICALL-PROM-NOT: !"VP" diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 6a71a975fbb12..e6613a90dc7c5 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -1379,7 +1379,7 @@ TEST(SymtabTest, instr_prof_symtab_compression_test) { TEST_P(MaybeSparseInstrProfTest, remapping_test) { Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err); - Writer.addRecord({"file;_Z3barf", 0x567, {5, 6, 7}}, Err); + Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"( type i l @@ -1397,7 +1397,7 @@ TEST_P(MaybeSparseInstrProfTest, remapping_test) { EXPECT_EQ(4u, Counts[3]); } - for (StringRef BarName : {"file;_Z3barf", "file;_Z4quuxf"}) { + for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) { EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts), Succeeded()); ASSERT_EQ(3u, Counts.size()); From 672f1a036a28cf993e7b5b022bd5582a8924ee58 Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Mon, 18 Dec 2023 18:41:05 +0100 Subject: [PATCH 151/884] [mlir][memref] Make `LoadOp::verify` error more clear (#75831) While debugging https://github.com/llvm/llvm-project/issues/71326, the `LoadOp::verify` code and error were very confusing. This PR improves that. This code was a part from the reverted PR https://github.com/llvm/llvm-project/pull/75519. Fixing the `-convert-vector-to-scf` issue is going to take a bit longer and this code was out of scope anyway. Co-authored-by: Benjamin Maxwell --- mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 6 ++++-- mlir/test/Dialect/MemRef/invalid.mlir | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp index 93327a28234ea..a332fe253ba64 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp @@ -1615,8 +1615,10 @@ GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { //===----------------------------------------------------------------------===// LogicalResult LoadOp::verify() { - if (getNumOperands() != 1 + getMemRefType().getRank()) - return emitOpError("incorrect number of indices for load"); + if (static_cast(getIndices().size()) != getMemRefType().getRank()) { + return emitOpError("incorrect number of indices for load, expected ") + << getMemRefType().getRank() << " but got " << getIndices().size(); + } return success(); } diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir index 55b759cbb3ce7..f9b870f77266e 100644 --- a/mlir/test/Dialect/MemRef/invalid.mlir +++ b/mlir/test/Dialect/MemRef/invalid.mlir @@ -896,6 +896,15 @@ func.func @bad_alloc_wrong_symbol_count() { // ----- +func.func @load_invalid_memref_indexes() { + %0 = memref.alloca() : memref<10xi32> + %c0 = arith.constant 0 : index + // expected-error@+1 {{incorrect number of indices for load, expected 1 but got 2}} + %1 = memref.load %0[%c0, %c0] : memref<10xi32> +} + +// ----- + func.func @test_store_zero_results() { ^bb0: %0 = memref.alloc() : memref<1024x64xf32, affine_map<(d0, d1) -> (d0, d1)>, 1> From 96aca7c51701f9b3c5dd8567fcddf29492008e6d Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 18 Dec 2023 09:46:58 -0800 Subject: [PATCH 152/884] [LTO] Improve diagnostics handling when parsing module-level inline assembly (#75726) Non-LTO compiles set the buffer name to "" (`AsmPrinter::addInlineAsmDiagBuffer`) and pass diagnostics to `ClangDiagnosticHandler` (through the `MCContext` handler in `MachineModuleInfoWrapperPass::doInitialization`) to ensure that the exit code is 1 in the presence of errors. In contrast, LTO compiles spuriously succeed even if error messages are printed. ``` % cat a.c void _start() {} asm("unknown instruction"); % clang -c a.c :1:1: error: invalid instruction mnemonic 'unknown' 1 | unknown instruction | ^ 1 error generated. % clang -c -flto a.c; echo $? # -flto=thin is the same error: invalid instruction mnemonic 'unknown' unknown instruction ^~~~~~~ error: invalid instruction mnemonic 'unknown' unknown instruction ^~~~~~~ 0 ``` `CollectAsmSymbols` parses inline assembly and is transitively called by both `ModuleSummaryIndexAnalysis::run` and `WriteBitcodeToFile`, leading to duplicate diagnostics. This patch updates `CollectAsmSymbols` to be similar to non-LTO compiles. ``` % clang -c -flto=thin a.c; echo $? :1:1: error: invalid instruction mnemonic 'unknown' 1 | unknown instruction | ^ 1 errors generated. 1 ``` The `HasErrors` check does not prevent duplicate warnings but assembler warnings are very uncommon. --- clang/lib/CodeGen/CodeGenAction.cpp | 2 ++ clang/test/CodeGen/invalid_global_asm.c | 5 +++++ lld/test/MachO/lto-module-asm-err.ll | 8 +++----- llvm/include/llvm/IR/DiagnosticHandler.h | 1 + llvm/lib/Object/ModuleSymbolTable.cpp | 16 +++++++++++++++- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp index 753a8fd74fa69..4121a3709bc3a 100644 --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -418,6 +418,8 @@ void BackendConsumer::anchor() { } } // namespace clang bool ClangDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) { + if (DI.getSeverity() == DS_Error) + HasErrors = true; BackendCon->DiagnosticHandlerImpl(DI); return true; } diff --git a/clang/test/CodeGen/invalid_global_asm.c b/clang/test/CodeGen/invalid_global_asm.c index 5b7e8b43d752d..d5645f7fc92bf 100644 --- a/clang/test/CodeGen/invalid_global_asm.c +++ b/clang/test/CodeGen/invalid_global_asm.c @@ -1,5 +1,10 @@ // REQUIRES: arm-registered-target // RUN: not %clang_cc1 -emit-obj -triple armv6-unknown-unknown -o %t %s 2>&1 | FileCheck %s +// RUN: not %clang_cc1 -emit-obj -triple armv6-unknown-unknown -flto -o %t %s 2>&1 | FileCheck %s + +/// Test the diagnostic behavior considering the whole system including the driver. +// RUN: not %clang --target=armv6-unknown-unknown -c -flto=thin -o %t %s 2>&1 | FileCheck %s #pragma clang diagnostic ignored "-Wmissing-noreturn" __asm__(".Lfoo: movw r2, #:lower16:.Lbar - .Lfoo"); // CHECK: :1:8: error: instruction requires: armv6t2 +// CHECK-NOT: error: diff --git a/lld/test/MachO/lto-module-asm-err.ll b/lld/test/MachO/lto-module-asm-err.ll index 45cd75967dbe7..45dff241a653b 100644 --- a/lld/test/MachO/lto-module-asm-err.ll +++ b/lld/test/MachO/lto-module-asm-err.ll @@ -3,13 +3,11 @@ ; RUN: not %lld %t.bc -o /dev/null 2>&1 | FileCheck %s --check-prefix=REGULAR ;; For regular LTO, the original module name is lost. -;; TODO Fix the line number -; REGULAR: error: ld-temp.o :3:1: invalid instruction mnemonic 'invalid' +; REGULAR: error: :2:1: invalid instruction mnemonic 'invalid' -; RUN: opt -module-summary %s -o %t.bc -; RUN: not %lld %t.bc -o /dev/null 2>&1 | FileCheck %s --check-prefix=THIN +; RUN: not opt -module-summary %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=THIN -; THIN: error: {{.*}}.bc :2:1: invalid instruction mnemonic 'invalid' +; THIN: error: :2:1: invalid instruction mnemonic 'invalid' target triple = "x86_64-apple-macosx10.15.0" target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" diff --git a/llvm/include/llvm/IR/DiagnosticHandler.h b/llvm/include/llvm/IR/DiagnosticHandler.h index 55e5e5975808d..db7d7444f75f0 100644 --- a/llvm/include/llvm/IR/DiagnosticHandler.h +++ b/llvm/include/llvm/IR/DiagnosticHandler.h @@ -23,6 +23,7 @@ class DiagnosticInfo; /// which remarks are enabled. struct DiagnosticHandler { void *DiagnosticContext = nullptr; + bool HasErrors = false; DiagnosticHandler(void *DiagContext = nullptr) : DiagnosticContext(DiagContext) {} virtual ~DiagnosticHandler() = default; diff --git a/llvm/lib/Object/ModuleSymbolTable.cpp b/llvm/lib/Object/ModuleSymbolTable.cpp index ab073e18cb466..07f76688fa43e 100644 --- a/llvm/lib/Object/ModuleSymbolTable.cpp +++ b/llvm/lib/Object/ModuleSymbolTable.cpp @@ -16,6 +16,7 @@ #include "RecordStreamer.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" @@ -68,6 +69,11 @@ void ModuleSymbolTable::addModule(Module *M) { static void initializeRecordStreamer(const Module &M, function_ref Init) { + // This function may be called twice, once for ModuleSummaryIndexAnalysis and + // the other when writing the IR symbol table. If parsing inline assembly has + // caused errors in the first run, suppress the second run. + if (M.getContext().getDiagHandlerPtr()->HasErrors) + return; StringRef InlineAsm = M.getModuleInlineAsm(); if (InlineAsm.empty()) return; @@ -95,7 +101,8 @@ initializeRecordStreamer(const Module &M, if (!MCII) return; - std::unique_ptr Buffer(MemoryBuffer::getMemBuffer(InlineAsm)); + std::unique_ptr Buffer( + MemoryBuffer::getMemBuffer(InlineAsm, "")); SourceMgr SrcMgr; SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc()); @@ -115,6 +122,13 @@ initializeRecordStreamer(const Module &M, if (!TAP) return; + MCCtx.setDiagnosticHandler([&](const SMDiagnostic &SMD, bool IsInlineAsm, + const SourceMgr &SrcMgr, + std::vector &LocInfos) { + M.getContext().diagnose( + DiagnosticInfoSrcMgr(SMD, M.getName(), IsInlineAsm, /*LocCookie=*/0)); + }); + // Module-level inline asm is assumed to use At&t syntax (see // AsmPrinter::doInitialization()). Parser->setAssemblerDialect(InlineAsm::AD_ATT); From 644e6d7d8be29b170d3fabe8e388ae5478b4adcf Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 18 Dec 2023 09:53:44 -0800 Subject: [PATCH 153/884] [llvm-objdump] --disassemble-symbols: skip inline relocs from symbols that are not dumped (#75724) When a section contains two functions x1 and x2, we incorrectly display x1's relocations when dumping x2 for `--disassemble-symbols=x2 -r`. Fix #75539 by ignoring these relocations. --- .../llvm-objdump/X86/disassemble-zeroes-relocations.test | 5 +++++ .../llvm-objdump/X86/elf-disassemble-relocs-exec.test | 3 --- .../tools/llvm-objdump/X86/elf-disassemble-relocs.test | 6 +----- llvm/tools/llvm-objdump/llvm-objdump.cpp | 9 ++++++++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test index 62776f8c5a990..6f268247e8ebb 100644 --- a/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test +++ b/llvm/test/tools/llvm-objdump/X86/disassemble-zeroes-relocations.test @@ -35,6 +35,11 @@ # CHECK2-EMPTY: # CHECK2-NEXT: 0000000000000037 : # CHECK2-NEXT: ... +# CHECK2-NEXT: 3f: 00 00 addb %al, (%rax) +# CHECK2-NEXT: 000000000000003f: R_X86_64_64 x3 +# CHECK2-NEXT: 41: 00 00 addb %al, (%rax) +# CHECK2-NEXT: 43: 00 00 addb %al, (%rax) +# CHECK2-NEXT: 45: 00 00 addb %al, (%rax) # CHECK2-NOT: {{.}} ## Check that without -reloc all zeroes would be omitted. diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test index 6c3c0bd1c9803..d9f486ce28a77 100644 --- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test +++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs-exec.test @@ -22,11 +22,8 @@ # CHECK-NEXT: 000000000040000d: R_X86_64_32 .rodata # CHECK-NOT: {{.}} -## FIXME: --disassemble-symbols: remove inline relocs from skipped functions # CHECK2:000000000040000c : #CHECK2-NEXT: 40000c: bf 10 00 40 00 movl $4194320, %edi # imm = 0x400010 -#CHECK2-NEXT: 0000000000400002: R_X86_64_32 .rodata -#CHECK2-NEXT: 0000000000400007: R_X86_64_PLT32 puts-0x4 #CHECK2-NEXT: 000000000040000d: R_X86_64_32 .rodata #CHECK2-NOT: {{.}} diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test index 33438cbfa126e..cce0712e8fa0d 100644 --- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test +++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test @@ -28,19 +28,15 @@ # CHECK-NEXT: 000000000000001a: R_X86_64_REX_GOTPCRELX var-0x4 # CHECK-NOT: {{.}} -## FIXME: --disassemble-symbols: remove inline relocs from skipped functions # CHECK2: 000000000000000a : # CHECK2-NEXT: a: 90 nop -# CHECK2-NEXT: 0000000000000001: R_X86_64_PC32 foo-0x4 -# CHECK2-NEXT: 0000000000000002: R_X86_64_NONE bar+0x8 -# CHECK2-NEXT: 0000000000000006: R_X86_64_PLT32 foo+0x1 # CHECK2-NEXT: b: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x12 # CHECK2-NEXT: 000000000000000e: R_X86_64_REX_GOTPCRELX var-0x4 # CHECK2-EMPTY: # CHECK2-NEXT: 0000000000000017 : # CHECK2-NEXT: 17: 48 8b 05 00 00 00 00 movq (%rip), %rax # 0x1e -# CHECK2-NEXT: 0000000000000013: R_X86_64_PLT32 foo-0x4 # CHECK2-NEXT: 000000000000001a: R_X86_64_REX_GOTPCRELX var-0x4 +# CHECK2-NOT: {{.}} .globl x1, x2, x3, x4 x1: diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index 12bb70d5537d8..1cdd84b20970f 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -1948,6 +1948,13 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, continue; } + // Skip relocations from symbols that are not dumped. + for (; RelCur != RelEnd; ++RelCur) { + uint64_t Offset = RelCur->getOffset() - RelAdjustment; + if (Index <= Offset) + break; + } + bool DumpARMELFData = false; bool DumpTracebackTableForXCOFFFunction = Obj.isXCOFF() && Section.isText() && TracebackTable && @@ -2214,7 +2221,7 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, while (RelCur != RelEnd) { uint64_t Offset = RelCur->getOffset() - RelAdjustment; // If this relocation is hidden, skip it. - if (getHidden(*RelCur) || SectionAddr + Offset < StartAddress) { + if (getHidden(*RelCur)) { ++RelCur; continue; } From b26ee9753777bdb6430e830397d0d6532597a0da Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:02:04 -0600 Subject: [PATCH 154/884] [MLIR][Linalg] Support dynamic sizes in `lower_unpack` (#75494) --- .../Dialect/Linalg/Transforms/Transforms.cpp | 18 +-- .../Dialect/Linalg/transform-lower-pack.mlir | 123 ++++++++++++++++++ 2 files changed, 132 insertions(+), 9 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp index 10dfbe6cec781..9d230e2c2e574 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @@ -380,17 +380,11 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, if (!unPackOp.getOuterDimsPerm().empty()) return rewriter.notifyMatchFailure(unPackOp, "outer dims perm NYI"); - RankedTensorType packedTensorType = unPackOp.getSourceType(); - if (!packedTensorType.hasStaticShape()) { - return rewriter.notifyMatchFailure( - unPackOp, - "non-static shape NYI, needs a more powerful tensor.expand_shape op"); - } - Location loc = unPackOp->getLoc(); OpBuilder::InsertionGuard g(rewriter); rewriter.setInsertionPoint(unPackOp); + RankedTensorType packedTensorType = unPackOp.getSourceType(); int64_t packedRank = packedTensorType.getRank(); OpFoldResult zero = rewriter.getIndexAttr(0), one = rewriter.getIndexAttr(1); @@ -434,8 +428,14 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape); RankedTensorType collapsedType = tensor::CollapseShapeOp::inferCollapsedType( stripMinedTensorType, packingMetadata.reassociations); - auto emptyOp = - rewriter.create(loc, stripMinedTensorType, ValueRange{}); + + // Get dynamic dims from input tensor based on lastDimsToInsertPositionsPerm + // permutation. + SmallVector dims = + tensor::getMixedSizes(rewriter, loc, unPackOp.getSource()); + applyPermutationToVector(dims, lastDimsToInsertPositionsPerm); + auto emptyOp = rewriter.create( + loc, dims, stripMinedTensorType.getElementType()); auto transposeOp = rewriter.create( loc, unPackOp.getSource(), emptyOp, lastDimsToInsertPositionsPerm); diff --git a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir index b9706eed54b60..316df431a9c0c 100644 --- a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir +++ b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir @@ -464,6 +464,129 @@ module attributes {transform.with_named_sequence} { // ----- +// Check that we can lower unpack with dynamic dimensions in the input and destination. +// CHECK-LABEL: func.func @unpack_with_dynamic_input_dest( +// CHECK-SAME: %[[ARG0:.*]]: tensor, %[[ARG1:.*]]: tensor) +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[DIM00:.*]] = tensor.dim %[[ARG0]], %[[C0]] +// CHECK-DAG: %[[DIM01:.*]] = tensor.dim %[[ARG0]], %[[C1]] +// CHECK: %[[EMPTY:.*]] = tensor.empty(%[[DIM00]], %[[DIM01]]) : tensor +// CHECK: %[[TRAN:.*]] = linalg.transpose +// CHECK-SAME: ins(%[[ARG0]] : tensor) +// CHECK-SAME: outs(%[[EMPTY]] : tensor) +// CHECK-SAME: permutation = [0, 2, 1, 3] +// CHECK: %[[CLP:.*]] = tensor.collapse_shape %[[TRAN]] {{\[}}[0, 1], [2, 3]] +// CHECK-SAME: : tensor into tensor +// CHECK: %[[DIM10:.*]] = tensor.dim %[[ARG1]], %[[C0]] : tensor +// CHECK: %[[DIM11:.*]] = tensor.dim %[[ARG1]], %[[C1]] : tensor +// CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[CLP]][0, 0] [%[[DIM10]], %[[DIM11]]] [1, 1] +// CHECK-SAME: : tensor to tensor +// CHECK: linalg.copy ins(%[[SLICE]] : tensor) +// CHECK-SAME: outs(%[[ARG1]] : tensor) +func.func @unpack_with_dynamic_input_dest(%arg0: tensor, %arg1: tensor) -> tensor { + %unpack = tensor.unpack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 16] into %arg1 : tensor -> tensor + return %unpack : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { + %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op + : (!transform.any_op) -> !transform.op<"tensor.unpack"> + transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">) + -> (!transform.op<"tensor.empty">, + !transform.op<"linalg.transpose">, + !transform.op<"tensor.collapse_shape">, + !transform.op<"tensor.extract_slice">) + transform.yield + } +} + +// ----- + +// Check that we can lower unpack with dynamic dimensions in the input, destination, inner_tiles. +// CHECK-LABEL: func.func @unpack_fully_dynamic( +// CHECK-SAME: %[[ARG0:.*]]: tensor, %[[ARG1:.*]]: tensor, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index) +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[DIM00:.*]] = tensor.dim %[[ARG0]], %[[C0]] +// CHECK-DAG: %[[DIM01:.*]] = tensor.dim %[[ARG0]], %[[C1]] +// CHECK-DAG: %[[DIM02:.*]] = tensor.dim %[[ARG0]], %[[C2]] +// CHECK-DAG: %[[DIM03:.*]] = tensor.dim %[[ARG0]], %[[C3]] +// CHECK: %[[EMPTY:.*]] = tensor.empty(%[[DIM00]], %[[DIM02]], %[[DIM01]], %[[DIM03]]) : tensor +// CHECK: %[[TRAN:.*]] = linalg.transpose +// CHECK-SAME: ins(%[[ARG0]] : tensor) +// CHECK-SAME: outs(%[[EMPTY]] : tensor) +// CHECK-SAME: permutation = [0, 2, 1, 3] +// CHECK: %[[CLP:.*]] = tensor.collapse_shape %[[TRAN]] {{\[}}[0, 1], [2, 3]] +// CHECK-SAME: : tensor into tensor +// CHECK: %[[DIM10:.*]] = tensor.dim %[[ARG1]], %[[C0]] : tensor +// CHECK: %[[DIM11:.*]] = tensor.dim %[[ARG1]], %[[C1]] : tensor +// CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[CLP]][0, 0] [%[[DIM10]], %[[DIM11]]] [1, 1] +// CHECK-SAME: : tensor to tensor +// CHECK: linalg.copy ins(%[[SLICE]] : tensor) +// CHECK-SAME: outs(%[[ARG1]] : tensor) +func.func @unpack_fully_dynamic(%source: tensor, %dest: tensor, %tile_n : index, %tile_m : index) -> tensor { + %0 = tensor.unpack %source inner_dims_pos = [0, 1] inner_tiles = [%tile_n, %tile_m] into %dest : tensor -> tensor + return %0 : tensor +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { + %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op + : (!transform.any_op) -> !transform.op<"tensor.unpack"> + transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">) + -> (!transform.op<"tensor.empty">, + !transform.op<"linalg.transpose">, + !transform.op<"tensor.collapse_shape">, + !transform.op<"tensor.extract_slice">) + transform.yield + } +} + +// ----- + +// Check that we can lower unpack "as unpad" with dynamic dims. +// CHECK-LABEL: func.func @unpack_as_pad_dynamic( +// CHECK-SAME: %[[ARG0:.*]]: tensor<1x1x1x1x?x?x?x?xf32>, %[[ARG1:.*]]: tensor +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[DIM0:.*]] = tensor.dim %[[ARG1]], %[[C0]] +// CHECK-DAG: %[[DIM1:.*]] = tensor.dim %[[ARG1]], %[[C1]] +// CHECK-DAG: %[[DIM2:.*]] = tensor.dim %[[ARG1]], %[[C2]] +// CHECK-DAG: %[[DIM3:.*]] = tensor.dim %[[ARG1]], %[[C3]] +// CHECK: %[[RES:.*]] = tensor.extract_slice %[[ARG0]] +// offsets. +// CHECK-SAME: [0, 0, 0, 0, 0, 0, 0, 0] +// sizes. +// CHECK-SAME: [1, 1, 1, 1, %[[DIM0]], %[[DIM1]], %[[DIM2]], %[[DIM3]]] +// strides multiplers. +// CHECK-SAME: [1, 1, 1, 1, 1, 1, 1, 1] +// CHECK-SAME: : tensor<1x1x1x1x?x?x?x?xf32> to tensor +func.func @unpack_as_pad_dynamic(%arg0: tensor<1x1x1x1x?x?x?x?xf32>, %arg1: tensor) -> tensor { + %pack = tensor.unpack %arg0 inner_dims_pos = [0, 1, 2, 3] inner_tiles = [136, 64, 16, 16] into %arg1 + : tensor<1x1x1x1x?x?x?x?xf32> -> tensor + return %pack : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { + %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op + : (!transform.any_op) -> !transform.op<"tensor.unpack"> + transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">) + -> (!transform.op<"tensor.empty">, + !transform.op<"linalg.transpose">, + !transform.op<"tensor.collapse_shape">, + !transform.op<"tensor.extract_slice">) + transform.yield + } +} + +// ----- + // At the moment, we cannot lower tensor.unpack with outer_dims_perm. func.func @diagnostic_unpack(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> { // expected-note @below {{target payload op}} From 791c5d0eb81e0078571ff6694dd23726b9471048 Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:07:05 -0500 Subject: [PATCH 155/884] [libc] Improve get_object_files_for_test to reduce CMake configure time for tests. (#75552) Profiling cmake shows that a significant time configuring `libc` folder is spent on running `get_object_files_for_test` in the `test` folder (13 sec in `libc/test` folder / 16 sec in `libc` folder). By caching all needed objects for each target instead of resolving every time, the time cmake spends on configuring `libc/test` folder is reduced to ~1s. --- libc/cmake/modules/LLVMLibCTestRules.cmake | 79 ++++++++++++++-------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake index 6cae0859149d5..51d484b875aef 100644 --- a/libc/cmake/modules/LLVMLibCTestRules.cmake +++ b/libc/cmake/modules/LLVMLibCTestRules.cmake @@ -18,62 +18,81 @@ function(get_object_files_for_test result skipped_entrypoints_list) set(checked_list "") set(unchecked_list "${ARGN}") list(REMOVE_DUPLICATES unchecked_list) - list(LENGTH unchecked_list length) - while(length) - set(indirect_list "") + foreach(dep IN LISTS unchecked_list) + if (NOT TARGET ${dep}) + # Skip tests with undefined dependencies. + list(APPEND skipped_list ${dep}) + continue() + endif() + get_target_property(aliased_target ${dep} "ALIASED_TARGET") + if(aliased_target) + # If the target is just an alias, switch to the real target. + set(dep ${aliased_target}) + endif() - foreach(dep IN LISTS unchecked_list) - if (NOT TARGET ${dep}) - # Skip tests with undefined dependencies. - list(APPEND skipped_list ${dep}) - continue() - endif() - get_target_property(dep_type ${dep} "TARGET_TYPE") - if(NOT dep_type) - # Skip tests with no object dependencies. - continue() - endif() + get_target_property(dep_type ${dep} "TARGET_TYPE") + if(NOT dep_type) + # Skip tests with no object dependencies. + continue() + endif() + + get_target_property(dep_checked ${dep} "CHECK_OBJ_FOR_TESTS") + + if(dep_checked) + # Target full dependency has already been checked. Just use the results. + get_target_property(dep_obj ${dep} "OBJECT_FILES_FOR_TESTS") + get_target_property(dep_skip ${dep} "SKIPPED_LIST_FOR_TESTS") + else() + # Target full dependency hasn't been checked. Recursively check its DEPS. + set(dep_obj "${dep}") + set(dep_skip "") + + get_target_property(indirect_deps ${dep} "DEPS") + get_object_files_for_test(dep_obj dep_skip ${indirect_deps}) if(${dep_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE}) get_target_property(dep_object_files ${dep} "OBJECT_FILES") if(dep_object_files) - list(APPEND object_files ${dep_object_files}) + list(APPEND dep_obj ${dep_object_files}) endif() elseif(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}) get_target_property(is_skipped ${dep} "SKIPPED") if(is_skipped) - list(APPEND skipped_list ${dep}) - continue() + list(APPEND dep_skip ${dep}) + list(REMOVE_ITEM dep_obj ${dep}) endif() get_target_property(object_file_raw ${dep} "OBJECT_FILE_RAW") if(object_file_raw) - list(APPEND object_files ${object_file_raw}) + list(APPEND dep_obj ${object_file_raw}) endif() elseif(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE}) # Skip tests for externally implemented entrypoints. - list(APPEND skipped_list ${dep}) - continue() + list(APPEND dep_skip ${dep}) + list(REMOVE_ITEM dep_obj ${dep}) endif() - get_target_property(indirect_deps ${dep} "DEPS") - list(APPEND indirect_list "${indirect_deps}") - endforeach(dep) + set_target_properties(${dep} PROPERTIES + OBJECT_FILES_FOR_TESTS "${dep_obj}" + SKIPPED_LIST_FOR_TESTS "${dep_skip}" + CHECK_OBJ_FOR_TESTS "YES" + ) + + endif() + + list(APPEND object_files ${dep_obj}) + list(APPEND skipped_list ${dep_skip}) - # Only add new indirect dependencies to check. - list(APPEND checked_list "${unchecked_list}") - list(REMOVE_DUPLICATES indirect_list) - list(REMOVE_ITEM indirect_list checked_list) - set(unchecked_list "${indirect_list}") - list(LENGTH unchecked_list length) - endwhile() + endforeach(dep) list(REMOVE_DUPLICATES object_files) set(${result} ${object_files} PARENT_SCOPE) list(REMOVE_DUPLICATES skipped_list) set(${skipped_entrypoints_list} ${skipped_list} PARENT_SCOPE) + endfunction(get_object_files_for_test) + # Rule to add a libc unittest. # Usage # add_libc_unittest( From 363f708fcbf92299f4defa8057b8d38a61f168d0 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Mon, 18 Dec 2023 10:22:25 -0800 Subject: [PATCH 156/884] [gn build] Manually port 945c645a and a0a3c793 --- .../secondary/clang/include/clang/Basic/BUILD.gn | 15 +++++++++++++++ llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn | 3 +++ 2 files changed, 18 insertions(+) diff --git a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn index 0061440771e1d..cf4149f8e6ff2 100644 --- a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn @@ -142,6 +142,11 @@ clang_tablegen("arm_sve_sema_rangechecks") { td_file = "arm_sve.td" } +clang_tablegen("arm_sve_streaming_attrs") { + args = [ "-gen-arm-sve-streaming-attrs" ] + td_file = "arm_sve.td" +} + clang_tablegen("arm_sme_builtins") { args = [ "-gen-arm-sme-builtins" ] td_file = "arm_sme.td" @@ -157,6 +162,16 @@ clang_tablegen("arm_sme_sema_rangechecks") { td_file = "arm_sme.td" } +clang_tablegen("arm_sme_streaming_attrs") { + args = [ "-gen-arm-sme-streaming-attrs" ] + td_file = "arm_sme.td" +} + +clang_tablegen("arm_sme_builtins_za_state") { + args = [ "-gen-arm-sme-builtin-za-state" ] + td_file = "arm_sme.td" +} + clang_tablegen("arm_cde_builtins") { args = [ "-gen-arm-cde-builtin-def" ] td_file = "arm_cde.td" diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn index 272995e3c0ba2..27ed94468f171 100644 --- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn @@ -13,8 +13,11 @@ static_library("Sema") { "//clang/include/clang/Basic:arm_cde_builtin_sema", "//clang/include/clang/Basic:arm_mve_builtin_aliases", "//clang/include/clang/Basic:arm_mve_builtin_sema", + "//clang/include/clang/Basic:arm_sme_builtins_za_state", "//clang/include/clang/Basic:arm_sme_sema_rangechecks", + "//clang/include/clang/Basic:arm_sme_streaming_attrs", "//clang/include/clang/Basic:arm_sve_sema_rangechecks", + "//clang/include/clang/Basic:arm_sve_streaming_attrs", "//clang/include/clang/Basic:riscv_sifive_vector_builtin_sema", "//clang/include/clang/Basic:riscv_vector_builtin_sema", "//clang/include/clang/Sema:AttrParsedAttrImpl", From 4123b95949727cb171bfcac6fcec89ea8cb76ef2 Mon Sep 17 00:00:00 2001 From: Ziqing Luo Date: Mon, 18 Dec 2023 10:39:15 -0800 Subject: [PATCH 157/884] [-Wunsafe-buffer-usage] Add a subgroup `-Wunsafe-buffer-usage-in-container` (#75665) Add a sub diagnostic group under `-Wunsafe-buffer-usage` controlled by `-Wunsafe-buffer-usage-in-container`. The subgroup will include warnings on misuses of `std::span`, `std::vector`, and `std::array`. --- clang/include/clang/Basic/DiagnosticGroups.td | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 80b5680b94f6c..7cf347e92d997 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -1488,4 +1488,5 @@ def DXILValidation : DiagGroup<"dxil-validation">; def ReadOnlyPlacementChecks : DiagGroup<"read-only-types">; // Warnings and fixes to support the "safe buffers" programming model. -def UnsafeBufferUsage : DiagGroup<"unsafe-buffer-usage">; +def UnsafeBufferUsageInContainer : DiagGroup<"unsafe-buffer-usage-in-container">; +def UnsafeBufferUsage : DiagGroup<"unsafe-buffer-usage", [UnsafeBufferUsageInContainer]>; From 4f54d71501815877898aaa6d764b8468d0708ed6 Mon Sep 17 00:00:00 2001 From: Justin Bogner Date: Mon, 18 Dec 2023 11:43:52 -0700 Subject: [PATCH 158/884] [HLSL][DirectX] Move handling of resource element types into the frontend Rather than shepherding a type name all the way to the backend as a string and attempting to parse it, get the element type out of the AST and store that in the resource annotation metadata directly. Pull Request: https://github.com/llvm/llvm-project/pull/75674 --- clang/lib/CodeGen/CGHLSLRuntime.cpp | 63 +++++++++++-- clang/lib/CodeGen/CGHLSLRuntime.h | 2 +- .../builtins/RWBuffer-annotations.hlsl | 14 +-- .../builtins/RWBuffer-elementtype.hlsl | 52 +++++++++++ .../RasterizerOrderedBuffer-annotations.hlsl | 12 +-- clang/test/CodeGenHLSL/cbuf.hlsl | 4 +- .../include/llvm/Frontend/HLSL/HLSLResource.h | 27 +++++- llvm/lib/Frontend/HLSL/HLSLResource.cpp | 17 ++-- llvm/lib/Target/DirectX/DXILResource.cpp | 92 ++++++------------- llvm/lib/Target/DirectX/DXILResource.h | 37 ++------ llvm/test/CodeGen/DirectX/UAVMetadata.ll | 22 ++--- 11 files changed, 204 insertions(+), 138 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp index 3e8a40e7540be..e887d35198b3c 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -182,10 +182,8 @@ void CGHLSLRuntime::finishCodeGen() { llvm::hlsl::ResourceKind RK = Buf.IsCBuffer ? llvm::hlsl::ResourceKind::CBuffer : llvm::hlsl::ResourceKind::TBuffer; - std::string TyName = - Buf.Name.str() + (Buf.IsCBuffer ? ".cb." : ".tb.") + "ty"; - addBufferResourceAnnotation(GV, TyName, RC, RK, /*IsROV=*/false, - Buf.Binding); + addBufferResourceAnnotation(GV, RC, RK, /*IsROV=*/false, + llvm::hlsl::ElementType::Invalid, Buf.Binding); } } @@ -194,10 +192,10 @@ CGHLSLRuntime::Buffer::Buffer(const HLSLBufferDecl *D) Binding(D->getAttr()) {} void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, - llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, bool IsROV, + llvm::hlsl::ElementType ET, BufferResBinding &Binding) { llvm::Module &M = CGM.getModule(); @@ -216,15 +214,62 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, assert(false && "Unsupported buffer type!"); return; } - assert(ResourceMD != nullptr && "ResourceMD must have been set by the switch above."); llvm::hlsl::FrontendResource Res( - GV, TyName, RK, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); + GV, RK, ET, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); ResourceMD->addOperand(Res.getMetadata()); } +static llvm::hlsl::ElementType +calculateElementType(const ASTContext &Context, const clang::Type *ResourceTy) { + using llvm::hlsl::ElementType; + + // TODO: We may need to update this when we add things like ByteAddressBuffer + // that don't have a template parameter (or, indeed, an element type). + const auto *TST = ResourceTy->getAs(); + assert(TST && "Resource types must be template specializations"); + ArrayRef Args = TST->template_arguments(); + assert(!Args.empty() && "Resource has no element type"); + + // At this point we have a resource with an element type, so we can assume + // that it's valid or we would have diagnosed the error earlier. + QualType ElTy = Args[0].getAsType(); + + // We should either have a basic type or a vector of a basic type. + if (const auto *VecTy = ElTy->getAs()) + ElTy = VecTy->getElementType(); + + if (ElTy->isSignedIntegerType()) { + switch (Context.getTypeSize(ElTy)) { + case 16: + return ElementType::I16; + case 32: + return ElementType::I32; + case 64: + return ElementType::I64; + } + } else if (ElTy->isUnsignedIntegerType()) { + switch (Context.getTypeSize(ElTy)) { + case 16: + return ElementType::U16; + case 32: + return ElementType::U32; + case 64: + return ElementType::U64; + } + } else if (ElTy->isSpecificBuiltinType(BuiltinType::Half)) + return ElementType::F16; + else if (ElTy->isSpecificBuiltinType(BuiltinType::Float)) + return ElementType::F32; + else if (ElTy->isSpecificBuiltinType(BuiltinType::Double)) + return ElementType::F64; + + // TODO: We need to handle unorm/snorm float types here once we support them + llvm_unreachable("Invalid element type for resource"); +} + void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { const Type *Ty = D->getType()->getPointeeOrArrayElementType(); if (!Ty) @@ -239,10 +284,10 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { llvm::hlsl::ResourceClass RC = Attr->getResourceClass(); llvm::hlsl::ResourceKind RK = Attr->getResourceKind(); bool IsROV = Attr->getIsROV(); + llvm::hlsl::ElementType ET = calculateElementType(CGM.getContext(), Ty); - QualType QT(Ty, 0); BufferResBinding Binding(D->getAttr()); - addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, IsROV, Binding); + addBufferResourceAnnotation(GV, RC, RK, IsROV, ET, Binding); } CGHLSLRuntime::BufferResBinding::BufferResBinding( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index bb500cb5c979f..bffefb66740a0 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -90,9 +90,9 @@ class CGHLSLRuntime { private: void addBufferResourceAnnotation(llvm::GlobalVariable *GV, - llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, bool IsROV, + llvm::hlsl::ElementType ET, BufferResBinding &Binding); void addConstant(VarDecl *D, Buffer &CB); void addBufferDecls(const DeclContext *DC, Buffer &CB); diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl index a70e224b81e4b..7ca78e60fb9c5 100644 --- a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s RWBuffer Buffer1; RWBuffer > BufferArray[4]; @@ -16,9 +16,9 @@ void main() { } // CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} -// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 -1, i32 0} -// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 -1, i32 0} -// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 0} -// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 0} -// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 1} -// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 1} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", i32 10, i32 9, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", i32 10, i32 9, i1 false, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 false, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", i32 10, i32 9, i1 false, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 false, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl new file mode 100644 index 0000000000000..87002ccd462d3 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl @@ -0,0 +1,52 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s + +RWBuffer BufI16; +RWBuffer BufU16; +RWBuffer BufI32; +RWBuffer BufU32; +RWBuffer BufI64; +RWBuffer BufU64; +RWBuffer BufF16; +RWBuffer BufF32; +RWBuffer BufF64; +RWBuffer< vector > BufI16x4; +RWBuffer< vector > BufU32x3; +RWBuffer BufF16x2; +RWBuffer BufF32x3; +// TODO: RWBuffer BufSNormF16; -> 11 +// TODO: RWBuffer BufUNormF16; -> 12 +// TODO: RWBuffer BufSNormF32; -> 13 +// TODO: RWBuffer BufUNormF32; -> 14 +// TODO: RWBuffer BufSNormF64; -> 15 +// TODO: RWBuffer BufUNormF64; -> 16 + +[numthreads(1,1,1)] +void main(int GI : SV_GroupIndex) { + BufI16[GI] = 0; + BufU16[GI] = 0; + BufI32[GI] = 0; + BufU32[GI] = 0; + BufI64[GI] = 0; + BufU64[GI] = 0; + BufF16[GI] = 0; + BufF32[GI] = 0; + BufF64[GI] = 0; + BufI16x4[GI] = 0; + BufU32x3[GI] = 0; + BufF16x2[GI] = 0; + BufF32x3[GI] = 0; +} + +// CHECK: !{{[0-9]+}} = !{ptr @"?BufI16@@3V?$RWBuffer@F@hlsl@@A", i32 10, i32 2, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufU16@@3V?$RWBuffer@G@hlsl@@A", i32 10, i32 3, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufI32@@3V?$RWBuffer@H@hlsl@@A", i32 10, i32 4, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufU32@@3V?$RWBuffer@I@hlsl@@A", i32 10, i32 5, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufI64@@3V?$RWBuffer@J@hlsl@@A", i32 10, i32 6, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufU64@@3V?$RWBuffer@K@hlsl@@A", i32 10, i32 7, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufF16@@3V?$RWBuffer@$f16@@hlsl@@A", i32 10, i32 8, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufF32@@3V?$RWBuffer@M@hlsl@@A", i32 10, i32 9, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufF64@@3V?$RWBuffer@N@hlsl@@A", i32 10, i32 10, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufI16x4@@3V?$RWBuffer@T?$__vector@F$03@__clang@@@hlsl@@A", i32 10, i32 2, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufU32x3@@3V?$RWBuffer@T?$__vector@I$02@__clang@@@hlsl@@A", i32 10, i32 5, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufF16x2@@3V?$RWBuffer@T?$__vector@$f16@$01@__clang@@@hlsl@@A", i32 10, i32 8, +// CHECK: !{{[0-9]+}} = !{ptr @"?BufF32x3@@3V?$RWBuffer@T?$__vector@M$02@__clang@@@hlsl@@A", i32 10, i32 9, diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl index ce7d84ecf5b14..bf70cc2456c8b 100644 --- a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl @@ -12,9 +12,9 @@ RasterizerOrderedBuffer > BufferArray3[4] : register(u4, space1 void main() {} // CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} -// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 -1, i32 0} -// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 -1, i32 0} -// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 0} -// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 0} -// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 1} -// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 1} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", i32 10, i32 9, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", i32 10, i32 9, i1 true, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 true, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", i32 10, i32 9, i1 true, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", i32 10, i32 9, i1 true, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/cbuf.hlsl b/clang/test/CodeGenHLSL/cbuf.hlsl index 5dee1feb902aa..dc2a6aaa8f433 100644 --- a/clang/test/CodeGenHLSL/cbuf.hlsl +++ b/clang/test/CodeGenHLSL/cbuf.hlsl @@ -24,5 +24,5 @@ float foo() { // CHECK: !hlsl.cbufs = !{![[CBMD:[0-9]+]]} // CHECK: !hlsl.srvs = !{![[TBMD:[0-9]+]]} -// CHECK: ![[CBMD]] = !{ptr @[[CB]], !"A.cb.ty", i32 13, i1 false, i32 0, i32 2} -// CHECK: ![[TBMD]] = !{ptr @[[TB]], !"A.tb.ty", i32 15, i1 false, i32 2, i32 1} +// CHECK: ![[CBMD]] = !{ptr @[[CB]], i32 13, i32 0, i1 false, i32 0, i32 2} +// CHECK: ![[TBMD]] = !{ptr @[[TB]], i32 15, i32 0, i1 false, i32 2, i32 1} diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h index eedecaea4e58d..068b4c66711e4 100644 --- a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h +++ b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h @@ -54,6 +54,30 @@ enum class ResourceKind : uint32_t { NumEntries, }; +// The value ordering of this enumeration is part of the DXIL ABI. Elements +// can only be added to the end, and not removed. +enum class ElementType : uint32_t { + Invalid = 0, + I1, + I16, + U16, + I32, + U32, + I64, + U64, + F16, + F32, + F64, + SNormF16, + UNormF16, + SNormF32, + UNormF32, + SNormF64, + UNormF64, + PackedS8x32, + PackedU8x32, +}; + class FrontendResource { MDNode *Entry; @@ -62,12 +86,13 @@ class FrontendResource { assert(Entry->getNumOperands() == 6 && "Unexpected metadata shape"); } - FrontendResource(GlobalVariable *GV, StringRef TypeStr, ResourceKind RK, + FrontendResource(GlobalVariable *GV, ResourceKind RK, ElementType ElTy, bool IsROV, uint32_t ResIndex, uint32_t Space); GlobalVariable *getGlobalVariable(); StringRef getSourceType(); ResourceKind getResourceKind(); + ElementType getElementType(); bool getIsROV(); uint32_t getResourceIndex(); uint32_t getSpace(); diff --git a/llvm/lib/Frontend/HLSL/HLSLResource.cpp b/llvm/lib/Frontend/HLSL/HLSLResource.cpp index 709fe3212623e..bcdbe5eadc69e 100644 --- a/llvm/lib/Frontend/HLSL/HLSLResource.cpp +++ b/llvm/lib/Frontend/HLSL/HLSLResource.cpp @@ -23,12 +23,14 @@ GlobalVariable *FrontendResource::getGlobalVariable() { cast(Entry->getOperand(0))->getValue()); } -StringRef FrontendResource::getSourceType() { - return cast(Entry->getOperand(1))->getString(); -} - ResourceKind FrontendResource::getResourceKind() { return static_cast( + cast( + cast(Entry->getOperand(1))->getValue()) + ->getLimitedValue()); +} +ElementType FrontendResource::getElementType() { + return static_cast( cast( cast(Entry->getOperand(2))->getValue()) ->getLimitedValue()); @@ -49,14 +51,15 @@ uint32_t FrontendResource::getSpace() { ->getLimitedValue(); } -FrontendResource::FrontendResource(GlobalVariable *GV, StringRef TypeStr, - ResourceKind RK, bool IsROV, +FrontendResource::FrontendResource(GlobalVariable *GV, ResourceKind RK, + ElementType ElTy, bool IsROV, uint32_t ResIndex, uint32_t Space) { auto &Ctx = GV->getContext(); IRBuilder<> B(Ctx); Entry = MDNode::get( - Ctx, {ValueAsMetadata::get(GV), MDString::get(Ctx, TypeStr), + Ctx, {ValueAsMetadata::get(GV), ConstantAsMetadata::get(B.getInt32(static_cast(RK))), + ConstantAsMetadata::get(B.getInt32(static_cast(ElTy))), ConstantAsMetadata::get(B.getInt1(IsROV)), ConstantAsMetadata::get(B.getInt32(ResIndex)), ConstantAsMetadata::get(B.getInt32(Space))}); diff --git a/llvm/lib/Target/DirectX/DXILResource.cpp b/llvm/lib/Target/DirectX/DXILResource.cpp index d3ff12a1f7b34..621852f2453ff 100644 --- a/llvm/lib/Target/DirectX/DXILResource.cpp +++ b/llvm/lib/Target/DirectX/DXILResource.cpp @@ -63,57 +63,56 @@ ResourceBase::ResourceBase(uint32_t I, FrontendResource R) RangeSize = ArrTy->getNumElements(); } -StringRef ResourceBase::getComponentTypeName(ComponentType CompType) { - switch (CompType) { - case ComponentType::LastEntry: - case ComponentType::Invalid: +StringRef ResourceBase::getElementTypeName(ElementType ElTy) { + switch (ElTy) { + case ElementType::Invalid: return "invalid"; - case ComponentType::I1: + case ElementType::I1: return "i1"; - case ComponentType::I16: + case ElementType::I16: return "i16"; - case ComponentType::U16: + case ElementType::U16: return "u16"; - case ComponentType::I32: + case ElementType::I32: return "i32"; - case ComponentType::U32: + case ElementType::U32: return "u32"; - case ComponentType::I64: + case ElementType::I64: return "i64"; - case ComponentType::U64: + case ElementType::U64: return "u64"; - case ComponentType::F16: + case ElementType::F16: return "f16"; - case ComponentType::F32: + case ElementType::F32: return "f32"; - case ComponentType::F64: + case ElementType::F64: return "f64"; - case ComponentType::SNormF16: + case ElementType::SNormF16: return "snorm_f16"; - case ComponentType::UNormF16: + case ElementType::UNormF16: return "unorm_f16"; - case ComponentType::SNormF32: + case ElementType::SNormF32: return "snorm_f32"; - case ComponentType::UNormF32: + case ElementType::UNormF32: return "unorm_f32"; - case ComponentType::SNormF64: + case ElementType::SNormF64: return "snorm_f64"; - case ComponentType::UNormF64: + case ElementType::UNormF64: return "unorm_f64"; - case ComponentType::PackedS8x32: + case ElementType::PackedS8x32: return "p32i8"; - case ComponentType::PackedU8x32: + case ElementType::PackedU8x32: return "p32u8"; } - llvm_unreachable("All ComponentType enums are handled in switch"); + llvm_unreachable("All ElementType enums are handled in switch"); } -void ResourceBase::printComponentType(Kinds Kind, ComponentType CompType, - unsigned Alignment, raw_ostream &OS) { +void ResourceBase::printElementType(Kinds Kind, ElementType ElTy, + unsigned Alignment, raw_ostream &OS) { switch (Kind) { default: // TODO: add vector size. - OS << right_justify(getComponentTypeName(CompType), Alignment); + OS << right_justify(getElementTypeName(ElTy), Alignment); break; case Kinds::RawBuffer: OS << right_justify("byte", Alignment); @@ -232,19 +231,13 @@ void ResourceBase::print(raw_ostream &OS, StringRef IDPrefix, OS << right_justify("unbounded", 6) << "\n"; } -UAVResource::UAVResource(uint32_t I, FrontendResource R) - : ResourceBase(I, R), Shape(R.getResourceKind()), GloballyCoherent(false), - HasCounter(false), IsROV(R.getIsROV()), ExtProps() { - parseSourceType(R.getSourceType()); -} - void UAVResource::print(raw_ostream &OS) const { OS << "; " << left_justify(Name, 31); OS << right_justify("UAV", 10); - printComponentType( - Shape, ExtProps.ElementType.value_or(ComponentType::Invalid), 8, OS); + printElementType(Shape, ExtProps.ElementType.value_or(ElementType::Invalid), + 8, OS); // FIXME: support SampleCount. // See https://github.com/llvm/llvm-project/issues/58175 @@ -253,35 +246,6 @@ void UAVResource::print(raw_ostream &OS) const { ResourceBase::print(OS, "U", "u"); } -// FIXME: Capture this in HLSL source. I would go do this right now, but I want -// to get this in first so that I can make sure to capture all the extra -// information we need to remove the source type string from here (See issue: -// https://github.com/llvm/llvm-project/issues/57991). -void UAVResource::parseSourceType(StringRef S) { - S = S.substr(S.find("<") + 1); - - constexpr size_t PrefixLen = StringRef("vector<").size(); - if (S.starts_with("vector<")) - S = S.substr(PrefixLen, S.find(",") - PrefixLen); - else - S = S.substr(0, S.find(">")); - - ComponentType ElTy = StringSwitch(S) - .Case("bool", ComponentType::I1) - .Case("int16_t", ComponentType::I16) - .Case("uint16_t", ComponentType::U16) - .Case("int32_t", ComponentType::I32) - .Case("uint32_t", ComponentType::U32) - .Case("int64_t", ComponentType::I64) - .Case("uint64_t", ComponentType::U64) - .Case("half", ComponentType::F16) - .Case("float", ComponentType::F32) - .Case("double", ComponentType::F64) - .Default(ComponentType::Invalid); - if (ElTy != ComponentType::Invalid) - ExtProps.ElementType = ElTy; -} - ConstantBuffer::ConstantBuffer(uint32_t I, hlsl::FrontendResource R) : ResourceBase(I, R) {} @@ -294,7 +258,7 @@ void ConstantBuffer::print(raw_ostream &OS) const { OS << right_justify("cbuffer", 10); - printComponentType(Kinds::CBuffer, ComponentType::Invalid, 8, OS); + printElementType(Kinds::CBuffer, ElementType::Invalid, 8, OS); printKind(Kinds::CBuffer, 12, OS, /*SRV*/ false, /*HasCounter*/ false); // Print the binding part. diff --git a/llvm/lib/Target/DirectX/DXILResource.h b/llvm/lib/Target/DirectX/DXILResource.h index cb39020bc61eb..5f8b0badd145c 100644 --- a/llvm/lib/Target/DirectX/DXILResource.h +++ b/llvm/lib/Target/DirectX/DXILResource.h @@ -46,38 +46,13 @@ class ResourceBase { bool SRV = false, bool HasCounter = false, uint32_t SampleCount = 0); - // The value ordering of this enumeration is part of the DXIL ABI. Elements - // can only be added to the end, and not removed. - enum class ComponentType : uint32_t { - Invalid = 0, - I1, - I16, - U16, - I32, - U32, - I64, - U64, - F16, - F32, - F64, - SNormF16, - UNormF16, - SNormF32, - UNormF32, - SNormF64, - UNormF64, - PackedS8x32, - PackedU8x32, - LastEntry - }; - - static StringRef getComponentTypeName(ComponentType CompType); - static void printComponentType(Kinds Kind, ComponentType CompType, - unsigned Alignment, raw_ostream &OS); + static StringRef getElementTypeName(hlsl::ElementType CompType); + static void printElementType(Kinds Kind, hlsl::ElementType CompType, + unsigned Alignment, raw_ostream &OS); public: struct ExtendedProperties { - std::optional ElementType; + std::optional ElementType; // The value ordering of this enumeration is part of the DXIL ABI. Elements // can only be added to the end, and not removed. @@ -102,7 +77,9 @@ class UAVResource : public ResourceBase { void parseSourceType(StringRef S); public: - UAVResource(uint32_t I, hlsl::FrontendResource R); + UAVResource(uint32_t I, hlsl::FrontendResource R) + : ResourceBase(I, R), Shape(R.getResourceKind()), GloballyCoherent(false), + HasCounter(false), IsROV(R.getIsROV()), ExtProps{R.getElementType()} {} MDNode *write() const; void print(raw_ostream &O) const; diff --git a/llvm/test/CodeGen/DirectX/UAVMetadata.ll b/llvm/test/CodeGen/DirectX/UAVMetadata.ll index 3d95723d6e49f..0bc8a8cfcd713 100644 --- a/llvm/test/CodeGen/DirectX/UAVMetadata.ll +++ b/llvm/test/CodeGen/DirectX/UAVMetadata.ll @@ -37,22 +37,22 @@ target triple = "dxil-pc-shadermodel6.0-library" !hlsl.uavs = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9} -!0 = !{ptr @Zero, !"RWBuffer", i32 10, i1 false, i32 0, i32 0} -!1 = !{ptr @One, !"Buffer>", i32 10, i1 false, i32 1, i32 0} -!2 = !{ptr @Two, !"Buffer", i32 10, i1 false, i32 2, i32 0} -!3 = !{ptr @Three, !"Buffer", i32 10, i1 false, i32 3, i32 0} -!4 = !{ptr @Four, !"ByteAddressBuffer", i32 11, i1 false, i32 5, i32 0} -!5 = !{ptr @Five, !"StructuredBuffer", i32 12, i1 false, i32 6, i32 0} -!6 = !{ptr @Six, !"RasterizerOrderedBuffer", i32 10, i1 true, i32 7, i32 0} -!7 = !{ptr @Seven, !"RasterizerOrderedStructuredBuffer", i32 12, i1 true, i32 8, i32 0} -!8 = !{ptr @Eight, !"RasterizerOrderedByteAddressBuffer", i32 11, i1 true, i32 9, i32 0} -!9 = !{ptr @Nine, !"RWBuffer", i32 10, i1 false, i32 10, i32 2} +!0 = !{ptr @Zero, i32 10, i32 8, i1 false, i32 0, i32 0} +!1 = !{ptr @One, i32 10, i32 9, i1 false, i32 1, i32 0} +!2 = !{ptr @Two, i32 10, i32 10, i1 false, i32 2, i32 0} +!3 = !{ptr @Three, i32 10, i32 1, i1 false, i32 3, i32 0} +!4 = !{ptr @Four, i32 11, i32 2, i1 false, i32 5, i32 0} +!5 = !{ptr @Five, i32 12, i32 3, i1 false, i32 6, i32 0} +!6 = !{ptr @Six, i32 10, i32 4, i1 true, i32 7, i32 0} +!7 = !{ptr @Seven, i32 12, i32 5, i1 true, i32 8, i32 0} +!8 = !{ptr @Eight, i32 11, i32 6, i1 true, i32 9, i32 0} +!9 = !{ptr @Nine, i32 10, i32 7, i1 false, i32 10, i32 2} ; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]} ; CHECK: [[ResList]] = !{null, [[UAVList:[!][0-9]+]], null, null} ; CHECK: [[UAVList]] = !{[[Zero:[!][0-9]+]], [[One:[!][0-9]+]], -; CHECK-SAME: [[Two:[!][0-9]+]], [[Three:[!][0-9]+]], [[Four:[!][0-9]+]], +; CHECK-SAME: [[Two:[!][0-9]+]], [[Three:[!][0-9]+]], [[Four:[!][0-9]+]], ; CHECK-SAME: [[Five:[!][0-9]+]], [[Six:[!][0-9]+]], [[Seven:[!][0-9]+]], ; CHECK-SAME: [[Eight:[!][0-9]+]], [[Nine:[!][0-9]+]]} ; CHECK: [[Zero]] = !{i32 0, ptr @Zero, !"", i32 0, i32 0, i32 1, i32 10, i1 false, i1 false, i1 false, [[Half:[!][0-9]+]]} From 94230ce548c2f69db4cf9ef92ccb51d59030e7e3 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin Date: Mon, 18 Dec 2023 10:58:50 -0800 Subject: [PATCH 159/884] [AMDGPU] Fix lack of LDS DMA check in the AA handling (#75249) SIInstrInfo::areMemAccessesTriviallyDisjoint does a DS offset checks, but does not account for LDS DMA instructions. Added these checks. Without it code falls through and returns true which is wrong. As a result mayAlias would always return false for LDS DMA and a regular LDS instruction or 2 LDS DMA instructions. At the moment this is NFCI because we do not use this AA in a context which may touch LDS DMA instructions. This is also unreacheable now because of the ordered memory ref checks just above in the function and LDS DMA is marked as volatile. This volatile marking is removed in PR #75247, therefore I'd submit this check before #75247. --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 70ef1fff274a4..5d6462f355fab 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -3654,6 +3654,9 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) return false; + if (isLDSDMA(MIa) || isLDSDMA(MIb)) + return false; + // TODO: Should we check the address space from the MachineMemOperand? That // would allow us to distinguish objects we know don't alias based on the // underlying address space, even if it was lowered to a different one, From e5c523e8610492b3256dde6856811b527b4dcb35 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin Date: Mon, 18 Dec 2023 11:01:12 -0800 Subject: [PATCH 160/884] [AMDGPU] Produce better memoperand for LDS DMA (#75247) 1) It was marked as volatile. This is not needed and the only reason it was done is because it is both load and store and handled together with atomics. Global load to LDS was marked as volatile just because buffer load was done that way. 2) Preserve at least LDS (store) pointer which we always have with the intrinsics. 3) Use PoisonValue instead of nullptr for load memop as a Value. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 34826809c1a6b..4f4bc45e49b43 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1145,11 +1145,10 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable; - // XXX - Should this be volatile without known ordering? - Info.flags |= MachineMemOperand::MOVolatile; - switch (IntrID) { default: + // XXX - Should this be volatile without known ordering? + Info.flags |= MachineMemOperand::MOVolatile; break; case Intrinsic::amdgcn_raw_buffer_load_lds: case Intrinsic::amdgcn_raw_ptr_buffer_load_lds: @@ -1157,6 +1156,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: { unsigned Width = cast(CI.getArgOperand(2))->getZExtValue(); Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8); + Info.ptrVal = CI.getArgOperand(1); return true; } } @@ -1289,8 +1289,8 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.opc = ISD::INTRINSIC_VOID; unsigned Width = cast(CI.getArgOperand(2))->getZExtValue(); Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8); - Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore | - MachineMemOperand::MOVolatile; + Info.ptrVal = CI.getArgOperand(1); + Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; return true; } case Intrinsic::amdgcn_ds_bvh_stack_rtn: { @@ -9231,7 +9231,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo(); MachinePointerInfo StorePtrI = LoadPtrI; - StorePtrI.V = nullptr; + LoadPtrI.V = PoisonValue::get( + PointerType::get(*DAG.getContext(), AMDGPUAS::GLOBAL_ADDRESS)); + LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS; StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS; auto F = LoadMMO->getFlags() & @@ -9309,6 +9311,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo(); LoadPtrI.Offset = Op->getConstantOperandVal(5); MachinePointerInfo StorePtrI = LoadPtrI; + LoadPtrI.V = PoisonValue::get( + PointerType::get(*DAG.getContext(), AMDGPUAS::GLOBAL_ADDRESS)); LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS; StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS; auto F = LoadMMO->getFlags() & From 9783f28cbb155e4a8d49c12e1c60ce14dcfaf0c7 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 18 Dec 2023 14:01:33 -0500 Subject: [PATCH 161/884] [libc++] Format the code base (#74334) This patch runs clang-format on all of libcxx/include and libcxx/src, in accordance with the RFC discussed at [1]. Follow-up patches will format the benchmarks, the test suite and remaining parts of the code. I'm splitting this one into its own patch so the diff is a bit easier to review. This patch was generated with: find libcxx/include libcxx/src -type f \ | grep -v 'module.modulemap.in' \ | grep -v 'CMakeLists.txt' \ | grep -v 'README.txt' \ | grep -v 'libcxx.imp' \ | grep -v '__config_site.in' \ | xargs clang-format -i A Git merge driver is available in libcxx/utils/clang-format-merge-driver.sh to help resolve merge and rebase issues across these formatting changes. [1]: https://discourse.llvm.org/t/rfc-clang-formatting-all-of-libc-once-and-for-all --- .gitattributes | 142 + libcxx/include/__algorithm/binary_search.h | 20 +- libcxx/include/__algorithm/comp_ref_type.h | 66 +- libcxx/include/__algorithm/copy_backward.h | 16 +- libcxx/include/__algorithm/copy_if.h | 23 +- libcxx/include/__algorithm/copy_move_common.h | 25 +- libcxx/include/__algorithm/copy_n.h | 59 +- libcxx/include/__algorithm/equal.h | 29 +- libcxx/include/__algorithm/equal_range.h | 11 +- libcxx/include/__algorithm/fill.h | 26 +- libcxx/include/__algorithm/fill_n.h | 20 +- libcxx/include/__algorithm/find_end.h | 92 +- libcxx/include/__algorithm/find_first_of.h | 21 +- libcxx/include/__algorithm/for_each_n.h | 5 +- libcxx/include/__algorithm/generate.h | 10 +- libcxx/include/__algorithm/generate_n.h | 16 +- libcxx/include/__algorithm/half_positive.h | 14 +- libcxx/include/__algorithm/in_found_result.h | 2 +- libcxx/include/__algorithm/in_fun_result.h | 2 +- libcxx/include/__algorithm/in_in_out_result.h | 14 +- libcxx/include/__algorithm/in_in_result.h | 6 +- .../include/__algorithm/in_out_out_result.h | 14 +- libcxx/include/__algorithm/in_out_result.h | 8 +- libcxx/include/__algorithm/includes.h | 31 +- libcxx/include/__algorithm/inplace_merge.h | 304 +- libcxx/include/__algorithm/is_heap.h | 20 +- libcxx/include/__algorithm/is_heap_until.h | 56 +- libcxx/include/__algorithm/is_partitioned.h | 21 +- libcxx/include/__algorithm/is_permutation.h | 177 +- libcxx/include/__algorithm/is_sorted.h | 20 +- libcxx/include/__algorithm/is_sorted_until.h | 33 +- libcxx/include/__algorithm/iter_swap.h | 3 +- .../include/__algorithm/iterator_operations.h | 67 +- .../__algorithm/lexicographical_compare.h | 49 +- libcxx/include/__algorithm/lower_bound.h | 15 +- libcxx/include/__algorithm/make_heap.h | 16 +- libcxx/include/__algorithm/make_projected.h | 47 +- libcxx/include/__algorithm/max.h | 39 +- libcxx/include/__algorithm/max_element.h | 33 +- libcxx/include/__algorithm/merge.h | 65 +- libcxx/include/__algorithm/min.h | 39 +- libcxx/include/__algorithm/min_element.h | 23 +- libcxx/include/__algorithm/min_max_result.h | 2 +- libcxx/include/__algorithm/minmax.h | 48 +- libcxx/include/__algorithm/minmax_element.h | 31 +- libcxx/include/__algorithm/mismatch.h | 25 +- libcxx/include/__algorithm/move.h | 3 +- libcxx/include/__algorithm/move_backward.h | 6 +- libcxx/include/__algorithm/next_permutation.h | 61 +- libcxx/include/__algorithm/nth_element.h | 386 +- libcxx/include/__algorithm/partial_sort.h | 39 +- .../include/__algorithm/partial_sort_copy.h | 94 +- libcxx/include/__algorithm/partition.h | 84 +- libcxx/include/__algorithm/partition_copy.h | 36 +- libcxx/include/__algorithm/partition_point.h | 34 +- libcxx/include/__algorithm/pop_heap.h | 18 +- libcxx/include/__algorithm/prev_permutation.h | 64 +- libcxx/include/__algorithm/push_heap.h | 24 +- libcxx/include/__algorithm/remove.h | 26 +- libcxx/include/__algorithm/remove_copy.h | 20 +- libcxx/include/__algorithm/remove_copy_if.h | 20 +- libcxx/include/__algorithm/remove_if.h | 26 +- libcxx/include/__algorithm/replace.h | 12 +- libcxx/include/__algorithm/replace_copy.h | 23 +- libcxx/include/__algorithm/replace_copy_if.h | 23 +- libcxx/include/__algorithm/replace_if.h | 12 +- libcxx/include/__algorithm/reverse.h | 40 +- libcxx/include/__algorithm/reverse_copy.h | 12 +- libcxx/include/__algorithm/rotate.h | 264 +- libcxx/include/__algorithm/rotate_copy.h | 8 +- libcxx/include/__algorithm/sample.h | 81 +- libcxx/include/__algorithm/search.h | 123 +- libcxx/include/__algorithm/search_n.h | 96 +- libcxx/include/__algorithm/set_difference.h | 2 +- libcxx/include/__algorithm/shift_left.h | 43 +- libcxx/include/__algorithm/shift_right.h | 123 +- libcxx/include/__algorithm/shuffle.h | 127 +- libcxx/include/__algorithm/sift_down.h | 141 +- libcxx/include/__algorithm/sort.h | 245 +- libcxx/include/__algorithm/sort_heap.h | 16 +- libcxx/include/__algorithm/stable_partition.h | 487 +- libcxx/include/__algorithm/stable_sort.h | 332 +- libcxx/include/__algorithm/swap_ranges.h | 9 +- libcxx/include/__algorithm/transform.h | 29 +- .../uniform_random_bit_generator_adaptor.h | 14 +- libcxx/include/__algorithm/unwrap_iter.h | 10 +- libcxx/include/__atomic/atomic.h | 812 +- libcxx/include/__atomic/atomic_base.h | 321 +- libcxx/include/__atomic/atomic_flag.h | 246 +- libcxx/include/__atomic/atomic_lock_free.h | 52 +- libcxx/include/__atomic/atomic_sync.h | 90 +- libcxx/include/__atomic/check_memory_order.h | 22 +- libcxx/include/__atomic/contention_t.h | 4 +- libcxx/include/__atomic/cxx_atomic_impl.h | 560 +- libcxx/include/__atomic/fence.h | 16 +- libcxx/include/__atomic/kill_dependency.h | 6 +- libcxx/include/__atomic/memory_order.h | 11 +- libcxx/include/__availability | 73 +- libcxx/include/__bit/bit_cast.h | 3 +- libcxx/include/__bit/bit_ceil.h | 2 +- libcxx/include/__bit/byteswap.h | 11 +- libcxx/include/__bit/countl.h | 84 +- libcxx/include/__bit/countr.h | 17 +- libcxx/include/__bit/endian.h | 2 +- libcxx/include/__bit/popcount.h | 15 +- libcxx/include/__chrono/calendar.h | 12 +- libcxx/include/__chrono/convert_to_timespec.h | 15 +- libcxx/include/__chrono/day.h | 91 +- libcxx/include/__chrono/duration.h | 693 +- libcxx/include/__chrono/file_clock.h | 24 +- libcxx/include/__chrono/formatter.h | 63 +- libcxx/include/__chrono/hh_mm_ss.h | 117 +- .../include/__chrono/high_resolution_clock.h | 3 +- libcxx/include/__chrono/literals.h | 24 +- libcxx/include/__chrono/month.h | 94 +- libcxx/include/__chrono/month_weekday.h | 104 +- libcxx/include/__chrono/monthday.h | 134 +- libcxx/include/__chrono/steady_clock.h | 18 +- libcxx/include/__chrono/system_clock.h | 24 +- libcxx/include/__chrono/time_point.h | 186 +- libcxx/include/__chrono/weekday.h | 201 +- libcxx/include/__chrono/year.h | 104 +- libcxx/include/__chrono/year_month.h | 102 +- libcxx/include/__chrono/year_month_day.h | 460 +- libcxx/include/__chrono/year_month_weekday.h | 369 +- .../__compare/common_comparison_category.h | 27 +- .../compare_partial_order_fallback.h | 68 +- .../__compare/compare_strong_order_fallback.h | 62 +- libcxx/include/__compare/compare_three_way.h | 19 +- .../__compare/compare_three_way_result.h | 23 +- .../__compare/compare_weak_order_fallback.h | 62 +- libcxx/include/__compare/ordering.h | 164 +- libcxx/include/__compare/partial_order.h | 63 +- libcxx/include/__compare/strong_order.h | 173 +- .../include/__compare/three_way_comparable.h | 36 +- libcxx/include/__compare/weak_order.h | 115 +- libcxx/include/__config | 4 +- libcxx/include/__coroutine/coroutine_handle.h | 257 +- libcxx/include/__coroutine/coroutine_traits.h | 9 +- .../__coroutine/noop_coroutine_handle.h | 103 +- .../include/__coroutine/trivial_awaitables.h | 18 +- .../include/__debug_utils/randomize_range.h | 3 +- libcxx/include/__expected/expected.h | 205 +- libcxx/include/__filesystem/copy_options.h | 50 +- libcxx/include/__filesystem/directory_entry.h | 251 +- .../include/__filesystem/directory_iterator.h | 73 +- .../include/__filesystem/directory_options.h | 44 +- libcxx/include/__filesystem/file_status.h | 28 +- libcxx/include/__filesystem/file_type.h | 14 +- libcxx/include/__filesystem/operations.h | 224 +- libcxx/include/__filesystem/path.h | 652 +- libcxx/include/__filesystem/path_iterator.h | 50 +- libcxx/include/__filesystem/perm_options.h | 39 +- libcxx/include/__filesystem/perms.h | 56 +- .../recursive_directory_iterator.h | 85 +- libcxx/include/__filesystem/u8path.h | 48 +- libcxx/include/__format/buffer.h | 40 +- libcxx/include/__format/concepts.h | 2 +- libcxx/include/__format/format_arg.h | 32 +- libcxx/include/__format/format_arg_store.h | 10 +- libcxx/include/__format/format_args.h | 5 +- libcxx/include/__format/format_context.h | 72 +- libcxx/include/__format/format_error.h | 17 +- libcxx/include/__format/format_functions.h | 184 +- .../include/__format/format_parse_context.h | 30 +- libcxx/include/__format/format_string.h | 20 +- libcxx/include/__format/formatter.h | 4 +- .../__format/formatter_floating_point.h | 92 +- libcxx/include/__format/formatter_integer.h | 37 +- libcxx/include/__format/formatter_integral.h | 44 +- libcxx/include/__format/formatter_output.h | 24 +- libcxx/include/__format/formatter_pointer.h | 9 +- libcxx/include/__format/formatter_string.h | 18 +- .../include/__format/parser_std_format_spec.h | 79 +- libcxx/include/__format/unicode.h | 10 +- libcxx/include/__format/write_escaped.h | 19 +- libcxx/include/__functional/binary_function.h | 16 +- libcxx/include/__functional/binary_negate.h | 25 +- libcxx/include/__functional/bind.h | 385 +- libcxx/include/__functional/bind_back.h | 34 +- libcxx/include/__functional/bind_front.h | 21 +- libcxx/include/__functional/binder1st.h | 35 +- libcxx/include/__functional/binder2nd.h | 35 +- .../__functional/boyer_moore_searcher.h | 128 +- libcxx/include/__functional/compose.h | 25 +- .../include/__functional/default_searcher.h | 31 +- libcxx/include/__functional/function.h | 1645 ++- libcxx/include/__functional/hash.h | 682 +- libcxx/include/__functional/identity.h | 11 +- libcxx/include/__functional/invoke.h | 30 +- libcxx/include/__functional/is_transparent.h | 3 +- libcxx/include/__functional/mem_fn.h | 36 +- libcxx/include/__functional/mem_fun_ref.h | 183 +- libcxx/include/__functional/not_fn.h | 25 +- libcxx/include/__functional/operations.h | 540 +- libcxx/include/__functional/perfect_forward.h | 48 +- .../__functional/pointer_to_binary_function.h | 20 +- .../__functional/pointer_to_unary_function.h | 20 +- .../include/__functional/ranges_operations.h | 24 +- .../include/__functional/reference_wrapper.h | 97 +- libcxx/include/__functional/unary_function.h | 12 +- libcxx/include/__functional/unary_negate.h | 25 +- .../include/__functional/weak_result_type.h | 225 +- libcxx/include/__fwd/get.h | 48 +- libcxx/include/__fwd/span.h | 3 +- libcxx/include/__fwd/string_view.h | 8 +- libcxx/include/__fwd/subrange.h | 2 +- libcxx/include/__hash_table | 3216 +++--- libcxx/include/__ios/fpos.h | 9 +- libcxx/include/__iterator/access.h | 67 +- libcxx/include/__iterator/advance.h | 57 +- .../include/__iterator/back_insert_iterator.h | 53 +- libcxx/include/__iterator/common_iterator.h | 211 +- libcxx/include/__iterator/concepts.h | 338 +- libcxx/include/__iterator/counted_iterator.h | 191 +- libcxx/include/__iterator/data.h | 32 +- libcxx/include/__iterator/default_sentinel.h | 2 +- libcxx/include/__iterator/distance.h | 49 +- libcxx/include/__iterator/empty.h | 19 +- .../include/__iterator/erase_if_container.h | 4 +- .../__iterator/front_insert_iterator.h | 51 +- .../include/__iterator/incrementable_traits.h | 33 +- .../__iterator/indirectly_comparable.h | 3 +- libcxx/include/__iterator/insert_iterator.h | 57 +- libcxx/include/__iterator/istream_iterator.h | 95 +- .../include/__iterator/istreambuf_iterator.h | 131 +- libcxx/include/__iterator/iter_move.h | 68 +- libcxx/include/__iterator/iter_swap.h | 100 +- libcxx/include/__iterator/iterator.h | 16 +- libcxx/include/__iterator/iterator_traits.h | 397 +- libcxx/include/__iterator/mergeable.h | 15 +- libcxx/include/__iterator/move_iterator.h | 419 +- libcxx/include/__iterator/move_sentinel.h | 21 +- libcxx/include/__iterator/next.h | 13 +- libcxx/include/__iterator/ostream_iterator.h | 55 +- .../include/__iterator/ostreambuf_iterator.h | 61 +- libcxx/include/__iterator/permutable.h | 3 +- libcxx/include/__iterator/prev.h | 13 +- libcxx/include/__iterator/readable_traits.h | 46 +- libcxx/include/__iterator/reverse_access.h | 60 +- libcxx/include/__iterator/reverse_iterator.h | 366 +- libcxx/include/__iterator/size.h | 32 +- libcxx/include/__iterator/sortable.h | 4 +- .../include/__iterator/unreachable_sentinel.h | 5 +- libcxx/include/__iterator/wrap_iter.h | 258 +- libcxx/include/__locale | 2434 ++-- .../locale_base_api/bsd_locale_defaults.h | 28 +- .../locale_base_api/bsd_locale_fallbacks.h | 144 +- .../locale_base_api/locale_guard.h | 67 +- libcxx/include/__mbstate_t.h | 16 +- libcxx/include/__memory/addressof.h | 35 +- libcxx/include/__memory/aligned_alloc.h | 42 +- libcxx/include/__memory/allocate_at_least.h | 10 +- libcxx/include/__memory/allocation_guard.h | 98 +- libcxx/include/__memory/allocator.h | 331 +- libcxx/include/__memory/allocator_arg_t.h | 45 +- .../include/__memory/allocator_destructor.h | 24 +- libcxx/include/__memory/allocator_traits.h | 379 +- libcxx/include/__memory/assume_aligned.h | 3 +- libcxx/include/__memory/auto_ptr.h | 94 +- .../include/__memory/builtin_new_allocator.h | 23 +- libcxx/include/__memory/compressed_pair.h | 81 +- libcxx/include/__memory/concepts.h | 15 +- libcxx/include/__memory/construct_at.h | 65 +- libcxx/include/__memory/destruct_n.h | 61 +- libcxx/include/__memory/pointer_traits.h | 174 +- libcxx/include/__memory/ranges_construct_at.h | 28 +- .../ranges_uninitialized_algorithms.h | 89 +- .../include/__memory/raw_storage_iterator.h | 66 +- libcxx/include/__memory/shared_ptr.h | 2495 ++--- libcxx/include/__memory/temp_value.h | 32 +- libcxx/include/__memory/temporary_buffer.h | 78 +- .../__memory/uninitialized_algorithms.h | 461 +- libcxx/include/__memory/unique_ptr.h | 373 +- libcxx/include/__memory/uses_allocator.h | 27 +- libcxx/include/__node_handle | 224 +- libcxx/include/__numeric/accumulate.h | 28 +- .../include/__numeric/adjacent_difference.h | 57 +- libcxx/include/__numeric/exclusive_scan.h | 2 +- libcxx/include/__numeric/gcd_lcm.h | 82 +- libcxx/include/__numeric/inclusive_scan.h | 9 +- libcxx/include/__numeric/inner_product.h | 34 +- libcxx/include/__numeric/iota.h | 10 +- libcxx/include/__numeric/midpoint.h | 63 +- libcxx/include/__numeric/partial_sum.h | 53 +- libcxx/include/__numeric/reduce.h | 8 +- .../__numeric/transform_exclusive_scan.h | 33 +- .../__numeric/transform_inclusive_scan.h | 42 +- libcxx/include/__numeric/transform_reduce.h | 24 +- .../include/__random/bernoulli_distribution.h | 160 +- .../include/__random/binomial_distribution.h | 291 +- libcxx/include/__random/cauchy_distribution.h | 194 +- .../__random/chi_squared_distribution.h | 161 +- libcxx/include/__random/clamp_to_integral.h | 22 +- .../include/__random/discard_block_engine.h | 256 +- .../include/__random/discrete_distribution.h | 340 +- .../__random/exponential_distribution.h | 171 +- .../__random/extreme_value_distribution.h | 196 +- .../include/__random/fisher_f_distribution.h | 194 +- libcxx/include/__random/gamma_distribution.h | 280 +- libcxx/include/__random/generate_canonical.h | 28 +- .../include/__random/geometric_distribution.h | 157 +- .../__random/independent_bits_engine.h | 331 +- libcxx/include/__random/is_seed_sequence.h | 8 +- libcxx/include/__random/is_valid.h | 53 +- .../__random/linear_congruential_engine.h | 491 +- libcxx/include/__random/log2.h | 39 +- .../include/__random/lognormal_distribution.h | 189 +- .../__random/mersenne_twister_engine.h | 1224 ++- .../__random/negative_binomial_distribution.h | 232 +- libcxx/include/__random/normal_distribution.h | 278 +- .../piecewise_constant_distribution.h | 508 +- .../__random/piecewise_linear_distribution.h | 537 +- .../include/__random/poisson_distribution.h | 392 +- libcxx/include/__random/random_device.h | 87 +- libcxx/include/__random/ranlux.h | 2 +- libcxx/include/__random/seed_seq.h | 231 +- .../include/__random/shuffle_order_engine.h | 355 +- .../include/__random/student_t_distribution.h | 169 +- .../__random/subtract_with_carry_engine.h | 468 +- .../__random/uniform_int_distribution.h | 403 +- .../__random/uniform_random_bit_generator.h | 14 +- .../__random/uniform_real_distribution.h | 194 +- .../include/__random/weibull_distribution.h | 185 +- libcxx/include/__ranges/access.h | 237 +- libcxx/include/__ranges/all.h | 55 +- libcxx/include/__ranges/common_view.h | 83 +- libcxx/include/__ranges/concepts.h | 126 +- libcxx/include/__ranges/counted.h | 62 +- libcxx/include/__ranges/data.h | 91 +- libcxx/include/__ranges/drop_view.h | 303 +- libcxx/include/__ranges/empty.h | 71 +- libcxx/include/__ranges/empty_view.h | 40 +- libcxx/include/__ranges/enable_view.h | 11 +- libcxx/include/__ranges/filter_view.h | 381 +- libcxx/include/__ranges/iota_view.h | 569 +- libcxx/include/__ranges/istream_view.h | 3 +- libcxx/include/__ranges/join_view.h | 647 +- libcxx/include/__ranges/lazy_split_view.h | 245 +- .../include/__ranges/non_propagating_cache.h | 129 +- libcxx/include/__ranges/owning_view.h | 106 +- libcxx/include/__ranges/range_adaptor.h | 32 +- libcxx/include/__ranges/rbegin.h | 67 +- libcxx/include/__ranges/ref_view.h | 80 +- libcxx/include/__ranges/rend.h | 71 +- libcxx/include/__ranges/reverse_view.h | 297 +- libcxx/include/__ranges/single_view.h | 16 +- libcxx/include/__ranges/size.h | 52 +- libcxx/include/__ranges/subrange.h | 356 +- libcxx/include/__ranges/take_view.h | 218 +- libcxx/include/__ranges/transform_view.h | 300 +- libcxx/include/__ranges/view_interface.h | 92 +- libcxx/include/__ranges/views.h | 2 +- libcxx/include/__ranges/zip_view.h | 235 +- libcxx/include/__split_buffer | 604 +- libcxx/include/__string/char_traits.h | 1093 +- .../include/__support/android/locale_bionic.h | 50 +- libcxx/include/__support/fuchsia/xlocale.h | 8 +- libcxx/include/__support/ibm/gettod_zos.h | 14 +- .../include/__support/ibm/locale_mgmt_zos.h | 34 +- libcxx/include/__support/ibm/nanosleep.h | 6 +- libcxx/include/__support/ibm/xlocale.h | 43 +- libcxx/include/__support/newlib/xlocale.h | 11 +- libcxx/include/__support/win32/locale_win32.h | 321 +- .../__support/xlocale/__nop_locale_mgmt.h | 18 +- libcxx/include/__system_error/errc.h | 170 +- libcxx/include/__thread/formatter.h | 62 +- libcxx/include/__thread/poll_with_backoff.h | 37 +- libcxx/include/__thread/this_thread.h | 63 +- libcxx/include/__thread/thread.h | 324 +- .../include/__thread/timed_backoff_policy.h | 28 +- libcxx/include/__threading_support | 410 +- libcxx/include/__tree | 3613 +++--- libcxx/include/__tuple/make_tuple_types.h | 24 +- libcxx/include/__tuple/sfinae_helpers.h | 136 +- libcxx/include/__tuple/tuple_element.h | 64 +- libcxx/include/__tuple/tuple_indices.h | 10 +- libcxx/include/__tuple/tuple_like_ext.h | 24 +- libcxx/include/__tuple/tuple_size.h | 47 +- libcxx/include/__tuple/tuple_types.h | 3 +- libcxx/include/__utility/as_const.h | 4 +- libcxx/include/__utility/cmp.h | 73 +- .../include/__utility/convert_to_integral.h | 50 +- libcxx/include/__utility/exchange.h | 14 +- libcxx/include/__utility/in_place.h | 12 +- libcxx/include/__utility/integer_sequence.h | 104 +- libcxx/include/__utility/pair.h | 1162 +- .../include/__utility/piecewise_construct.h | 4 +- libcxx/include/__utility/priority_tag.h | 6 +- libcxx/include/__utility/rel_ops.h | 41 +- libcxx/include/__utility/to_underlying.h | 6 +- libcxx/include/__utility/unreachable.h | 4 +- libcxx/include/__variant/monostate.h | 2 +- libcxx/include/__verbose_abort | 4 +- libcxx/include/any | 788 +- libcxx/include/array | 535 +- libcxx/include/barrier | 356 +- libcxx/include/bitset | 1290 +-- libcxx/include/cctype | 29 +- libcxx/include/cmath | 173 +- libcxx/include/codecvt | 776 +- libcxx/include/complex | 1921 ++-- libcxx/include/condition_variable | 223 +- libcxx/include/cstddef | 52 +- libcxx/include/ctype.h | 28 +- libcxx/include/cuchar | 2 +- libcxx/include/deque | 3695 +++---- libcxx/include/errno.h | 522 +- libcxx/include/expected | 1 - libcxx/include/experimental/__config | 22 +- libcxx/include/experimental/__memory | 90 +- libcxx/include/experimental/iterator | 69 +- libcxx/include/experimental/propagate_const | 305 +- libcxx/include/experimental/type_traits | 69 +- libcxx/include/experimental/utility | 2 +- libcxx/include/ext/__hash | 130 +- libcxx/include/ext/hash_map | 1132 +- libcxx/include/ext/hash_set | 634 +- libcxx/include/fenv.h | 70 +- libcxx/include/float.h | 12 +- libcxx/include/forward_list | 2031 ++-- libcxx/include/fstream | 2093 ++-- libcxx/include/future | 2551 ++--- libcxx/include/initializer_list | 71 +- libcxx/include/inttypes.h | 10 +- libcxx/include/iomanip | 754 +- libcxx/include/ios | 953 +- libcxx/include/iosfwd | 65 +- libcxx/include/istream | 2051 ++-- libcxx/include/latch | 98 +- libcxx/include/limits | 1078 +- libcxx/include/list | 2390 ++-- libcxx/include/locale | 6071 +++++----- libcxx/include/locale.h | 2 +- libcxx/include/map | 2617 ++--- libcxx/include/math.h | 81 +- libcxx/include/mutex | 467 +- libcxx/include/new | 220 +- libcxx/include/numbers | 93 +- libcxx/include/optional | 1759 ++- libcxx/include/ostream | 1202 +- libcxx/include/queue | 1008 +- libcxx/include/ranges | 2 +- libcxx/include/ratio | 396 +- libcxx/include/regex | 8356 ++++++-------- libcxx/include/scoped_allocator | 821 +- libcxx/include/semaphore | 193 +- libcxx/include/set | 1577 ++- libcxx/include/span | 701 +- libcxx/include/sstream | 1274 +-- libcxx/include/stack | 349 +- libcxx/include/stdatomic.h | 18 +- libcxx/include/stdbool.h | 10 +- libcxx/include/stddef.h | 28 +- libcxx/include/stdexcept | 232 +- libcxx/include/stdint.h | 6 +- libcxx/include/stdio.h | 36 +- libcxx/include/stdlib.h | 86 +- libcxx/include/streambuf | 530 +- libcxx/include/string | 4675 ++++---- libcxx/include/string.h | 2 +- libcxx/include/string_view | 1135 +- libcxx/include/strstream | 334 +- libcxx/include/thread | 2 +- libcxx/include/tuple | 2126 ++-- libcxx/include/typeindex | 65 +- libcxx/include/typeinfo | 242 +- libcxx/include/uchar.h | 12 +- libcxx/include/unordered_map | 3111 +++--- libcxx/include/unordered_set | 1953 ++-- libcxx/include/valarray | 4682 ++++---- libcxx/include/variant | 1235 +-- libcxx/include/vector | 3829 +++---- libcxx/include/wchar.h | 157 +- libcxx/include/wctype.h | 43 +- libcxx/src/any.cpp | 16 +- libcxx/src/atomic.cpp | 216 +- libcxx/src/barrier.cpp | 106 +- libcxx/src/bind.cpp | 23 +- libcxx/src/call_once.cpp | 62 +- libcxx/src/charconv.cpp | 33 +- libcxx/src/chrono.cpp | 118 +- libcxx/src/condition_variable.cpp | 95 +- libcxx/src/condition_variable_destructor.cpp | 20 +- libcxx/src/exception.cpp | 28 +- libcxx/src/filesystem/directory_entry.cpp | 9 +- libcxx/src/filesystem/directory_iterator.cpp | 84 +- libcxx/src/filesystem/error.h | 24 +- libcxx/src/filesystem/file_descriptor.h | 53 +- libcxx/src/filesystem/filesystem_clock.cpp | 20 +- libcxx/src/filesystem/filesystem_error.cpp | 12 +- libcxx/src/filesystem/int128_builtins.cpp | 14 +- libcxx/src/filesystem/operations.cpp | 323 +- libcxx/src/filesystem/path.cpp | 79 +- libcxx/src/filesystem/path_parser.h | 58 +- libcxx/src/filesystem/posix_compat.h | 231 +- libcxx/src/filesystem/time_utils.h | 164 +- libcxx/src/functional.cpp | 10 +- libcxx/src/future.cpp | 319 +- libcxx/src/hash.cpp | 899 +- libcxx/src/include/apple_availability.h | 34 +- libcxx/src/include/atomic_support.h | 156 +- libcxx/src/include/config_elast.h | 18 +- libcxx/src/include/refstring.h | 137 +- libcxx/src/include/ryu/ryu.h | 1 - libcxx/src/include/sso_allocator.h | 81 +- libcxx/src/ios.cpp | 569 +- libcxx/src/iostream.cpp | 165 +- libcxx/src/locale.cpp | 9786 ++++++++--------- libcxx/src/memory.cpp | 229 +- libcxx/src/mutex.cpp | 207 +- libcxx/src/mutex_destructor.cpp | 20 +- libcxx/src/optional.cpp | 14 +- libcxx/src/random.cpp | 181 +- libcxx/src/random_shuffle.cpp | 35 +- libcxx/src/regex.cpp | 293 +- libcxx/src/std_stream.h | 570 +- libcxx/src/string.cpp | 368 +- libcxx/src/support/ibm/mbsnrtowcs.cpp | 30 +- libcxx/src/support/ibm/wcsnrtombs.cpp | 27 +- libcxx/src/support/ibm/xlocale_zos.cpp | 42 +- .../support/runtime/exception_fallback.ipp | 139 +- .../src/support/runtime/exception_glibcxx.ipp | 18 +- .../support/runtime/exception_libcxxabi.ipp | 15 +- .../support/runtime/exception_libcxxrt.ipp | 11 +- libcxx/src/support/runtime/exception_msvc.ipp | 145 +- .../runtime/exception_pointer_cxxabi.ipp | 73 +- .../runtime/exception_pointer_glibcxx.ipp | 61 +- .../runtime/exception_pointer_msvc.ipp | 22 +- .../exception_pointer_unimplemented.ipp | 43 +- .../src/support/runtime/stdexcept_default.ipp | 8 +- .../support/runtime/stdexcept_vcruntime.ipp | 2 +- libcxx/src/support/win32/locale_win32.cpp | 167 +- libcxx/src/support/win32/support.cpp | 246 +- libcxx/src/support/win32/thread_win32.cpp | 166 +- libcxx/src/system_error.cpp | 225 +- libcxx/src/thread.cpp | 222 +- libcxx/src/typeinfo.cpp | 30 +- libcxx/src/valarray.cpp | 56 +- libcxx/src/variant.cpp | 6 +- libcxx/src/vector.cpp | 8 +- libcxx/src/verbose_abort.cpp | 6 +- 542 files changed, 67543 insertions(+), 84889 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000..6b281f33f737d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,142 @@ +libcxx/src/**/*.cpp merge=libcxx-reformat +libcxx/include/**/*.h merge=libcxx-reformat + +# Explicitly handle files with no extension +libcxx/include/__availability merge=libcxx-reformat +libcxx/include/__bit_reference merge=libcxx-reformat +libcxx/include/__config merge=libcxx-reformat +libcxx/include/__hash_table merge=libcxx-reformat +libcxx/include/__locale merge=libcxx-reformat +libcxx/include/__node_handle merge=libcxx-reformat +libcxx/include/__split_buffer merge=libcxx-reformat +libcxx/include/__std_clang_module merge=libcxx-reformat +libcxx/include/__threading_support merge=libcxx-reformat +libcxx/include/__tree merge=libcxx-reformat +libcxx/include/__verbose_abort merge=libcxx-reformat +libcxx/include/algorithm merge=libcxx-reformat +libcxx/include/any merge=libcxx-reformat +libcxx/include/array merge=libcxx-reformat +libcxx/include/atomic merge=libcxx-reformat +libcxx/include/barrier merge=libcxx-reformat +libcxx/include/bit merge=libcxx-reformat +libcxx/include/bitset merge=libcxx-reformat +libcxx/include/cassert merge=libcxx-reformat +libcxx/include/ccomplex merge=libcxx-reformat +libcxx/include/cctype merge=libcxx-reformat +libcxx/include/cerrno merge=libcxx-reformat +libcxx/include/cfenv merge=libcxx-reformat +libcxx/include/cfloat merge=libcxx-reformat +libcxx/include/charconv merge=libcxx-reformat +libcxx/include/chrono merge=libcxx-reformat +libcxx/include/cinttypes merge=libcxx-reformat +libcxx/include/ciso646 merge=libcxx-reformat +libcxx/include/climits merge=libcxx-reformat +libcxx/include/clocale merge=libcxx-reformat +libcxx/include/cmath merge=libcxx-reformat +libcxx/include/codecvt merge=libcxx-reformat +libcxx/include/compare merge=libcxx-reformat +libcxx/include/complex merge=libcxx-reformat +libcxx/include/concepts merge=libcxx-reformat +libcxx/include/condition_variable merge=libcxx-reformat +libcxx/include/coroutine merge=libcxx-reformat +libcxx/include/csetjmp merge=libcxx-reformat +libcxx/include/csignal merge=libcxx-reformat +libcxx/include/cstdarg merge=libcxx-reformat +libcxx/include/cstdbool merge=libcxx-reformat +libcxx/include/cstddef merge=libcxx-reformat +libcxx/include/cstdint merge=libcxx-reformat +libcxx/include/cstdio merge=libcxx-reformat +libcxx/include/cstdlib merge=libcxx-reformat +libcxx/include/cstring merge=libcxx-reformat +libcxx/include/ctgmath merge=libcxx-reformat +libcxx/include/ctime merge=libcxx-reformat +libcxx/include/cuchar merge=libcxx-reformat +libcxx/include/cwchar merge=libcxx-reformat +libcxx/include/cwctype merge=libcxx-reformat +libcxx/include/deque merge=libcxx-reformat +libcxx/include/exception merge=libcxx-reformat +libcxx/include/execution merge=libcxx-reformat +libcxx/include/expected merge=libcxx-reformat +libcxx/include/experimental/__config merge=libcxx-reformat +libcxx/include/experimental/__memory merge=libcxx-reformat +libcxx/include/experimental/deque merge=libcxx-reformat +libcxx/include/experimental/forward_list merge=libcxx-reformat +libcxx/include/experimental/iterator merge=libcxx-reformat +libcxx/include/experimental/list merge=libcxx-reformat +libcxx/include/experimental/map merge=libcxx-reformat +libcxx/include/experimental/memory merge=libcxx-reformat +libcxx/include/experimental/memory_resource merge=libcxx-reformat +libcxx/include/experimental/propagate_const merge=libcxx-reformat +libcxx/include/experimental/regex merge=libcxx-reformat +libcxx/include/experimental/set merge=libcxx-reformat +libcxx/include/experimental/simd merge=libcxx-reformat +libcxx/include/experimental/string merge=libcxx-reformat +libcxx/include/experimental/type_traits merge=libcxx-reformat +libcxx/include/experimental/unordered_map merge=libcxx-reformat +libcxx/include/experimental/unordered_set merge=libcxx-reformat +libcxx/include/experimental/utility merge=libcxx-reformat +libcxx/include/experimental/vector merge=libcxx-reformat +libcxx/include/ext/__hash merge=libcxx-reformat +libcxx/include/ext/hash_map merge=libcxx-reformat +libcxx/include/ext/hash_set merge=libcxx-reformat +libcxx/include/filesystem merge=libcxx-reformat +libcxx/include/format merge=libcxx-reformat +libcxx/include/forward_list merge=libcxx-reformat +libcxx/include/fstream merge=libcxx-reformat +libcxx/include/functional merge=libcxx-reformat +libcxx/include/future merge=libcxx-reformat +libcxx/include/initializer_list merge=libcxx-reformat +libcxx/include/iomanip merge=libcxx-reformat +libcxx/include/ios merge=libcxx-reformat +libcxx/include/iosfwd merge=libcxx-reformat +libcxx/include/iostream merge=libcxx-reformat +libcxx/include/istream merge=libcxx-reformat +libcxx/include/iterator merge=libcxx-reformat +libcxx/include/latch merge=libcxx-reformat +libcxx/include/limits merge=libcxx-reformat +libcxx/include/list merge=libcxx-reformat +libcxx/include/locale merge=libcxx-reformat +libcxx/include/map merge=libcxx-reformat +libcxx/include/mdspan merge=libcxx-reformat +libcxx/include/memory merge=libcxx-reformat +libcxx/include/memory_resource merge=libcxx-reformat +libcxx/include/mutex merge=libcxx-reformat +libcxx/include/new merge=libcxx-reformat +libcxx/include/numbers merge=libcxx-reformat +libcxx/include/numeric merge=libcxx-reformat +libcxx/include/optional merge=libcxx-reformat +libcxx/include/ostream merge=libcxx-reformat +libcxx/include/print merge=libcxx-reformat +libcxx/include/queue merge=libcxx-reformat +libcxx/include/random merge=libcxx-reformat +libcxx/include/ranges merge=libcxx-reformat +libcxx/include/ratio merge=libcxx-reformat +libcxx/include/regex merge=libcxx-reformat +libcxx/include/scoped_allocator merge=libcxx-reformat +libcxx/include/semaphore merge=libcxx-reformat +libcxx/include/set merge=libcxx-reformat +libcxx/include/shared_mutex merge=libcxx-reformat +libcxx/include/source_location merge=libcxx-reformat +libcxx/include/span merge=libcxx-reformat +libcxx/include/sstream merge=libcxx-reformat +libcxx/include/stack merge=libcxx-reformat +libcxx/include/stdexcept merge=libcxx-reformat +libcxx/include/stop_token merge=libcxx-reformat +libcxx/include/streambuf merge=libcxx-reformat +libcxx/include/string merge=libcxx-reformat +libcxx/include/string_view merge=libcxx-reformat +libcxx/include/strstream merge=libcxx-reformat +libcxx/include/syncstream merge=libcxx-reformat +libcxx/include/system_error merge=libcxx-reformat +libcxx/include/thread merge=libcxx-reformat +libcxx/include/tuple merge=libcxx-reformat +libcxx/include/type_traits merge=libcxx-reformat +libcxx/include/typeindex merge=libcxx-reformat +libcxx/include/typeinfo merge=libcxx-reformat +libcxx/include/unordered_map merge=libcxx-reformat +libcxx/include/unordered_set merge=libcxx-reformat +libcxx/include/utility merge=libcxx-reformat +libcxx/include/valarray merge=libcxx-reformat +libcxx/include/variant merge=libcxx-reformat +libcxx/include/vector merge=libcxx-reformat +libcxx/include/version merge=libcxx-reformat diff --git a/libcxx/include/__algorithm/binary_search.h b/libcxx/include/__algorithm/binary_search.h index 5a1d49f5f43e2..7a77d7b5447bd 100644 --- a/libcxx/include/__algorithm/binary_search.h +++ b/libcxx/include/__algorithm/binary_search.h @@ -22,22 +22,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) -{ - __first = std::lower_bound<_ForwardIterator, _Tp, __comp_ref_type<_Compare> >(__first, __last, __value, __comp); - return __first != __last && !__comp(__value, *__first); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) { + __first = std::lower_bound<_ForwardIterator, _Tp, __comp_ref_type<_Compare> >(__first, __last, __value, __comp); + return __first != __last && !__comp(__value, *__first); } template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) -{ - return std::binary_search(__first, __last, __value, __less<>()); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + return std::binary_search(__first, __last, __value, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/comp_ref_type.h b/libcxx/include/__algorithm/comp_ref_type.h index 2797e9da66585..15f4a535a30bf 100644 --- a/libcxx/include/__algorithm/comp_ref_type.h +++ b/libcxx/include/__algorithm/comp_ref_type.h @@ -20,52 +20,42 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -struct __debug_less -{ - _Compare &__comp_; - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI __debug_less(_Compare& __c) : __comp_(__c) {} +struct __debug_less { + _Compare& __comp_; + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI __debug_less(_Compare& __c) : __comp_(__c) {} - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Up& __y) - { - bool __r = __comp_(__x, __y); - if (__r) - __do_compare_assert(0, __y, __x); - return __r; - } + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Up& __y) { + bool __r = __comp_(__x, __y); + if (__r) + __do_compare_assert(0, __y, __x); + return __r; + } - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(_Tp& __x, _Up& __y) - { - bool __r = __comp_(__x, __y); - if (__r) - __do_compare_assert(0, __y, __x); - return __r; - } + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(_Tp& __x, _Up& __y) { + bool __r = __comp_(__x, __y); + if (__r) + __do_compare_assert(0, __y, __x); + return __r; + } - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 - inline _LIBCPP_HIDE_FROM_ABI - decltype((void)std::declval<_Compare&>()( - std::declval<_LHS &>(), std::declval<_RHS &>())) - __do_compare_assert(int, _LHS & __l, _RHS & __r) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__comp_(__l, __r), - "Comparator does not induce a strict weak ordering"); - (void)__l; - (void)__r; - } + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 inline _LIBCPP_HIDE_FROM_ABI decltype((void)std::declval<_Compare&>()( + std::declval<_LHS&>(), std::declval<_RHS&>())) + __do_compare_assert(int, _LHS& __l, _RHS& __r) { + _LIBCPP_ASSERT_UNCATEGORIZED(!__comp_(__l, __r), "Comparator does not induce a strict weak ordering"); + (void)__l; + (void)__r; + } - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 - inline _LIBCPP_HIDE_FROM_ABI - void __do_compare_assert(long, _LHS &, _RHS &) {} + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 inline _LIBCPP_HIDE_FROM_ABI void __do_compare_assert(long, _LHS&, _RHS&) {} }; // Pass the comparator by lvalue reference. Or in debug mode, using a // debugging wrapper that stores a reference. -# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG +#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG template using __comp_ref_type = __debug_less<_Comp>; #else diff --git a/libcxx/include/__algorithm/copy_backward.h b/libcxx/include/__algorithm/copy_backward.h index d85d297b3226c..3ec88d8bd5cc3 100644 --- a/libcxx/include/__algorithm/copy_backward.h +++ b/libcxx/include/__algorithm/copy_backward.h @@ -108,8 +108,7 @@ struct __copy_backward_loop { struct __copy_backward_trivial { // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. - template ::value, int> = 0> + template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> operator()(_In* __first, _In* __last, _Out* __result) const { return std::__copy_backward_trivial_impl(__first, __last, __result); @@ -124,16 +123,13 @@ __copy_backward(_BidirectionalIterator1 __first, _Sentinel __last, _Bidirectiona } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_BidirectionalIterator2 -copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, - _BidirectionalIterator2 __result) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator2 +copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) { static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value && - std::is_copy_constructible<_BidirectionalIterator1>::value, "Iterators must be copy constructible."); + std::is_copy_constructible<_BidirectionalIterator1>::value, + "Iterators must be copy constructible."); - return std::__copy_backward<_ClassicAlgPolicy>( - std::move(__first), std::move(__last), std::move(__result)).second; + return std::__copy_backward<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/copy_if.h b/libcxx/include/__algorithm/copy_if.h index d68d1dbbadf8a..228e4d22323e3 100644 --- a/libcxx/include/__algorithm/copy_if.h +++ b/libcxx/include/__algorithm/copy_if.h @@ -17,21 +17,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -copy_if(_InputIterator __first, _InputIterator __last, - _OutputIterator __result, _Predicate __pred) -{ - for (; __first != __last; ++__first) - { - if (__pred(*__first)) - { - *__result = *__first; - ++__result; - } +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Predicate __pred) { + for (; __first != __last; ++__first) { + if (__pred(*__first)) { + *__result = *__first; + ++__result; } - return __result; + } + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/copy_move_common.h b/libcxx/include/__algorithm/copy_move_common.h index c06892e9e3c79..b350507e32bae 100644 --- a/libcxx/include/__algorithm/copy_move_common.h +++ b/libcxx/include/__algorithm/copy_move_common.h @@ -38,22 +38,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD template struct __can_lower_copy_assignment_to_memmove { static const bool value = - // If the types are always bitcastable, it's valid to do a bitwise copy between them. - __is_always_bitcastable<_From, _To>::value && - // Reject conversions that wouldn't be performed by the regular built-in assignment (e.g. between arrays). - is_trivially_assignable<_To&, const _From&>::value && - // `memmove` doesn't accept `volatile` pointers, make sure the optimization SFINAEs away in that case. - !is_volatile<_From>::value && - !is_volatile<_To>::value; + // If the types are always bitcastable, it's valid to do a bitwise copy between them. + __is_always_bitcastable<_From, _To>::value && + // Reject conversions that wouldn't be performed by the regular built-in assignment (e.g. between arrays). + is_trivially_assignable<_To&, const _From&>::value && + // `memmove` doesn't accept `volatile` pointers, make sure the optimization SFINAEs away in that case. + !is_volatile<_From>::value && !is_volatile<_To>::value; }; template struct __can_lower_move_assignment_to_memmove { static const bool value = - __is_always_bitcastable<_From, _To>::value && - is_trivially_assignable<_To&, _From&&>::value && - !is_volatile<_From>::value && - !is_volatile<_To>::value; + __is_always_bitcastable<_From, _To>::value && is_trivially_assignable<_To&, _From&&>::value && + !is_volatile<_From>::value && !is_volatile<_To>::value; }; // `memmove` algorithms implementation. @@ -95,8 +92,8 @@ struct __can_rewrap<_InIter, _Sent, _OutIter, // Note that sentinels are always copy-constructible. - __enable_if_t< is_copy_constructible<_InIter>::value && - is_copy_constructible<_OutIter>::value > > : true_type {}; + __enable_if_t< is_copy_constructible<_InIter>::value && is_copy_constructible<_OutIter>::value > > + : true_type {}; template (std::move(__first), std::move(__result.first)), - std::__rewrap_iter(std::move(__out_first), std::move(__result.second))); + std::__rewrap_iter(std::move(__out_first), std::move(__result.second))); } template ::value && - !__has_random_access_iterator_category<_InputIterator>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) -{ - typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; - _IntegralSize __n = __orig_n; - if (__n > 0) - { - *__result = *__first; - ++__result; - for (--__n; __n > 0; --__n) - { - ++__first; - *__result = *__first; - ++__result; - } +template ::value && + !__has_random_access_iterator_category<_InputIterator>::value, + int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) { + typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; + _IntegralSize __n = __orig_n; + if (__n > 0) { + *__result = *__first; + ++__result; + for (--__n; __n > 0; --__n) { + ++__first; + *__result = *__first; + ++__result; } - return __result; + } + return __result; } -template::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) -{ - typedef typename iterator_traits<_InputIterator>::difference_type difference_type; - typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; - _IntegralSize __n = __orig_n; - return std::copy(__first, __first + difference_type(__n), __result); +template ::value, int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) { + typedef typename iterator_traits<_InputIterator>::difference_type difference_type; + typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; + _IntegralSize __n = __orig_n; + return std::copy(__first, __first + difference_type(__n), __result); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/equal.h b/libcxx/include/__algorithm/equal.h index fb358613e2101..f03f010aa51ab 100644 --- a/libcxx/include/__algorithm/equal.h +++ b/libcxx/include/__algorithm/equal.h @@ -68,8 +68,13 @@ equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first #if _LIBCPP_STD_VER >= 14 template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, - _BinaryPredicate __pred, input_iterator_tag, input_iterator_tag) { +__equal(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _BinaryPredicate __pred, + input_iterator_tag, + input_iterator_tag) { for (; __first1 != __last1 && __first2 != __last2; ++__first1, (void)++__first2) if (!__pred(*__first1, *__first2)) return false; @@ -104,8 +109,12 @@ __equal_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _Up*, _Pred&, _Proj1&, template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, - _RandomAccessIterator2 __last2, _BinaryPredicate __pred, random_access_iterator_tag, +__equal(_RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _BinaryPredicate __pred, + random_access_iterator_tag, random_access_iterator_tag) { if (std::distance(__first1, __last1) != std::distance(__first2, __last2)) return false; @@ -122,10 +131,18 @@ __equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _Random template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, +equal(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, _BinaryPredicate __pred) { return std::__equal<_BinaryPredicate&>( - __first1, __last1, __first2, __last2, __pred, typename iterator_traits<_InputIterator1>::iterator_category(), + __first1, + __last1, + __first2, + __last2, + __pred, + typename iterator_traits<_InputIterator1>::iterator_category(), typename iterator_traits<_InputIterator2>::iterator_category()); } diff --git a/libcxx/include/__algorithm/equal_range.h b/libcxx/include/__algorithm/equal_range.h index dc1268a6ff110..7ce54965fff05 100644 --- a/libcxx/include/__algorithm/equal_range.h +++ b/libcxx/include/__algorithm/equal_range.h @@ -49,9 +49,8 @@ __equal_range(_Iter __first, _Sent __last, const _Tp& __value, _Compare&& __comp __len = __half_len; } else { _Iter __mp1 = __mid; - return pair<_Iter, _Iter>( - std::__lower_bound<_AlgPolicy>(__first, __mid, __value, __comp, __proj), - std::__upper_bound<_AlgPolicy>(++__mp1, __end, __value, __comp, __proj)); + return pair<_Iter, _Iter>(std::__lower_bound<_AlgPolicy>(__first, __mid, __value, __comp, __proj), + std::__upper_bound<_AlgPolicy>(++__mp1, __end, __value, __comp, __proj)); } } return pair<_Iter, _Iter>(__first, __first); @@ -60,10 +59,8 @@ __equal_range(_Iter __first, _Sent __last, const _Tp& __value, _Compare&& __comp template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator> equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) { - static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, - "The comparator has to be callable"); - static_assert(is_copy_constructible<_ForwardIterator>::value, - "Iterator has to be copy constructible"); + static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, "The comparator has to be callable"); + static_assert(is_copy_constructible<_ForwardIterator>::value, "Iterator has to be copy constructible"); return std::__equal_range<_ClassicAlgPolicy>( std::move(__first), std::move(__last), diff --git a/libcxx/include/__algorithm/fill.h b/libcxx/include/__algorithm/fill.h index 360e2c3c3c858..1ce3eadb013d0 100644 --- a/libcxx/include/__algorithm/fill.h +++ b/libcxx/include/__algorithm/fill.h @@ -22,28 +22,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD // fill isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset. template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -__fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, forward_iterator_tag) -{ - for (; __first != __last; ++__first) - *__first = __value; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +__fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, forward_iterator_tag) { + for (; __first != __last; ++__first) + *__first = __value; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -__fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value, random_access_iterator_tag) -{ - std::fill_n(__first, __last - __first, __value); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +__fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value, random_access_iterator_tag) { + std::fill_n(__first, __last - __first, __value); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) -{ - std::__fill(__first, __last, __value, typename iterator_traits<_ForwardIterator>::iterator_category()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + std::__fill(__first, __last, __value, typename iterator_traits<_ForwardIterator>::iterator_category()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/fill_n.h b/libcxx/include/__algorithm/fill_n.h index 89681a342ea78..36f3349d9e7a3 100644 --- a/libcxx/include/__algorithm/fill_n.h +++ b/libcxx/include/__algorithm/fill_n.h @@ -22,21 +22,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD // fill_n isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset. template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) -{ - for (; __n > 0; ++__first, (void) --__n) - *__first = __value; - return __first; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) { + for (; __n > 0; ++__first, (void)--__n) + *__first = __value; + return __first; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) -{ - return std::__fill_n(__first, std::__convert_to_integral(__n), __value); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) { + return std::__fill_n(__first, std::__convert_to_integral(__n), __value); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/find_end.h b/libcxx/include/__algorithm/find_end.h index edb9891c66f70..4c26891666b22 100644 --- a/libcxx/include/__algorithm/find_end.h +++ b/libcxx/include/__algorithm/find_end.h @@ -28,15 +28,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template < - class _AlgPolicy, - class _Iter1, - class _Sent1, - class _Iter2, - class _Sent2, - class _Pred, - class _Proj1, - class _Proj2> +template < class _AlgPolicy, + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Pred, + class _Proj1, + class _Proj2> _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __find_end_impl( _Iter1 __first1, _Sent1 __last1, @@ -49,7 +48,7 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> forward_iterator_tag) { // modeled after search algorithm _Iter1 __match_first = _IterOps<_AlgPolicy>::next(__first1, __last1); // __last1 is the "default" answer - _Iter1 __match_last = __match_first; + _Iter1 __match_last = __match_first; if (__first2 == __last2) return pair<_Iter1, _Iter1>(__match_last, __match_last); while (true) { @@ -66,15 +65,14 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> while (true) { if (++__m2 == __last2) { // Pattern exhaused, record answer and search for another one __match_first = __first1; - __match_last = ++__m1; + __match_last = ++__m1; ++__first1; break; } if (++__m1 == __last1) // Source exhausted, return last answer return pair<_Iter1, _Iter1>(__match_first, __match_last); - // mismatch, restart with a new __first - if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) - { + // mismatch, restart with a new __first + if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) { ++__first1; break; } // else there is a match, check next elements @@ -82,15 +80,14 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> } } -template < - class _IterOps, - class _Pred, - class _Iter1, - class _Sent1, - class _Iter2, - class _Sent2, - class _Proj1, - class _Proj2> +template < class _IterOps, + class _Pred, + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter1 __find_end( _Iter1 __first1, _Sent1 __sent1, @@ -127,23 +124,21 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter1 __find_end( return __last1; // if there is a mismatch, restart with a new __l1 - if (!std::__invoke(__pred, std::__invoke(__proj1, *--__m1), std::__invoke(__proj2, *--__m2))) - { + if (!std::__invoke(__pred, std::__invoke(__proj1, *--__m1), std::__invoke(__proj2, *--__m2))) { break; } // else there is a match, check next elements } } } -template < - class _AlgPolicy, - class _Pred, - class _Iter1, - class _Sent1, - class _Iter2, - class _Sent2, - class _Proj1, - class _Proj2> +template < class _AlgPolicy, + class _Pred, + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter1 __find_end( _Iter1 __first1, _Sent1 __sent1, @@ -165,8 +160,8 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter1 __find_end( if (__len1 < __len2) return __last1; const _Iter1 __s = __first1 + _D1(__len2 - 1); // End of pattern match can't go before here - _Iter1 __l1 = __last1; - _Iter2 __l2 = __last2; + _Iter1 __l1 = __last1; + _Iter2 __l2 = __last2; --__l2; while (true) { while (true) { @@ -189,10 +184,12 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter1 __find_end( } template -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_ForwardIterator1 __find_end_classic(_ForwardIterator1 __first1, _ForwardIterator1 __last1, - _ForwardIterator2 __first2, _ForwardIterator2 __last2, - _BinaryPredicate& __pred) { +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_end_classic( + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate& __pred) { auto __proj = __identity(); return std::__find_end_impl<_ClassicAlgPolicy>( __first1, @@ -208,17 +205,18 @@ _ForwardIterator1 __find_end_classic(_ForwardIterator1 __first1, _ForwardIterato } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator1 find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1, - _ForwardIterator2 __first2, _ForwardIterator2 __last2, - _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 find_end( + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate __pred) { return std::__find_end_classic(__first1, __last1, __first2, __last2, __pred); } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator1 find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1, - _ForwardIterator2 __first2, _ForwardIterator2 __last2) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 +find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) { return std::find_end(__first1, __last1, __first2, __last2, __equal_to()); } diff --git a/libcxx/include/__algorithm/find_first_of.h b/libcxx/include/__algorithm/find_first_of.h index 15a147242b0e5..14271cccc42b1 100644 --- a/libcxx/include/__algorithm/find_first_of.h +++ b/libcxx/include/__algorithm/find_first_of.h @@ -21,12 +21,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_first_of_ce(_ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _ForwardIterator2 __last2, - _BinaryPredicate&& __pred) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_first_of_ce( + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate&& __pred) { for (; __first1 != __last1; ++__first1) for (_ForwardIterator2 __j = __first2; __j != __last2; ++__j) if (__pred(*__first1, *__j)) @@ -35,9 +35,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_first_of_ce(_ForwardItera } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 -find_first_of(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, - _ForwardIterator2 __last2, _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 find_first_of( + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate __pred) { return std::__find_first_of_ce(__first1, __last1, __first2, __last2, __pred); } diff --git a/libcxx/include/__algorithm/for_each_n.h b/libcxx/include/__algorithm/for_each_n.h index a72a0912cfa52..fce380b49df3e 100644 --- a/libcxx/include/__algorithm/for_each_n.h +++ b/libcxx/include/__algorithm/for_each_n.h @@ -22,9 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator for_each_n(_InputIterator __first, - _Size __orig_n, - _Function __f) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator +for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) { typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; _IntegralSize __n = __orig_n; while (__n > 0) { diff --git a/libcxx/include/__algorithm/generate.h b/libcxx/include/__algorithm/generate.h index e2051dc16dc03..c95b527402f5d 100644 --- a/libcxx/include/__algorithm/generate.h +++ b/libcxx/include/__algorithm/generate.h @@ -18,12 +18,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __gen) -{ - for (; __first != __last; ++__first) - *__first = __gen(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __gen) { + for (; __first != __last; ++__first) + *__first = __gen(); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/generate_n.h b/libcxx/include/__algorithm/generate_n.h index 5719826e93edf..f36403fd0f94a 100644 --- a/libcxx/include/__algorithm/generate_n.h +++ b/libcxx/include/__algorithm/generate_n.h @@ -19,15 +19,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen) -{ - typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; - _IntegralSize __n = __orig_n; - for (; __n > 0; ++__first, (void) --__n) - *__first = __gen(); - return __first; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen) { + typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize; + _IntegralSize __n = __orig_n; + for (; __n > 0; ++__first, (void)--__n) + *__first = __gen(); + return __first; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/half_positive.h b/libcxx/include/__algorithm/half_positive.h index 2315e154fb2ef..ebda0da372369 100644 --- a/libcxx/include/__algorithm/half_positive.h +++ b/libcxx/include/__algorithm/half_positive.h @@ -23,19 +23,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD // Perform division by two quickly for positive integers (llvm.org/PR39129) template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -_Integral -__half_positive(_Integral __value) -{ - return static_cast<_Integral>(static_cast<__make_unsigned_t<_Integral> >(__value) / 2); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Integral __half_positive(_Integral __value) { + return static_cast<_Integral>(static_cast<__make_unsigned_t<_Integral> >(__value) / 2); } template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -_Tp -__half_positive(_Tp __value) -{ - return __value / 2; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp __half_positive(_Tp __value) { + return __value / 2; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/in_found_result.h b/libcxx/include/__algorithm/in_found_result.h index d9ca287f017b6..88a0255d16983 100644 --- a/libcxx/include/__algorithm/in_found_result.h +++ b/libcxx/include/__algorithm/in_found_result.h @@ -30,7 +30,7 @@ struct in_found_result { template requires convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr operator in_found_result<_InIter2>() const & { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_found_result<_InIter2>() const& { return {in, found}; } diff --git a/libcxx/include/__algorithm/in_fun_result.h b/libcxx/include/__algorithm/in_fun_result.h index 33374eddc30d7..6110c1cf86cd5 100644 --- a/libcxx/include/__algorithm/in_fun_result.h +++ b/libcxx/include/__algorithm/in_fun_result.h @@ -30,7 +30,7 @@ struct in_fun_result { template requires convertible_to && convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr operator in_fun_result<_InIter2, _Func2>() const & { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_fun_result<_InIter2, _Func2>() const& { return {in, fun}; } diff --git a/libcxx/include/__algorithm/in_in_out_result.h b/libcxx/include/__algorithm/in_in_out_result.h index 6b50e0e245643..95ce4f4fd5bd4 100644 --- a/libcxx/include/__algorithm/in_in_out_result.h +++ b/libcxx/include/__algorithm/in_in_out_result.h @@ -31,18 +31,16 @@ struct in_in_out_result { _LIBCPP_NO_UNIQUE_ADDRESS _OutIter1 out; template - requires convertible_to - && convertible_to && convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() const& { + requires convertible_to && convertible_to && + convertible_to + _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() const& { return {in1, in2, out}; } template - requires convertible_to<_InIter1, _InIter3> - && convertible_to<_InIter2, _InIter4> && convertible_to<_OutIter1, _OutIter2> - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() && { + requires convertible_to<_InIter1, _InIter3> && convertible_to<_InIter2, _InIter4> && + convertible_to<_OutIter1, _OutIter2> + _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_out_result<_InIter3, _InIter4, _OutIter2>() && { return {std::move(in1), std::move(in2), std::move(out)}; } }; diff --git a/libcxx/include/__algorithm/in_in_result.h b/libcxx/include/__algorithm/in_in_result.h index 1eceb9de02334..d1d62dae7f670 100644 --- a/libcxx/include/__algorithm/in_in_result.h +++ b/libcxx/include/__algorithm/in_in_result.h @@ -31,15 +31,13 @@ struct in_in_result { template requires convertible_to && convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_in_result<_InIter3, _InIter4>() const & { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_result<_InIter3, _InIter4>() const& { return {in1, in2}; } template requires convertible_to<_InIter1, _InIter3> && convertible_to<_InIter2, _InIter4> - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_in_result<_InIter3, _InIter4>() && { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_in_result<_InIter3, _InIter4>() && { return {std::move(in1), std::move(in2)}; } }; diff --git a/libcxx/include/__algorithm/in_out_out_result.h b/libcxx/include/__algorithm/in_out_out_result.h index 2f7a09b5c3014..1436423687508 100644 --- a/libcxx/include/__algorithm/in_out_out_result.h +++ b/libcxx/include/__algorithm/in_out_out_result.h @@ -30,18 +30,16 @@ struct in_out_out_result { _LIBCPP_NO_UNIQUE_ADDRESS _OutIter2 out2; template - requires convertible_to - && convertible_to && convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() const& { + requires convertible_to && convertible_to && + convertible_to + _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() const& { return {in, out1, out2}; } template - requires convertible_to<_InIter1, _InIter2> - && convertible_to<_OutIter1, _OutIter3> && convertible_to<_OutIter2, _OutIter4> - _LIBCPP_HIDE_FROM_ABI constexpr - operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() && { + requires convertible_to<_InIter1, _InIter2> && convertible_to<_OutIter1, _OutIter3> && + convertible_to<_OutIter2, _OutIter4> + _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_out_result<_InIter2, _OutIter3, _OutIter4>() && { return {std::move(in), std::move(out1), std::move(out2)}; } }; diff --git a/libcxx/include/__algorithm/in_out_result.h b/libcxx/include/__algorithm/in_out_result.h index e4741cb71f66b..a7a986cf8e6c0 100644 --- a/libcxx/include/__algorithm/in_out_result.h +++ b/libcxx/include/__algorithm/in_out_result.h @@ -27,22 +27,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace ranges { -template +template struct in_out_result { _LIBCPP_NO_UNIQUE_ADDRESS _InIter1 in; _LIBCPP_NO_UNIQUE_ADDRESS _OutIter1 out; template requires convertible_to && convertible_to - _LIBCPP_HIDE_FROM_ABI - constexpr operator in_out_result<_InIter2, _OutIter2>() const & { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_result<_InIter2, _OutIter2>() const& { return {in, out}; } template requires convertible_to<_InIter1, _InIter2> && convertible_to<_OutIter1, _OutIter2> - _LIBCPP_HIDE_FROM_ABI - constexpr operator in_out_result<_InIter2, _OutIter2>() && { + _LIBCPP_HIDE_FROM_ABI constexpr operator in_out_result<_InIter2, _OutIter2>() && { return {std::move(in), std::move(out)}; } }; diff --git a/libcxx/include/__algorithm/includes.h b/libcxx/include/__algorithm/includes.h index 88253e2653d27..531752e931756 100644 --- a/libcxx/include/__algorithm/includes.h +++ b/libcxx/include/__algorithm/includes.h @@ -25,12 +25,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__includes(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, - _Comp&& __comp, _Proj1&& __proj1, _Proj2&& __proj2) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __includes( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Comp&& __comp, + _Proj1&& __proj1, + _Proj2&& __proj2) { for (; __first2 != __last2; ++__first1) { - if (__first1 == __last1 || std::__invoke( - __comp, std::__invoke(__proj2, *__first2), std::__invoke(__proj1, *__first1))) + if (__first1 == __last1 || + std::__invoke(__comp, std::__invoke(__proj2, *__first2), std::__invoke(__proj1, *__first1))) return false; if (!std::__invoke(__comp, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) ++__first2; @@ -39,14 +44,14 @@ __includes(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool includes( - _InputIterator1 __first1, - _InputIterator1 __last1, - _InputIterator2 __first2, - _InputIterator2 __last2, - _Compare __comp) { - static_assert(__is_callable<_Compare, decltype(*__first1), decltype(*__first2)>::value, - "Comparator has to be callable"); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +includes(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _Compare __comp) { + static_assert( + __is_callable<_Compare, decltype(*__first1), decltype(*__first2)>::value, "Comparator has to be callable"); return std::__includes( std::move(__first1), diff --git a/libcxx/include/__algorithm/inplace_merge.h b/libcxx/include/__algorithm/inplace_merge.h index d49c2e5cedda2..eb3c0bdbc2db7 100644 --- a/libcxx/include/__algorithm/inplace_merge.h +++ b/libcxx/include/__algorithm/inplace_merge.h @@ -42,54 +42,57 @@ template class __invert // invert the sense of a comparison { private: - _Predicate __p_; + _Predicate __p_; + public: - _LIBCPP_HIDE_FROM_ABI __invert() {} + _LIBCPP_HIDE_FROM_ABI __invert() {} - _LIBCPP_HIDE_FROM_ABI - explicit __invert(_Predicate __p) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI explicit __invert(_Predicate __p) : __p_(__p) {} - template - _LIBCPP_HIDE_FROM_ABI - bool operator()(const _T1& __x) {return !__p_(__x);} + template + _LIBCPP_HIDE_FROM_ABI bool operator()(const _T1& __x) { + return !__p_(__x); + } - template - _LIBCPP_HIDE_FROM_ABI - bool operator()(const _T1& __x, const _T2& __y) {return __p_(__y, __x);} + template + _LIBCPP_HIDE_FROM_ABI bool operator()(const _T1& __x, const _T2& __y) { + return __p_(__y, __x); + } }; -template -_LIBCPP_HIDE_FROM_ABI -void __half_inplace_merge(_InputIterator1 __first1, _Sent1 __last1, - _InputIterator2 __first2, _Sent2 __last2, - _OutputIterator __result, _Compare&& __comp) -{ - for (; __first1 != __last1; ++__result) - { - if (__first2 == __last2) - { - std::__move<_AlgPolicy>(__first1, __last1, __result); - return; - } +template +_LIBCPP_HIDE_FROM_ABI void __half_inplace_merge( + _InputIterator1 __first1, + _Sent1 __last1, + _InputIterator2 __first2, + _Sent2 __last2, + _OutputIterator __result, + _Compare&& __comp) { + for (; __first1 != __last1; ++__result) { + if (__first2 == __last2) { + std::__move<_AlgPolicy>(__first1, __last1, __result); + return; + } - if (__comp(*__first2, *__first1)) - { - *__result = _IterOps<_AlgPolicy>::__iter_move(__first2); - ++__first2; - } - else - { - *__result = _IterOps<_AlgPolicy>::__iter_move(__first1); - ++__first1; - } + if (__comp(*__first2, *__first1)) { + *__result = _IterOps<_AlgPolicy>::__iter_move(__first2); + ++__first2; + } else { + *__result = _IterOps<_AlgPolicy>::__iter_move(__first1); + ++__first1; } - // __first2 through __last2 are already in the right spot. + } + // __first2 through __last2 are already in the right spot. } template -_LIBCPP_HIDE_FROM_ABI -void __buffered_inplace_merge( +_LIBCPP_HIDE_FROM_ABI void __buffered_inplace_merge( _BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, @@ -98,27 +101,25 @@ void __buffered_inplace_merge( typename iterator_traits<_BidirectionalIterator>::difference_type __len2, typename iterator_traits<_BidirectionalIterator>::value_type* __buff) { typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - __destruct_n __d(0); - unique_ptr __h2(__buff, __d); - if (__len1 <= __len2) - { - value_type* __p = __buff; - for (_BidirectionalIterator __i = __first; __i != __middle; __d.template __incr(), (void) ++__i, (void) ++__p) - ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i)); - std::__half_inplace_merge<_AlgPolicy>(__buff, __p, __middle, __last, __first, __comp); - } - else - { - value_type* __p = __buff; - for (_BidirectionalIterator __i = __middle; __i != __last; __d.template __incr(), (void) ++__i, (void) ++__p) - ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i)); - typedef __unconstrained_reverse_iterator<_BidirectionalIterator> _RBi; - typedef __unconstrained_reverse_iterator _Rv; - typedef __invert<_Compare> _Inverted; - std::__half_inplace_merge<_AlgPolicy>(_Rv(__p), _Rv(__buff), - _RBi(__middle), _RBi(__first), - _RBi(__last), _Inverted(__comp)); - } + __destruct_n __d(0); + unique_ptr __h2(__buff, __d); + if (__len1 <= __len2) { + value_type* __p = __buff; + for (_BidirectionalIterator __i = __first; __i != __middle; + __d.template __incr(), (void)++__i, (void)++__p) + ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i)); + std::__half_inplace_merge<_AlgPolicy>(__buff, __p, __middle, __last, __first, __comp); + } else { + value_type* __p = __buff; + for (_BidirectionalIterator __i = __middle; __i != __last; + __d.template __incr(), (void)++__i, (void)++__p) + ::new ((void*)__p) value_type(_IterOps<_AlgPolicy>::__iter_move(__i)); + typedef __unconstrained_reverse_iterator<_BidirectionalIterator> _RBi; + typedef __unconstrained_reverse_iterator _Rv; + typedef __invert<_Compare> _Inverted; + std::__half_inplace_merge<_AlgPolicy>( + _Rv(__p), _Rv(__buff), _RBi(__middle), _RBi(__first), _RBi(__last), _Inverted(__comp)); + } } template @@ -131,107 +132,92 @@ void __inplace_merge( typename iterator_traits<_BidirectionalIterator>::difference_type __len2, typename iterator_traits<_BidirectionalIterator>::value_type* __buff, ptrdiff_t __buff_size) { - using _Ops = _IterOps<_AlgPolicy>; - - typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; - while (true) - { - // if __middle == __last, we're done - if (__len2 == 0) - return; - if (__len1 <= __buff_size || __len2 <= __buff_size) - return std::__buffered_inplace_merge<_AlgPolicy> - (__first, __middle, __last, __comp, __len1, __len2, __buff); - // shrink [__first, __middle) as much as possible (with no moves), returning if it shrinks to 0 - for (; true; ++__first, (void) --__len1) - { - if (__len1 == 0) - return; - if (__comp(*__middle, *__first)) - break; - } - // __first < __middle < __last - // *__first > *__middle - // partition [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last) such that - // all elements in: - // [__first, __m1) <= [__middle, __m2) - // [__middle, __m2) < [__m1, __middle) - // [__m1, __middle) <= [__m2, __last) - // and __m1 or __m2 is in the middle of its range - _BidirectionalIterator __m1; // "median" of [__first, __middle) - _BidirectionalIterator __m2; // "median" of [__middle, __last) - difference_type __len11; // distance(__first, __m1) - difference_type __len21; // distance(__middle, __m2) - // binary search smaller range - if (__len1 < __len2) - { // __len >= 1, __len2 >= 2 - __len21 = __len2 / 2; - __m2 = __middle; - _Ops::advance(__m2, __len21); - __m1 = std::__upper_bound<_AlgPolicy>(__first, __middle, *__m2, __comp, std::__identity()); - __len11 = _Ops::distance(__first, __m1); - } - else - { - if (__len1 == 1) - { // __len1 >= __len2 && __len2 > 0, therefore __len2 == 1 - // It is known *__first > *__middle - _Ops::iter_swap(__first, __middle); - return; - } - // __len1 >= 2, __len2 >= 1 - __len11 = __len1 / 2; - __m1 = __first; - _Ops::advance(__m1, __len11); - __m2 = std::lower_bound(__middle, __last, *__m1, __comp); - __len21 = _Ops::distance(__middle, __m2); - } - difference_type __len12 = __len1 - __len11; // distance(__m1, __middle) - difference_type __len22 = __len2 - __len21; // distance(__m2, __last) - // [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last) - // swap middle two partitions - __middle = std::__rotate<_AlgPolicy>(__m1, __middle, __m2).first; - // __len12 and __len21 now have swapped meanings - // merge smaller range with recursive call and larger with tail recursion elimination - if (__len11 + __len21 < __len12 + __len22) - { - std::__inplace_merge<_AlgPolicy>( - __first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size); - __first = __middle; - __middle = __m2; - __len1 = __len12; - __len2 = __len22; - } - else - { - std::__inplace_merge<_AlgPolicy>( - __middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size); - __last = __middle; - __middle = __m1; - __len1 = __len11; - __len2 = __len21; - } + using _Ops = _IterOps<_AlgPolicy>; + + typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; + while (true) { + // if __middle == __last, we're done + if (__len2 == 0) + return; + if (__len1 <= __buff_size || __len2 <= __buff_size) + return std::__buffered_inplace_merge<_AlgPolicy>(__first, __middle, __last, __comp, __len1, __len2, __buff); + // shrink [__first, __middle) as much as possible (with no moves), returning if it shrinks to 0 + for (; true; ++__first, (void)--__len1) { + if (__len1 == 0) + return; + if (__comp(*__middle, *__first)) + break; } + // __first < __middle < __last + // *__first > *__middle + // partition [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last) such that + // all elements in: + // [__first, __m1) <= [__middle, __m2) + // [__middle, __m2) < [__m1, __middle) + // [__m1, __middle) <= [__m2, __last) + // and __m1 or __m2 is in the middle of its range + _BidirectionalIterator __m1; // "median" of [__first, __middle) + _BidirectionalIterator __m2; // "median" of [__middle, __last) + difference_type __len11; // distance(__first, __m1) + difference_type __len21; // distance(__middle, __m2) + // binary search smaller range + if (__len1 < __len2) { // __len >= 1, __len2 >= 2 + __len21 = __len2 / 2; + __m2 = __middle; + _Ops::advance(__m2, __len21); + __m1 = std::__upper_bound<_AlgPolicy>(__first, __middle, *__m2, __comp, std::__identity()); + __len11 = _Ops::distance(__first, __m1); + } else { + if (__len1 == 1) { // __len1 >= __len2 && __len2 > 0, therefore __len2 == 1 + // It is known *__first > *__middle + _Ops::iter_swap(__first, __middle); + return; + } + // __len1 >= 2, __len2 >= 1 + __len11 = __len1 / 2; + __m1 = __first; + _Ops::advance(__m1, __len11); + __m2 = std::lower_bound(__middle, __last, *__m1, __comp); + __len21 = _Ops::distance(__middle, __m2); + } + difference_type __len12 = __len1 - __len11; // distance(__m1, __middle) + difference_type __len22 = __len2 - __len21; // distance(__m2, __last) + // [__first, __m1) [__m1, __middle) [__middle, __m2) [__m2, __last) + // swap middle two partitions + __middle = std::__rotate<_AlgPolicy>(__m1, __middle, __m2).first; + // __len12 and __len21 now have swapped meanings + // merge smaller range with recursive call and larger with tail recursion elimination + if (__len11 + __len21 < __len12 + __len22) { + std::__inplace_merge<_AlgPolicy>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size); + __first = __middle; + __middle = __m2; + __len1 = __len12; + __len2 = __len22; + } else { + std::__inplace_merge<_AlgPolicy>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size); + __last = __middle; + __middle = __m1; + __len1 = __len11; + __len2 = __len21; + } + } } template -_LIBCPP_HIDE_FROM_ABI -void -__inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, - _Compare&& __comp) -{ - typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; - difference_type __len1 = _IterOps<_AlgPolicy>::distance(__first, __middle); - difference_type __len2 = _IterOps<_AlgPolicy>::distance(__middle, __last); - difference_type __buf_size = std::min(__len1, __len2); -// TODO: Remove the use of std::get_temporary_buffer -_LIBCPP_SUPPRESS_DEPRECATED_PUSH - pair __buf = std::get_temporary_buffer(__buf_size); -_LIBCPP_SUPPRESS_DEPRECATED_POP - unique_ptr __h(__buf.first); - return std::__inplace_merge<_AlgPolicy>( - std::move(__first), std::move(__middle), std::move(__last), __comp, __len1, __len2, __buf.first, __buf.second); +_LIBCPP_HIDE_FROM_ABI void __inplace_merge( + _BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, _Compare&& __comp) { + typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; + typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; + difference_type __len1 = _IterOps<_AlgPolicy>::distance(__first, __middle); + difference_type __len2 = _IterOps<_AlgPolicy>::distance(__middle, __last); + difference_type __buf_size = std::min(__len1, __len2); + // TODO: Remove the use of std::get_temporary_buffer + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + pair __buf = std::get_temporary_buffer(__buf_size); + _LIBCPP_SUPPRESS_DEPRECATED_POP + unique_ptr __h(__buf.first); + return std::__inplace_merge<_AlgPolicy>( + std::move(__first), std::move(__middle), std::move(__last), __comp, __len1, __len2, __buf.first, __buf.second); } template @@ -242,11 +228,9 @@ inline _LIBCPP_HIDE_FROM_ABI void inplace_merge( } template -inline _LIBCPP_HIDE_FROM_ABI -void -inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last) -{ - std::inplace_merge(std::move(__first), std::move(__middle), std::move(__last), __less<>()); +inline _LIBCPP_HIDE_FROM_ABI void +inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last) { + std::inplace_merge(std::move(__first), std::move(__middle), std::move(__last), __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/is_heap.h b/libcxx/include/__algorithm/is_heap.h index dd78031579ec9..0d2d43c2c3abd 100644 --- a/libcxx/include/__algorithm/is_heap.h +++ b/libcxx/include/__algorithm/is_heap.h @@ -22,21 +22,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) -{ - return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp)) == __last; +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { + return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp)) == __last; } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) -{ - return std::is_heap(__first, __last, __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { + return std::is_heap(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/is_heap_until.h b/libcxx/include/__algorithm/is_heap_until.h index 85f70e649f568..1eae3b86b90df 100644 --- a/libcxx/include/__algorithm/is_heap_until.h +++ b/libcxx/include/__algorithm/is_heap_until.h @@ -22,43 +22,39 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator -__is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) -{ - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - difference_type __len = __last - __first; - difference_type __p = 0; - difference_type __c = 1; - _RandomAccessIterator __pp = __first; - while (__c < __len) - { - _RandomAccessIterator __cp = __first + __c; - if (__comp(*__pp, *__cp)) - return __cp; - ++__c; - ++__cp; - if (__c == __len) - return __last; - if (__comp(*__pp, *__cp)) - return __cp; - ++__p; - ++__pp; - __c = 2 * __p + 1; - } - return __last; +__is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) { + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + difference_type __len = __last - __first; + difference_type __p = 0; + difference_type __c = 1; + _RandomAccessIterator __pp = __first; + while (__c < __len) { + _RandomAccessIterator __cp = __first + __c; + if (__comp(*__pp, *__cp)) + return __cp; + ++__c; + ++__cp; + if (__c == __len) + return __last; + if (__comp(*__pp, *__cp)) + return __cp; + ++__p; + ++__pp; + __c = 2 * __p + 1; + } + return __last; } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator -is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) -{ - return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp)); +is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { + return std::__is_heap_until(__first, __last, static_cast<__comp_ref_type<_Compare> >(__comp)); } -template +template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator -is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last) -{ - return std::__is_heap_until(__first, __last, __less<>()); +is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last) { + return std::__is_heap_until(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/is_partitioned.h b/libcxx/include/__algorithm/is_partitioned.h index ab59d3cce5823..71feed3320605 100644 --- a/libcxx/include/__algorithm/is_partitioned.h +++ b/libcxx/include/__algorithm/is_partitioned.h @@ -19,18 +19,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -is_partitioned(_InputIterator __first, _InputIterator __last, _Predicate __pred) -{ - for (; __first != __last; ++__first) - if (!__pred(*__first)) - break; - if ( __first == __last ) - return true; - ++__first; - for (; __first != __last; ++__first) - if (__pred(*__first)) - return false; +is_partitioned(_InputIterator __first, _InputIterator __last, _Predicate __pred) { + for (; __first != __last; ++__first) + if (!__pred(*__first)) + break; + if (__first == __last) return true; + ++__first; + for (; __first != __last; ++__first) + if (__pred(*__first)) + return false; + return true; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/is_permutation.h b/libcxx/include/__algorithm/is_permutation.h index 105a0732283c9..4226151222bbd 100644 --- a/libcxx/include/__algorithm/is_permutation.h +++ b/libcxx/include/__algorithm/is_permutation.h @@ -37,18 +37,24 @@ struct _ConstTimeDistance : false_type {}; #if _LIBCPP_STD_VER >= 20 template -struct _ConstTimeDistance<_Iter1, _Sent1, _Iter2, _Sent2, __enable_if_t< - sized_sentinel_for<_Sent1, _Iter1> && - sized_sentinel_for<_Sent2, _Iter2> ->> : true_type {}; +struct _ConstTimeDistance<_Iter1, + _Sent1, + _Iter2, + _Sent2, + __enable_if_t< sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2> >> + : true_type {}; #else template -struct _ConstTimeDistance<_Iter1, _Iter1, _Iter2, _Iter2, __enable_if_t< - is_same::iterator_category, random_access_iterator_tag>::value && - is_same::iterator_category, random_access_iterator_tag>::value -> > : true_type {}; +struct _ConstTimeDistance< + _Iter1, + _Iter1, + _Iter2, + _Iter2, + __enable_if_t< is_same::iterator_category, random_access_iterator_tag>::value && + is_same::iterator_category, random_access_iterator_tag>::value > > + : true_type {}; #endif // _LIBCPP_STD_VER >= 20 @@ -56,11 +62,21 @@ struct _ConstTimeDistance<_Iter1, _Iter1, _Iter2, _Iter2, __enable_if_t< // For each element in [f1, l1) see if there are the same number of equal elements in [f2, l2) template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__is_permutation_impl(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, - _Pred&& __pred, _Proj1&& __proj1, _Proj2&& __proj2) { + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2, + class _Pred> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation_impl( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred&& __pred, + _Proj1&& __proj1, + _Proj2&& __proj2) { using _D1 = __iter_diff_t<_Iter1>; for (auto __i = __first1; __i != __last1; ++__i) { @@ -97,9 +113,8 @@ __is_permutation_impl(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 _ // 2+1 iterators, predicate. Not used by range algorithms. template -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__is_permutation(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, - _BinaryPredicate&& __pred) { +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation( + _ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _BinaryPredicate&& __pred) { // Shorten sequences as much as possible by lopping of any equal prefix. for (; __first1 != __last1; ++__first1, (void)++__first2) { if (!__pred(*__first1, *__first2)) @@ -111,24 +126,39 @@ __is_permutation(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterato // __first1 != __last1 && *__first1 != *__first2 using _D1 = __iter_diff_t<_ForwardIterator1>; - _D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1); + _D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1); if (__l1 == _D1(1)) return false; auto __last2 = _IterOps<_AlgPolicy>::next(__first2, __l1); return std::__is_permutation_impl<_AlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), - __pred, __identity(), __identity()); + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + __pred, + __identity(), + __identity()); } // 2+2 iterators, predicate, non-constant time `distance`. template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__is_permutation(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, - _Pred&& __pred, _Proj1&& __proj1, _Proj2&& __proj2, - /*_ConstTimeDistance=*/false_type) { + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2, + class _Pred> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred&& __pred, + _Proj1&& __proj1, + _Proj2&& __proj2, + /*_ConstTimeDistance=*/false_type) { // Shorten sequences as much as possible by lopping of any equal prefix. while (__first1 != __last1 && __first2 != __last2) { if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) @@ -143,44 +173,73 @@ __is_permutation(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last return false; using _D1 = __iter_diff_t<_Iter1>; - _D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1); + _D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1); using _D2 = __iter_diff_t<_Iter2>; - _D2 __l2 = _IterOps<_AlgPolicy>::distance(__first2, __last2); + _D2 __l2 = _IterOps<_AlgPolicy>::distance(__first2, __last2); if (__l1 != __l2) return false; return std::__is_permutation_impl<_AlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), - __pred, __proj1, __proj2); + std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2); } // 2+2 iterators, predicate, specialization for constant-time `distance` call. template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__is_permutation(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, - _Pred&& __pred, _Proj1&& __proj1, _Proj2&& __proj2, - /*_ConstTimeDistance=*/true_type) { + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2, + class _Pred> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred&& __pred, + _Proj1&& __proj1, + _Proj2&& __proj2, + /*_ConstTimeDistance=*/true_type) { if (std::distance(__first1, __last1) != std::distance(__first2, __last2)) return false; return std::__is_permutation<_AlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), - __pred, __proj1, __proj2, + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + __pred, + __proj1, + __proj2, /*_ConstTimeDistance=*/false_type()); } // 2+2 iterators, predicate template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__is_permutation(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, - _Pred&& __pred, _Proj1&& __proj1, _Proj2&& __proj2) { + class _Iter1, + class _Sent1, + class _Iter2, + class _Sent2, + class _Proj1, + class _Proj2, + class _Pred> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred&& __pred, + _Proj1&& __proj1, + _Proj2&& __proj2) { return std::__is_permutation<_AlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), - __pred, __proj1, __proj2, + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + __pred, + __proj1, + __proj2, _ConstTimeDistance<_Iter1, _Sent1, _Iter2, _Sent2>()); } @@ -188,14 +247,12 @@ __is_permutation(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last // 2+1 iterators, predicate template -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, - _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool is_permutation( + _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __pred) { static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value, - "The predicate has to be callable"); + "The predicate has to be callable"); - return std::__is_permutation<_ClassicAlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), __pred); + return std::__is_permutation<_ClassicAlgPolicy>(std::move(__first1), std::move(__last1), std::move(__first2), __pred); } // 2+1 iterators @@ -223,15 +280,23 @@ _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 // 2+2 iterators, predicate template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, - _ForwardIterator2 __last2, _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool is_permutation( + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate __pred) { static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value, - "The predicate has to be callable"); + "The predicate has to be callable"); return std::__is_permutation<_ClassicAlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), - __pred, __identity(), __identity()); + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + __pred, + __identity(), + __identity()); } #endif // _LIBCPP_STD_VER >= 14 diff --git a/libcxx/include/__algorithm/is_sorted.h b/libcxx/include/__algorithm/is_sorted.h index 0003097b362c5..1874cace882c1 100644 --- a/libcxx/include/__algorithm/is_sorted.h +++ b/libcxx/include/__algorithm/is_sorted.h @@ -22,21 +22,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -is_sorted(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp) == __last; +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +is_sorted(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp) == __last; } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -is_sorted(_ForwardIterator __first, _ForwardIterator __last) -{ - return std::is_sorted(__first, __last, __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +is_sorted(_ForwardIterator __first, _ForwardIterator __last) { + return std::is_sorted(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/is_sorted_until.h b/libcxx/include/__algorithm/is_sorted_until.h index aeb3f1c8be989..7450440df2d8b 100644 --- a/libcxx/include/__algorithm/is_sorted_until.h +++ b/libcxx/include/__algorithm/is_sorted_until.h @@ -22,33 +22,28 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -__is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - if (__first != __last) - { - _ForwardIterator __i = __first; - while (++__i != __last) - { - if (__comp(*__i, *__first)) - return __i; - __first = __i; - } +__is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + if (__first != __last) { + _ForwardIterator __i = __first; + while (++__i != __last) { + if (__comp(*__i, *__first)) + return __i; + __first = __i; } - return __last; + } + return __last; } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp); +is_sorted_until(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + return std::__is_sorted_until<__comp_ref_type<_Compare> >(__first, __last, __comp); } -template +template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -is_sorted_until(_ForwardIterator __first, _ForwardIterator __last) -{ - return std::is_sorted_until(__first, __last, __less<>()); +is_sorted_until(_ForwardIterator __first, _ForwardIterator __last) { + return std::is_sorted_until(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/iter_swap.h b/libcxx/include/__algorithm/iter_swap.h index f647e74324674..a1412e5d8720b 100644 --- a/libcxx/include/__algorithm/iter_swap.h +++ b/libcxx/include/__algorithm/iter_swap.h @@ -20,8 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void iter_swap(_ForwardIterator1 __a, - _ForwardIterator2 __b) +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void iter_swap(_ForwardIterator1 __a, _ForwardIterator2 __b) // _NOEXCEPT_(_NOEXCEPT_(swap(*__a, *__b))) _NOEXCEPT_(_NOEXCEPT_(swap(*std::declval<_ForwardIterator1>(), *std::declval<_ForwardIterator2>()))) { swap(*__a, *__b); diff --git a/libcxx/include/__algorithm/iterator_operations.h b/libcxx/include/__algorithm/iterator_operations.h index e6176da4f5606..5cf13f0a3f292 100644 --- a/libcxx/include/__algorithm/iterator_operations.h +++ b/libcxx/include/__algorithm/iterator_operations.h @@ -38,14 +38,14 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -template struct _IterOps; +template +struct _IterOps; #if _LIBCPP_STD_VER >= 20 struct _RangeAlgPolicy {}; template <> struct _IterOps<_RangeAlgPolicy> { - template using __value_type = iter_value_t<_Iter>; @@ -55,12 +55,12 @@ struct _IterOps<_RangeAlgPolicy> { template using __difference_type = iter_difference_t<_Iter>; - static constexpr auto advance = ranges::advance; - static constexpr auto distance = ranges::distance; - static constexpr auto __iter_move = ranges::iter_move; - static constexpr auto iter_swap = ranges::iter_swap; - static constexpr auto next = ranges::next; - static constexpr auto prev = ranges::prev; + static constexpr auto advance = ranges::advance; + static constexpr auto distance = ranges::distance; + static constexpr auto __iter_move = ranges::iter_move; + static constexpr auto iter_swap = ranges::iter_swap; + static constexpr auto next = ranges::next; + static constexpr auto prev = ranges::prev; static constexpr auto __advance_to = ranges::advance; }; @@ -70,7 +70,6 @@ struct _ClassicAlgPolicy {}; template <> struct _IterOps<_ClassicAlgPolicy> { - template using __value_type = typename iterator_traits<_Iter>::value_type; @@ -82,15 +81,14 @@ struct _IterOps<_ClassicAlgPolicy> { // advance template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - static void advance(_Iter& __iter, _Distance __count) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void advance(_Iter& __iter, _Distance __count) { std::advance(__iter, __count); } // distance template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - static typename iterator_traits<_Iter>::difference_type distance(_Iter __first, _Iter __last) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static typename iterator_traits<_Iter>::difference_type + distance(_Iter __first, _Iter __last) { return std::distance(__first, __last); } @@ -101,9 +99,9 @@ struct _IterOps<_ClassicAlgPolicy> { using __move_t = decltype(std::move(*std::declval<_Iter&>())); template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - static void __validate_iter_reference() { - static_assert(is_same<__deref_t<_Iter>, typename iterator_traits<__remove_cvref_t<_Iter> >::reference>::value, + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void __validate_iter_reference() { + static_assert( + is_same<__deref_t<_Iter>, typename iterator_traits<__remove_cvref_t<_Iter> >::reference>::value, "It looks like your iterator's `iterator_traits::reference` does not match the return type of " "dereferencing the iterator, i.e., calling `*it`. This is undefined behavior according to [input.iterators] " "and can lead to dangling reference issues at runtime, so we are flagging this."); @@ -112,10 +110,10 @@ struct _IterOps<_ClassicAlgPolicy> { // iter_move template >::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static - // If the result of dereferencing `_Iter` is a reference type, deduce the result of calling `std::move` on it. Note - // that the C++03 mode doesn't support `decltype(auto)` as the return type. - __move_t<_Iter> - __iter_move(_Iter&& __i) { + // If the result of dereferencing `_Iter` is a reference type, deduce the result of calling `std::move` on it. + // Note that the C++03 mode doesn't support `decltype(auto)` as the return type. + __move_t<_Iter> + __iter_move(_Iter&& __i) { __validate_iter_reference<_Iter>(); return std::move(*std::forward<_Iter>(__i)); @@ -123,11 +121,11 @@ struct _IterOps<_ClassicAlgPolicy> { template >::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static - // If the result of dereferencing `_Iter` is a value type, deduce the return value of this function to also be a - // value -- otherwise, after `operator*` returns a temporary, this function would return a dangling reference to that - // temporary. Note that the C++03 mode doesn't support `auto` as the return type. - __deref_t<_Iter> - __iter_move(_Iter&& __i) { + // If the result of dereferencing `_Iter` is a value type, deduce the return value of this function to also be a + // value -- otherwise, after `operator*` returns a temporary, this function would return a dangling reference to + // that temporary. Note that the C++03 mode doesn't support `auto` as the return type. + __deref_t<_Iter> + __iter_move(_Iter&& __i) { __validate_iter_reference<_Iter>(); return *std::forward<_Iter>(__i); @@ -135,36 +133,31 @@ struct _IterOps<_ClassicAlgPolicy> { // iter_swap template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - static void iter_swap(_Iter1&& __a, _Iter2&& __b) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static void iter_swap(_Iter1&& __a, _Iter2&& __b) { std::iter_swap(std::forward<_Iter1>(__a), std::forward<_Iter2>(__b)); } // next template - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 - _Iterator next(_Iterator, _Iterator __last) { + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iterator next(_Iterator, _Iterator __last) { return __last; } template - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 - __remove_cvref_t<_Iter> next(_Iter&& __it, - typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) { + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 __remove_cvref_t<_Iter> + next(_Iter&& __it, typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) { return std::next(std::forward<_Iter>(__it), __n); } // prev template - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 - __remove_cvref_t<_Iter> prev(_Iter&& __iter, - typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) { + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 __remove_cvref_t<_Iter> + prev(_Iter&& __iter, typename iterator_traits<__remove_cvref_t<_Iter> >::difference_type __n = 1) { return std::prev(std::forward<_Iter>(__iter), __n); } template - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 - void __advance_to(_Iter& __first, _Iter __last) { + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 void __advance_to(_Iter& __first, _Iter __last) { __first = __last; } }; diff --git a/libcxx/include/__algorithm/lexicographical_compare.h b/libcxx/include/__algorithm/lexicographical_compare.h index 1ee189a259025..3efd8e24bf6c9 100644 --- a/libcxx/include/__algorithm/lexicographical_compare.h +++ b/libcxx/include/__algorithm/lexicographical_compare.h @@ -21,38 +21,35 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool -__lexicographical_compare(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp) -{ - for (; __first2 != __last2; ++__first1, (void) ++__first2) - { - if (__first1 == __last1 || __comp(*__first1, *__first2)) - return true; - if (__comp(*__first2, *__first1)) - return false; - } - return false; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __lexicographical_compare( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _Compare __comp) { + for (; __first2 != __last2; ++__first1, (void)++__first2) { + if (__first1 == __last1 || __comp(*__first1, *__first2)) + return true; + if (__comp(*__first2, *__first1)) + return false; + } + return false; } template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -lexicographical_compare(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp) -{ - return std::__lexicographical_compare<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __comp); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool lexicographical_compare( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _Compare __comp) { + return std::__lexicographical_compare<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __comp); } template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -lexicographical_compare(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2) -{ - return std::lexicographical_compare(__first1, __last1, __first2, __last2, __less<>()); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool lexicographical_compare( + _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) { + return std::lexicographical_compare(__first1, __last1, __first2, __last2, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/lower_bound.h b/libcxx/include/__algorithm/lower_bound.h index 91c3bdaafd0cf..8f57f3592c4b2 100644 --- a/libcxx/include/__algorithm/lower_bound.h +++ b/libcxx/include/__algorithm/lower_bound.h @@ -28,8 +28,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_Iter __lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Iter +__lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) { auto __len = _IterOps<_AlgPolicy>::distance(__first, __last); while (__len != 0) { @@ -47,17 +47,16 @@ _Iter __lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __ } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) { - static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, - "The comparator has to be callable"); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator +lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, _Compare __comp) { + static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value, "The comparator has to be callable"); auto __proj = std::__identity(); return std::__lower_bound<_ClassicAlgPolicy>(__first, __last, __value, __comp, __proj); } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator +lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { return std::lower_bound(__first, __last, __value, __less<>()); } diff --git a/libcxx/include/__algorithm/make_heap.h b/libcxx/include/__algorithm/make_heap.h index eaf9259c0b315..e8f0cdb27333a 100644 --- a/libcxx/include/__algorithm/make_heap.h +++ b/libcxx/include/__algorithm/make_heap.h @@ -27,29 +27,29 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) { __comp_ref_type<_Compare> __comp_ref = __comp; using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; - difference_type __n = __last - __first; + difference_type __n = __last - __first; if (__n > 1) { // start from the first parent, there is no need to consider children for (difference_type __start = (__n - 2) / 2; __start >= 0; --__start) { - std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, __first + __start); + std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, __first + __start); } } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { std::__make_heap<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::make_heap(std::move(__first), std::move(__last), __less<>()); } diff --git a/libcxx/include/__algorithm/make_projected.h b/libcxx/include/__algorithm/make_projected.h index 3a86701118146..bb7bc7e8c0b58 100644 --- a/libcxx/include/__algorithm/make_projected.h +++ b/libcxx/include/__algorithm/make_projected.h @@ -36,44 +36,38 @@ struct _ProjectedPred { : __pred(__pred_arg), __proj(__proj_arg) {} template - typename __invoke_of<_Pred&, - decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_Tp>())) - >::type - _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI operator()(_Tp&& __v) const { + typename __invoke_of<_Pred&, decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_Tp>())) >:: + type _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI + operator()(_Tp&& __v) const { return std::__invoke(__pred, std::__invoke(__proj, std::forward<_Tp>(__v))); } template typename __invoke_of<_Pred&, decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_T1>())), - decltype(std::__invoke(std::declval<_Proj&>(), std::declval<_T2>())) - >::type - _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI operator()(_T1&& __lhs, _T2&& __rhs) const { - return std::__invoke(__pred, - std::__invoke(__proj, std::forward<_T1>(__lhs)), - std::__invoke(__proj, std::forward<_T2>(__rhs))); + decltype(std::__invoke(std::declval<_Proj&>(), + std::declval<_T2>())) >::type _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI + operator()(_T1&& __lhs, _T2&& __rhs) const { + return std::__invoke( + __pred, std::__invoke(__proj, std::forward<_T1>(__lhs)), std::__invoke(__proj, std::forward<_T2>(__rhs))); } - }; -template >::value && - __is_identity<__decay_t<_Proj> >::value), - int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ProjectedPred<_Pred, _Proj> -__make_projected(_Pred& __pred, _Proj& __proj) { +template < + class _Pred, + class _Proj, + __enable_if_t >::value && __is_identity<__decay_t<_Proj> >::value), int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ProjectedPred<_Pred, _Proj> __make_projected(_Pred& __pred, _Proj& __proj) { return _ProjectedPred<_Pred, _Proj>(__pred, __proj); } // Avoid creating the functor and just use the pristine comparator -- for certain algorithms, this would enable // optimizations that rely on the type of the comparator. Additionally, this results in less layers of indirection in // the call stack when the comparator is invoked, even in an unoptimized build. -template >::value && - __is_identity<__decay_t<_Proj> >::value, - int> = 0> +template < + class _Pred, + class _Proj, + __enable_if_t >::value && __is_identity<__decay_t<_Proj> >::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Pred& __make_projected(_Pred& __pred, _Proj&) { return __pred; } @@ -87,8 +81,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace ranges { template -_LIBCPP_HIDE_FROM_ABI constexpr -decltype(auto) __make_projected_comp(_Comp& __comp, _Proj1& __proj1, _Proj2& __proj2) { +_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) __make_projected_comp(_Comp& __comp, _Proj1& __proj1, _Proj2& __proj2) { if constexpr (__is_identity>::value && __is_identity>::value && !is_member_pointer_v>) { // Avoid creating the lambda and just use the pristine comparator -- for certain algorithms, this would enable @@ -98,8 +91,8 @@ decltype(auto) __make_projected_comp(_Comp& __comp, _Proj1& __proj1, _Proj2& __p } else { return [&](auto&& __lhs, auto&& __rhs) -> bool { return std::invoke(__comp, - std::invoke(__proj1, std::forward(__lhs)), - std::invoke(__proj2, std::forward(__rhs))); + std::invoke(__proj1, std::forward(__lhs)), + std::invoke(__proj2, std::forward(__rhs))); }; } } diff --git a/libcxx/include/__algorithm/max.h b/libcxx/include/__algorithm/max.h index 183a4e9cac062..8171677f155c9 100644 --- a/libcxx/include/__algorithm/max.h +++ b/libcxx/include/__algorithm/max.h @@ -25,41 +25,28 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp& -max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) -{ - return __comp(__a, __b) ? __b : __a; +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& +max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) { + return __comp(__a, __b) ? __b : __a; } template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp& -max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) -{ - return std::max(__a, __b, __less<>()); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& +max(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) { + return std::max(__a, __b, __less<>()); } #ifndef _LIBCPP_CXX03_LANG -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp -max(initializer_list<_Tp> __t, _Compare __comp) -{ - return *std::__max_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp +max(initializer_list<_Tp> __t, _Compare __comp) { + return *std::__max_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp); } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp -max(initializer_list<_Tp> __t) -{ - return *std::max_element(__t.begin(), __t.end(), __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp max(initializer_list<_Tp> __t) { + return *std::max_element(__t.begin(), __t.end(), __less<>()); } #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__algorithm/max_element.h b/libcxx/include/__algorithm/max_element.h index a2dc9707c00b8..f1d4f1cd0938c 100644 --- a/libcxx/include/__algorithm/max_element.h +++ b/libcxx/include/__algorithm/max_element.h @@ -22,33 +22,28 @@ _LIBCPP_BEGIN_NAMESPACE_STD template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -__max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - static_assert(__has_forward_iterator_category<_ForwardIterator>::value, - "std::max_element requires a ForwardIterator"); - if (__first != __last) - { - _ForwardIterator __i = __first; - while (++__i != __last) - if (__comp(*__first, *__i)) - __first = __i; - } - return __first; +__max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + static_assert( + __has_forward_iterator_category<_ForwardIterator>::value, "std::max_element requires a ForwardIterator"); + if (__first != __last) { + _ForwardIterator __i = __first; + while (++__i != __last) + if (__comp(*__first, *__i)) + __first = __i; + } + return __first; } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - return std::__max_element<__comp_ref_type<_Compare> >(__first, __last, __comp); +max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + return std::__max_element<__comp_ref_type<_Compare> >(__first, __last, __comp); } - template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -max_element(_ForwardIterator __first, _ForwardIterator __last) -{ - return std::max_element(__first, __last, __less<>()); +max_element(_ForwardIterator __first, _ForwardIterator __last) { + return std::max_element(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/merge.h b/libcxx/include/__algorithm/merge.h index 8a6021f1880b2..bad663c4b9f10 100644 --- a/libcxx/include/__algorithm/merge.h +++ b/libcxx/include/__algorithm/merge.h @@ -22,45 +22,46 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -__merge(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp) -{ - for (; __first1 != __last1; ++__result) - { - if (__first2 == __last2) - return std::copy(__first1, __last1, __result); - if (__comp(*__first2, *__first1)) - { - *__result = *__first2; - ++__first2; - } - else - { - *__result = *__first1; - ++__first1; - } +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator __merge( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _OutputIterator __result, + _Compare __comp) { + for (; __first1 != __last1; ++__result) { + if (__first2 == __last2) + return std::copy(__first1, __last1, __result); + if (__comp(*__first2, *__first1)) { + *__result = *__first2; + ++__first2; + } else { + *__result = *__first1; + ++__first1; } - return std::copy(__first2, __last2, __result); + } + return std::copy(__first2, __last2, __result); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -merge(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp) -{ - return std::__merge<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __result, __comp); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +merge(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _OutputIterator __result, + _Compare __comp) { + return std::__merge<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __result, __comp); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -merge(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result) -{ - return std::merge(__first1, __last1, __first2, __last2, __result, __less<>()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +merge(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _OutputIterator __result) { + return std::merge(__first1, __last1, __first2, __last2, __result, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/min.h b/libcxx/include/__algorithm/min.h index 58574517132f2..919508486fd5b 100644 --- a/libcxx/include/__algorithm/min.h +++ b/libcxx/include/__algorithm/min.h @@ -25,41 +25,28 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp& -min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) -{ - return __comp(__b, __a) ? __b : __a; +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& +min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) { + return __comp(__b, __a) ? __b : __a; } template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp& -min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) -{ - return std::min(__a, __b, __less<>()); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& +min(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) { + return std::min(__a, __b, __less<>()); } #ifndef _LIBCPP_CXX03_LANG -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp -min(initializer_list<_Tp> __t, _Compare __comp) -{ - return *std::__min_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp +min(initializer_list<_Tp> __t, _Compare __comp) { + return *std::__min_element<__comp_ref_type<_Compare> >(__t.begin(), __t.end(), __comp); } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp -min(initializer_list<_Tp> __t) -{ - return *std::min_element(__t.begin(), __t.end(), __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp min(initializer_list<_Tp> __t) { + return *std::min_element(__t.begin(), __t.end(), __less<>()); } #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__algorithm/min_element.h b/libcxx/include/__algorithm/min_element.h index 0ce7af69333ff..c576d665601db 100644 --- a/libcxx/include/__algorithm/min_element.h +++ b/libcxx/include/__algorithm/min_element.h @@ -28,8 +28,8 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Iter __min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter +__min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) { if (__first == __last) return __first; @@ -42,29 +42,26 @@ _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) { } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Iter __min_element(_Iter __first, _Sent __last, _Comp __comp) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp) { auto __proj = __identity(); return std::__min_element<_Comp>(std::move(__first), std::move(__last), __comp, __proj); } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) -{ - static_assert(__has_forward_iterator_category<_ForwardIterator>::value, - "std::min_element requires a ForwardIterator"); - static_assert(__is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, - "The comparator has to be callable"); +min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { + static_assert( + __has_forward_iterator_category<_ForwardIterator>::value, "std::min_element requires a ForwardIterator"); + static_assert( + __is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, "The comparator has to be callable"); return std::__min_element<__comp_ref_type<_Compare> >(std::move(__first), std::move(__last), __comp); } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -min_element(_ForwardIterator __first, _ForwardIterator __last) -{ - return std::min_element(__first, __last, __less<>()); +min_element(_ForwardIterator __first, _ForwardIterator __last) { + return std::min_element(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/min_max_result.h b/libcxx/include/__algorithm/min_max_result.h index ef2d99038087b..e988df7c114ee 100644 --- a/libcxx/include/__algorithm/min_max_result.h +++ b/libcxx/include/__algorithm/min_max_result.h @@ -34,7 +34,7 @@ struct min_max_result { template requires convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr operator min_max_result<_T2>() const & { + _LIBCPP_HIDE_FROM_ABI constexpr operator min_max_result<_T2>() const& { return {min, max}; } diff --git a/libcxx/include/__algorithm/minmax.h b/libcxx/include/__algorithm/minmax.h index 115bf93f90bad..5227b88571754 100644 --- a/libcxx/include/__algorithm/minmax.h +++ b/libcxx/include/__algorithm/minmax.h @@ -23,43 +23,33 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair -minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) -{ - return __comp(__b, __a) ? pair(__b, __a) : - pair(__a, __b); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair +minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b, _Compare __comp) { + return __comp(__b, __a) ? pair(__b, __a) : pair(__a, __b); } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair -minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) -{ - return std::minmax(__a, __b, __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair +minmax(_LIBCPP_LIFETIMEBOUND const _Tp& __a, _LIBCPP_LIFETIMEBOUND const _Tp& __b) { + return std::minmax(__a, __b, __less<>()); } #ifndef _LIBCPP_CXX03_LANG -template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Tp, _Tp> minmax(initializer_list<_Tp> __t, _Compare __comp) { - static_assert(__is_callable<_Compare, _Tp, _Tp>::value, "The comparator has to be callable"); - __identity __proj; - auto __ret = std::__minmax_element_impl(__t.begin(), __t.end(), __comp, __proj); - return pair<_Tp, _Tp>(*__ret.first, *__ret.second); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Tp, _Tp> +minmax(initializer_list<_Tp> __t, _Compare __comp) { + static_assert(__is_callable<_Compare, _Tp, _Tp>::value, "The comparator has to be callable"); + __identity __proj; + auto __ret = std::__minmax_element_impl(__t.begin(), __t.end(), __comp, __proj); + return pair<_Tp, _Tp>(*__ret.first, *__ret.second); } -template -_LIBCPP_NODISCARD_EXT inline -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Tp, _Tp> -minmax(initializer_list<_Tp> __t) -{ - return std::minmax(__t, __less<>()); +template +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Tp, _Tp> +minmax(initializer_list<_Tp> __t) { + return std::minmax(__t, __less<>()); } #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__algorithm/minmax_element.h b/libcxx/include/__algorithm/minmax_element.h index 5bcaf8354d9ff..ff8cda321cef4 100644 --- a/libcxx/include/__algorithm/minmax_element.h +++ b/libcxx/include/__algorithm/minmax_element.h @@ -29,19 +29,18 @@ class _MinmaxElementLessFunc { _Proj& __proj_; public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - _MinmaxElementLessFunc(_Comp& __comp, _Proj& __proj) : __comp_(__comp), __proj_(__proj) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _MinmaxElementLessFunc(_Comp& __comp, _Proj& __proj) + : __comp_(__comp), __proj_(__proj) {} template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - bool operator()(_Iter& __it1, _Iter& __it2) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(_Iter& __it1, _Iter& __it2) { return std::__invoke(__comp_, std::__invoke(__proj_, *__it1), std::__invoke(__proj_, *__it2)); } }; template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter, _Iter> __minmax_element_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter> +__minmax_element_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) { auto __less = _MinmaxElementLessFunc<_Comp, _Proj>(__comp, __proj); pair<_Iter, _Iter> __result(__first, __first); @@ -66,8 +65,8 @@ pair<_Iter, _Iter> __minmax_element_impl(_Iter __first, _Sent __last, _Comp& __c if (__less(__first, __i)) { if (__less(__first, __result.first)) __result.first = __first; - if (!__less(__i, __result.second)) - __result.second = __i; + if (!__less(__i, __result.second)) + __result.second = __i; } else { if (__less(__i, __result.first)) __result.first = __i; @@ -80,21 +79,21 @@ pair<_Iter, _Iter> __minmax_element_impl(_Iter __first, _Sent __last, _Comp& __c } template -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_ForwardIterator, _ForwardIterator> +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_ForwardIterator, _ForwardIterator> minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) { - static_assert(__has_forward_iterator_category<_ForwardIterator>::value, - "std::minmax_element requires a ForwardIterator"); - static_assert(__is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, - "The comparator has to be callable"); + static_assert( + __has_forward_iterator_category<_ForwardIterator>::value, "std::minmax_element requires a ForwardIterator"); + static_assert( + __is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, "The comparator has to be callable"); auto __proj = __identity(); return std::__minmax_element_impl(__first, __last, __comp, __proj); } template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_ForwardIterator, _ForwardIterator> minmax_element(_ForwardIterator __first, _ForwardIterator __last) { - return std::minmax_element(__first, __last, __less<>()); + pair<_ForwardIterator, _ForwardIterator> + minmax_element(_ForwardIterator __first, _ForwardIterator __last) { + return std::minmax_element(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index 8330d21d56f3b..d345b6048a7e9 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -22,9 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> - mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> +mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) { for (; __first1 != __last1; ++__first1, (void)++__first2) if (!__pred(*__first1, *__first2)) break; @@ -32,18 +31,19 @@ _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> - mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> +mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2) { return std::mismatch(__first1, __last1, __first2, __equal_to()); } #if _LIBCPP_STD_VER >= 14 template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> - mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, - _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> +mismatch(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _BinaryPredicate __pred) { for (; __first1 != __last1 && __first2 != __last2; ++__first1, (void)++__first2) if (!__pred(*__first1, *__first2)) break; @@ -51,9 +51,8 @@ _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> - mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> +mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) { return std::mismatch(__first1, __last1, __first2, __last2, __equal_to()); } #endif diff --git a/libcxx/include/__algorithm/move.h b/libcxx/include/__algorithm/move.h index e0da07117e66c..dba6d487fff77 100644 --- a/libcxx/include/__algorithm/move.h +++ b/libcxx/include/__algorithm/move.h @@ -99,8 +99,7 @@ struct __move_loop { struct __move_trivial { // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. - template ::value, int> = 0> + template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> operator()(_In* __first, _In* __last, _Out* __result) const { return std::__copy_trivial_impl(__first, __last, __result); diff --git a/libcxx/include/__algorithm/move_backward.h b/libcxx/include/__algorithm/move_backward.h index 8151e4ef3243f..aeedf4241dce9 100644 --- a/libcxx/include/__algorithm/move_backward.h +++ b/libcxx/include/__algorithm/move_backward.h @@ -108,8 +108,7 @@ struct __move_backward_loop { struct __move_backward_trivial { // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. - template ::value, int> = 0> + template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> operator()(_In* __first, _In* __last, _Out* __result) const { return std::__copy_backward_trivial_impl(__first, __last, __result); @@ -120,7 +119,8 @@ template __move_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result) { static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value && - std::is_copy_constructible<_BidirectionalIterator1>::value, "Iterators must be copy constructible."); + std::is_copy_constructible<_BidirectionalIterator1>::value, + "Iterators must be copy constructible."); return std::__dispatch_copy_or_move<_AlgPolicy, __move_backward_loop<_AlgPolicy>, __move_backward_trivial>( std::move(__first), std::move(__last), std::move(__result)); diff --git a/libcxx/include/__algorithm/next_permutation.h b/libcxx/include/__algorithm/next_permutation.h index ebaed15ae2ad0..d66ea9b973453 100644 --- a/libcxx/include/__algorithm/next_permutation.h +++ b/libcxx/include/__algorithm/next_permutation.h @@ -26,50 +26,43 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, bool> -__next_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) -{ - using _Result = pair<_BidirectionalIterator, bool>; +__next_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) { + using _Result = pair<_BidirectionalIterator, bool>; - _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); - _BidirectionalIterator __i = __last_iter; - if (__first == __last || __first == --__i) - return _Result(std::move(__last_iter), false); + _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); + _BidirectionalIterator __i = __last_iter; + if (__first == __last || __first == --__i) + return _Result(std::move(__last_iter), false); - while (true) - { - _BidirectionalIterator __ip1 = __i; - if (__comp(*--__i, *__ip1)) - { - _BidirectionalIterator __j = __last_iter; - while (!__comp(*__i, *--__j)) - ; - _IterOps<_AlgPolicy>::iter_swap(__i, __j); - std::__reverse<_AlgPolicy>(__ip1, __last_iter); - return _Result(std::move(__last_iter), true); - } - if (__i == __first) - { - std::__reverse<_AlgPolicy>(__first, __last_iter); - return _Result(std::move(__last_iter), false); - } + while (true) { + _BidirectionalIterator __ip1 = __i; + if (__comp(*--__i, *__ip1)) { + _BidirectionalIterator __j = __last_iter; + while (!__comp(*__i, *--__j)) + ; + _IterOps<_AlgPolicy>::iter_swap(__i, __j); + std::__reverse<_AlgPolicy>(__ip1, __last_iter); + return _Result(std::move(__last_iter), true); } + if (__i == __first) { + std::__reverse<_AlgPolicy>(__first, __last_iter); + return _Result(std::move(__last_iter), false); + } + } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { return std::__next_permutation<_ClassicAlgPolicy>( - std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp)).second; + std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp)) + .second; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) -{ - return std::next_permutation(__first, __last, __less<>()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) { + return std::next_permutation(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/nth_element.h b/libcxx/include/__algorithm/nth_element.h index 6b3b2bb434d5f..a059705125951 100644 --- a/libcxx/include/__algorithm/nth_element.h +++ b/libcxx/include/__algorithm/nth_element.h @@ -25,224 +25,207 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool -__nth_element_find_guard(_RandomAccessIterator& __i, _RandomAccessIterator& __j, - _RandomAccessIterator __m, _Compare __comp) -{ - // manually guard downward moving __j against __i - while (true) { - if (__i == --__j) { - return false; - } - if (__comp(*__j, *__m)) { - return true; // found guard for downward moving __j, now use unguarded partition - } +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool __nth_element_find_guard( + _RandomAccessIterator& __i, _RandomAccessIterator& __j, _RandomAccessIterator __m, _Compare __comp) { + // manually guard downward moving __j against __i + while (true) { + if (__i == --__j) { + return false; } + if (__comp(*__j, *__m)) { + return true; // found guard for downward moving __j, now use unguarded partition + } + } } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void // NOLINTNEXTLINE(readability-function-cognitive-complexity) -__nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) -{ - using _Ops = _IterOps<_AlgPolicy>; +__nth_element( + _RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) { + using _Ops = _IterOps<_AlgPolicy>; - // _Compare is known to be a reference type - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - const difference_type __limit = 7; - while (true) + // _Compare is known to be a reference type + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + const difference_type __limit = 7; + while (true) { + if (__nth == __last) + return; + difference_type __len = __last - __first; + switch (__len) { + case 0: + case 1: + return; + case 2: + if (__comp(*--__last, *__first)) + _Ops::iter_swap(__first, __last); + return; + case 3: { + _RandomAccessIterator __m = __first; + std::__sort3<_AlgPolicy, _Compare>(__first, ++__m, --__last, __comp); + return; + } + } + if (__len <= __limit) { + std::__selection_sort<_AlgPolicy, _Compare>(__first, __last, __comp); + return; + } + // __len > __limit >= 3 + _RandomAccessIterator __m = __first + __len / 2; + _RandomAccessIterator __lm1 = __last; + unsigned __n_swaps = std::__sort3<_AlgPolicy, _Compare>(__first, __m, --__lm1, __comp); + // *__m is median + // partition [__first, __m) < *__m and *__m <= [__m, __last) + // (this inhibits tossing elements equivalent to __m around unnecessarily) + _RandomAccessIterator __i = __first; + _RandomAccessIterator __j = __lm1; + // j points beyond range to be tested, *__lm1 is known to be <= *__m + // The search going up is known to be guarded but the search coming down isn't. + // Prime the downward search with a guard. + if (!__comp(*__i, *__m)) // if *__first == *__m { - if (__nth == __last) - return; - difference_type __len = __last - __first; - switch (__len) - { - case 0: - case 1: - return; - case 2: - if (__comp(*--__last, *__first)) - _Ops::iter_swap(__first, __last); - return; - case 3: - { - _RandomAccessIterator __m = __first; - std::__sort3<_AlgPolicy, _Compare>(__first, ++__m, --__last, __comp); - return; + // *__first == *__m, *__first doesn't go in first part + if (std::__nth_element_find_guard<_Compare>(__i, __j, __m, __comp)) { + _Ops::iter_swap(__i, __j); + ++__n_swaps; + } else { + // *__first == *__m, *__m <= all other elements + // Partition instead into [__first, __i) == *__first and *__first < [__i, __last) + ++__i; // __first + 1 + __j = __last; + if (!__comp(*__first, *--__j)) { // we need a guard if *__first == *(__last-1) + while (true) { + if (__i == __j) { + return; // [__first, __last) all equivalent elements + } else if (__comp(*__first, *__i)) { + _Ops::iter_swap(__i, __j); + ++__n_swaps; + ++__i; + break; } + ++__i; + } } - if (__len <= __limit) - { - std::__selection_sort<_AlgPolicy, _Compare>(__first, __last, __comp); - return; + // [__first, __i) == *__first and *__first < [__j, __last) and __j == __last - 1 + if (__i == __j) { + return; } - // __len > __limit >= 3 - _RandomAccessIterator __m = __first + __len/2; - _RandomAccessIterator __lm1 = __last; - unsigned __n_swaps = std::__sort3<_AlgPolicy, _Compare>(__first, __m, --__lm1, __comp); - // *__m is median - // partition [__first, __m) < *__m and *__m <= [__m, __last) - // (this inhibits tossing elements equivalent to __m around unnecessarily) - _RandomAccessIterator __i = __first; - _RandomAccessIterator __j = __lm1; - // j points beyond range to be tested, *__lm1 is known to be <= *__m - // The search going up is known to be guarded but the search coming down isn't. - // Prime the downward search with a guard. - if (!__comp(*__i, *__m)) // if *__first == *__m - { - // *__first == *__m, *__first doesn't go in first part - if (std::__nth_element_find_guard<_Compare>(__i, __j, __m, __comp)) { - _Ops::iter_swap(__i, __j); - ++__n_swaps; - } else { - // *__first == *__m, *__m <= all other elements - // Partition instead into [__first, __i) == *__first and *__first < [__i, __last) - ++__i; // __first + 1 - __j = __last; - if (!__comp(*__first, *--__j)) { // we need a guard if *__first == *(__last-1) - while (true) { - if (__i == __j) { - return; // [__first, __last) all equivalent elements - } else if (__comp(*__first, *__i)) { - _Ops::iter_swap(__i, __j); - ++__n_swaps; - ++__i; - break; - } - ++__i; - } - } - // [__first, __i) == *__first and *__first < [__j, __last) and __j == __last - 1 - if (__i == __j) { - return; - } - while (true) { - while (!__comp(*__first, *__i)) { - ++__i; - _LIBCPP_ASSERT_UNCATEGORIZED( - __i != __last, - "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); - } - do { - _LIBCPP_ASSERT_UNCATEGORIZED( - __j != __first, - "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); - --__j; - } while (__comp(*__first, *__j)); - if (__i >= __j) - break; - _Ops::iter_swap(__i, __j); - ++__n_swaps; - ++__i; - } - // [__first, __i) == *__first and *__first < [__i, __last) - // The first part is sorted, - if (__nth < __i) { - return; - } - // __nth_element the second part - // std::__nth_element<_Compare>(__i, __nth, __last, __comp); - __first = __i; - continue; - } + while (true) { + while (!__comp(*__first, *__i)) { + ++__i; + _LIBCPP_ASSERT_UNCATEGORIZED( + __i != __last, + "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); + } + do { + _LIBCPP_ASSERT_UNCATEGORIZED( + __j != __first, + "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); + --__j; + } while (__comp(*__first, *__j)); + if (__i >= __j) + break; + _Ops::iter_swap(__i, __j); + ++__n_swaps; + ++__i; } - ++__i; - // j points beyond range to be tested, *__lm1 is known to be <= *__m - // if not yet partitioned... - if (__i < __j) - { - // known that *(__i - 1) < *__m - while (true) - { - // __m still guards upward moving __i - while (__comp(*__i, *__m)) { - ++__i; - _LIBCPP_ASSERT_UNCATEGORIZED( - __i != __last, - "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); - } - // It is now known that a guard exists for downward moving __j - do { - _LIBCPP_ASSERT_UNCATEGORIZED( - __j != __first, - "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); - --__j; - } while (!__comp(*__j, *__m)); - if (__i >= __j) - break; - _Ops::iter_swap(__i, __j); - ++__n_swaps; - // It is known that __m != __j - // If __m just moved, follow it - if (__m == __i) - __m = __j; - ++__i; - } + // [__first, __i) == *__first and *__first < [__i, __last) + // The first part is sorted, + if (__nth < __i) { + return; } - // [__first, __i) < *__m and *__m <= [__i, __last) - if (__i != __m && __comp(*__m, *__i)) - { - _Ops::iter_swap(__i, __m); - ++__n_swaps; + // __nth_element the second part + // std::__nth_element<_Compare>(__i, __nth, __last, __comp); + __first = __i; + continue; + } + } + ++__i; + // j points beyond range to be tested, *__lm1 is known to be <= *__m + // if not yet partitioned... + if (__i < __j) { + // known that *(__i - 1) < *__m + while (true) { + // __m still guards upward moving __i + while (__comp(*__i, *__m)) { + ++__i; + _LIBCPP_ASSERT_UNCATEGORIZED( + __i != __last, + "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); } - // [__first, __i) < *__i and *__i <= [__i+1, __last) - if (__nth == __i) + // It is now known that a guard exists for downward moving __j + do { + _LIBCPP_ASSERT_UNCATEGORIZED( + __j != __first, + "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); + --__j; + } while (!__comp(*__j, *__m)); + if (__i >= __j) + break; + _Ops::iter_swap(__i, __j); + ++__n_swaps; + // It is known that __m != __j + // If __m just moved, follow it + if (__m == __i) + __m = __j; + ++__i; + } + } + // [__first, __i) < *__m and *__m <= [__i, __last) + if (__i != __m && __comp(*__m, *__i)) { + _Ops::iter_swap(__i, __m); + ++__n_swaps; + } + // [__first, __i) < *__i and *__i <= [__i+1, __last) + if (__nth == __i) + return; + if (__n_swaps == 0) { + // We were given a perfectly partitioned sequence. Coincidence? + if (__nth < __i) { + // Check for [__first, __i) already sorted + __j = __m = __first; + while (true) { + if (++__j == __i) { + // [__first, __i) sorted return; - if (__n_swaps == 0) - { - // We were given a perfectly partitioned sequence. Coincidence? - if (__nth < __i) - { - // Check for [__first, __i) already sorted - __j = __m = __first; - while (true) { - if (++__j == __i) { - // [__first, __i) sorted - return; - } - if (__comp(*__j, *__m)) { - // not yet sorted, so sort - break; - } - __m = __j; - } - } - else - { - // Check for [__i, __last) already sorted - __j = __m = __i; - while (true) { - if (++__j == __last) { - // [__i, __last) sorted - return; - } - if (__comp(*__j, *__m)) { - // not yet sorted, so sort - break; - } - __m = __j; - } - } - } - // __nth_element on range containing __nth - if (__nth < __i) - { - // std::__nth_element<_Compare>(__first, __nth, __i, __comp); - __last = __i; + } + if (__comp(*__j, *__m)) { + // not yet sorted, so sort + break; + } + __m = __j; } - else - { - // std::__nth_element<_Compare>(__i+1, __nth, __last, __comp); - __first = ++__i; + } else { + // Check for [__i, __last) already sorted + __j = __m = __i; + while (true) { + if (++__j == __last) { + // [__i, __last) sorted + return; + } + if (__comp(*__j, *__m)) { + // not yet sorted, so sort + break; + } + __m = __j; } + } + } + // __nth_element on range containing __nth + if (__nth < __i) { + // std::__nth_element<_Compare>(__first, __nth, __i, __comp); + __last = __i; + } else { + // std::__nth_element<_Compare>(__i+1, __nth, __last, __comp); + __first = ++__i; } + } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void __nth_element_impl(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, - _Compare& __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __nth_element_impl( + _RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare& __comp) { if (__nth == __last) return; @@ -257,15 +240,14 @@ void __nth_element_impl(_RandomAccessIterator __first, _RandomAccessIterator __n } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, - _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) { std::__nth_element_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__nth), std::move(__last), __comp); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last) { std::nth_element(std::move(__first), std::move(__nth), std::move(__last), __less<>()); } diff --git a/libcxx/include/__algorithm/partial_sort.h b/libcxx/include/__algorithm/partial_sort.h index de4ebfcbd8587..27511a124229b 100644 --- a/libcxx/include/__algorithm/partial_sort.h +++ b/libcxx/include/__algorithm/partial_sort.h @@ -29,8 +29,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_RandomAccessIterator __partial_sort_impl( +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator __partial_sort_impl( _RandomAccessIterator __first, _RandomAccessIterator __middle, _Sentinel __last, _Compare&& __comp) { if (__first == __middle) { return _IterOps<_AlgPolicy>::next(__middle, __last); @@ -39,14 +38,12 @@ _RandomAccessIterator __partial_sort_impl( std::__make_heap<_AlgPolicy>(__first, __middle, __comp); typename iterator_traits<_RandomAccessIterator>::difference_type __len = __middle - __first; - _RandomAccessIterator __i = __middle; - for (; __i != __last; ++__i) - { - if (__comp(*__i, *__first)) - { - _IterOps<_AlgPolicy>::iter_swap(__i, __first); - std::__sift_down<_AlgPolicy>(__first, __comp, __len, __first); - } + _RandomAccessIterator __i = __middle; + for (; __i != __last; ++__i) { + if (__comp(*__i, *__first)) { + _IterOps<_AlgPolicy>::iter_swap(__i, __first); + std::__sift_down<_AlgPolicy>(__first, __comp, __len, __first); + } } std::__sort_heap<_AlgPolicy>(std::move(__first), std::move(__middle), __comp); @@ -54,11 +51,10 @@ _RandomAccessIterator __partial_sort_impl( } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_RandomAccessIterator __partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _Sentinel __last, - _Compare& __comp) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator +__partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _Sentinel __last, _Compare& __comp) { if (__first == __middle) - return _IterOps<_AlgPolicy>::next(__middle, __last); + return _IterOps<_AlgPolicy>::next(__middle, __last); std::__debug_randomize_range<_AlgPolicy>(__first, __last); @@ -71,11 +67,8 @@ _RandomAccessIterator __partial_sort(_RandomAccessIterator __first, _RandomAcces } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last, - _Compare __comp) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void partial_sort( + _RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp) { static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible."); static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable."); @@ -83,11 +76,9 @@ partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _Ran } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) -{ - std::partial_sort(__first, __middle, __last, __less<>()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) { + std::partial_sort(__first, __middle, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/partial_sort_copy.h b/libcxx/include/__algorithm/partial_sort_copy.h index 2c1e786d030eb..e7d8df4de89f9 100644 --- a/libcxx/include/__algorithm/partial_sort_copy.h +++ b/libcxx/include/__algorithm/partial_sort_copy.h @@ -30,56 +30,70 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator, _RandomAccessIterator> -__partial_sort_copy(_InputIterator __first, _Sentinel1 __last, - _RandomAccessIterator __result_first, _Sentinel2 __result_last, - _Compare&& __comp, _Proj1&& __proj1, _Proj2&& __proj2) -{ - _RandomAccessIterator __r = __result_first; - auto&& __projected_comp = std::__make_projected(__comp, __proj2); +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator, _RandomAccessIterator> __partial_sort_copy( + _InputIterator __first, + _Sentinel1 __last, + _RandomAccessIterator __result_first, + _Sentinel2 __result_last, + _Compare&& __comp, + _Proj1&& __proj1, + _Proj2&& __proj2) { + _RandomAccessIterator __r = __result_first; + auto&& __projected_comp = std::__make_projected(__comp, __proj2); - if (__r != __result_last) - { - for (; __first != __last && __r != __result_last; ++__first, (void) ++__r) - *__r = *__first; - std::__make_heap<_AlgPolicy>(__result_first, __r, __projected_comp); - typename iterator_traits<_RandomAccessIterator>::difference_type __len = __r - __result_first; - for (; __first != __last; ++__first) - if (std::__invoke(__comp, std::__invoke(__proj1, *__first), std::__invoke(__proj2, *__result_first))) { - *__result_first = *__first; - std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, __result_first); - } - std::__sort_heap<_AlgPolicy>(__result_first, __r, __projected_comp); - } + if (__r != __result_last) { + for (; __first != __last && __r != __result_last; ++__first, (void)++__r) + *__r = *__first; + std::__make_heap<_AlgPolicy>(__result_first, __r, __projected_comp); + typename iterator_traits<_RandomAccessIterator>::difference_type __len = __r - __result_first; + for (; __first != __last; ++__first) + if (std::__invoke(__comp, std::__invoke(__proj1, *__first), std::__invoke(__proj2, *__result_first))) { + *__result_first = *__first; + std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, __result_first); + } + std::__sort_heap<_AlgPolicy>(__result_first, __r, __projected_comp); + } - return pair<_InputIterator, _RandomAccessIterator>( - _IterOps<_AlgPolicy>::next(std::move(__first), std::move(__last)), std::move(__r)); + return pair<_InputIterator, _RandomAccessIterator>( + _IterOps<_AlgPolicy>::next(std::move(__first), std::move(__last)), std::move(__r)); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_RandomAccessIterator -partial_sort_copy(_InputIterator __first, _InputIterator __last, - _RandomAccessIterator __result_first, _RandomAccessIterator __result_last, _Compare __comp) -{ - static_assert(__is_callable<_Compare, decltype(*__first), decltype(*__result_first)>::value, - "Comparator has to be callable"); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator partial_sort_copy( + _InputIterator __first, + _InputIterator __last, + _RandomAccessIterator __result_first, + _RandomAccessIterator __result_last, + _Compare __comp) { + static_assert( + __is_callable<_Compare, decltype(*__first), decltype(*__result_first)>::value, "Comparator has to be callable"); - auto __result = std::__partial_sort_copy<_ClassicAlgPolicy>(__first, __last, __result_first, __result_last, - static_cast<__comp_ref_type<_Compare> >(__comp), __identity(), __identity()); + auto __result = std::__partial_sort_copy<_ClassicAlgPolicy>( + __first, + __last, + __result_first, + __result_last, + static_cast<__comp_ref_type<_Compare> >(__comp), + __identity(), + __identity()); return __result.second; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_RandomAccessIterator -partial_sort_copy(_InputIterator __first, _InputIterator __last, - _RandomAccessIterator __result_first, _RandomAccessIterator __result_last) -{ - return std::partial_sort_copy(__first, __last, __result_first, __result_last, __less<>()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator partial_sort_copy( + _InputIterator __first, + _InputIterator __last, + _RandomAccessIterator __result_first, + _RandomAccessIterator __result_last) { + return std::partial_sort_copy(__first, __last, __result_first, __result_last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/partition.h b/libcxx/include/__algorithm/partition.h index a6aaf634aaa5b..e2ceb07bf1958 100644 --- a/libcxx/include/__algorithm/partition.h +++ b/libcxx/include/__algorithm/partition.h @@ -23,70 +23,58 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator> -__partition_impl(_ForwardIterator __first, _Sentinel __last, _Predicate __pred, forward_iterator_tag) -{ - while (true) - { - if (__first == __last) - return std::make_pair(std::move(__first), std::move(__first)); - if (!__pred(*__first)) - break; - ++__first; - } +__partition_impl(_ForwardIterator __first, _Sentinel __last, _Predicate __pred, forward_iterator_tag) { + while (true) { + if (__first == __last) + return std::make_pair(std::move(__first), std::move(__first)); + if (!__pred(*__first)) + break; + ++__first; + } - _ForwardIterator __p = __first; - while (++__p != __last) - { - if (__pred(*__p)) - { - _IterOps<_AlgPolicy>::iter_swap(__first, __p); - ++__first; - } + _ForwardIterator __p = __first; + while (++__p != __last) { + if (__pred(*__p)) { + _IterOps<_AlgPolicy>::iter_swap(__first, __p); + ++__first; } - return std::make_pair(std::move(__first), std::move(__p)); + } + return std::make_pair(std::move(__first), std::move(__p)); } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, _BidirectionalIterator> -__partition_impl(_BidirectionalIterator __first, _Sentinel __sentinel, _Predicate __pred, - bidirectional_iterator_tag) -{ - _BidirectionalIterator __original_last = _IterOps<_AlgPolicy>::next(__first, __sentinel); - _BidirectionalIterator __last = __original_last; +__partition_impl(_BidirectionalIterator __first, _Sentinel __sentinel, _Predicate __pred, bidirectional_iterator_tag) { + _BidirectionalIterator __original_last = _IterOps<_AlgPolicy>::next(__first, __sentinel); + _BidirectionalIterator __last = __original_last; - while (true) - { - while (true) - { - if (__first == __last) - return std::make_pair(std::move(__first), std::move(__original_last)); - if (!__pred(*__first)) - break; - ++__first; - } - do - { - if (__first == --__last) - return std::make_pair(std::move(__first), std::move(__original_last)); - } while (!__pred(*__last)); - _IterOps<_AlgPolicy>::iter_swap(__first, __last); - ++__first; + while (true) { + while (true) { + if (__first == __last) + return std::make_pair(std::move(__first), std::move(__original_last)); + if (!__pred(*__first)) + break; + ++__first; } + do { + if (__first == --__last) + return std::make_pair(std::move(__first), std::move(__original_last)); + } while (!__pred(*__last)); + _IterOps<_AlgPolicy>::iter_swap(__first, __last); + ++__first; + } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -pair<_ForwardIterator, _ForwardIterator> __partition( - _ForwardIterator __first, _Sentinel __last, _Predicate&& __pred, _IterCategory __iter_category) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator, _ForwardIterator> +__partition(_ForwardIterator __first, _Sentinel __last, _Predicate&& __pred, _IterCategory __iter_category) { return std::__partition_impl<__remove_cvref_t<_Predicate>&, _AlgPolicy>( std::move(__first), std::move(__last), __pred, __iter_category); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator -partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator +partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { using _IterCategory = typename iterator_traits<_ForwardIterator>::iterator_category; auto __result = std::__partition<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __pred, _IterCategory()); return __result.first; diff --git a/libcxx/include/__algorithm/partition_copy.h b/libcxx/include/__algorithm/partition_copy.h index ff8826a937125..147b45c7882a5 100644 --- a/libcxx/include/__algorithm/partition_copy.h +++ b/libcxx/include/__algorithm/partition_copy.h @@ -19,27 +19,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_OutputIterator1, _OutputIterator2> -partition_copy(_InputIterator __first, _InputIterator __last, - _OutputIterator1 __out_true, _OutputIterator2 __out_false, - _Predicate __pred) -{ - for (; __first != __last; ++__first) - { - if (__pred(*__first)) - { - *__out_true = *__first; - ++__out_true; - } - else - { - *__out_false = *__first; - ++__out_false; - } +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_OutputIterator1, _OutputIterator2> partition_copy( + _InputIterator __first, + _InputIterator __last, + _OutputIterator1 __out_true, + _OutputIterator2 __out_false, + _Predicate __pred) { + for (; __first != __last; ++__first) { + if (__pred(*__first)) { + *__out_true = *__first; + ++__out_true; + } else { + *__out_false = *__first; + ++__out_false; } - return pair<_OutputIterator1, _OutputIterator2>(__out_true, __out_false); + } + return pair<_OutputIterator1, _OutputIterator2>(__out_true, __out_false); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/partition_point.h b/libcxx/include/__algorithm/partition_point.h index a61ed5ec5b41b..504dbf1d1a055 100644 --- a/libcxx/include/__algorithm/partition_point.h +++ b/libcxx/include/__algorithm/partition_point.h @@ -21,26 +21,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template +template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -partition_point(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) -{ - typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type; - difference_type __len = std::distance(__first, __last); - while (__len != 0) - { - difference_type __l2 = std::__half_positive(__len); - _ForwardIterator __m = __first; - std::advance(__m, __l2); - if (__pred(*__m)) - { - __first = ++__m; - __len -= __l2 + 1; - } - else - __len = __l2; - } - return __first; +partition_point(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type; + difference_type __len = std::distance(__first, __last); + while (__len != 0) { + difference_type __l2 = std::__half_positive(__len); + _ForwardIterator __m = __first; + std::advance(__m, __l2); + if (__pred(*__m)) { + __first = ++__m; + __len -= __l2 + 1; + } else + __len = __l2; + } + return __first; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/pop_heap.h b/libcxx/include/__algorithm/pop_heap.h index 033af6f2f80fd..a93a9875f7058 100644 --- a/libcxx/include/__algorithm/pop_heap.h +++ b/libcxx/include/__algorithm/pop_heap.h @@ -31,16 +31,18 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__pop_heap(_RandomAccessIterator __first, + _RandomAccessIterator __last, + _Compare& __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len) { _LIBCPP_ASSERT_UNCATEGORIZED(__len > 0, "The heap given to pop_heap must be non-empty"); __comp_ref_type<_Compare> __comp_ref = __comp; using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; if (__len > 1) { - value_type __top = _IterOps<_AlgPolicy>::__iter_move(__first); // create a hole at __first + value_type __top = _IterOps<_AlgPolicy>::__iter_move(__first); // create a hole at __first _RandomAccessIterator __hole = std::__floyd_sift_down<_AlgPolicy>(__first, __comp_ref, __len); --__last; @@ -56,8 +58,8 @@ void __pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Co } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible."); static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable."); @@ -66,8 +68,8 @@ void pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::pop_heap(std::move(__first), std::move(__last), __less<>()); } diff --git a/libcxx/include/__algorithm/prev_permutation.h b/libcxx/include/__algorithm/prev_permutation.h index 4a6bf4a664399..3e4bbb3fbb167 100644 --- a/libcxx/include/__algorithm/prev_permutation.h +++ b/libcxx/include/__algorithm/prev_permutation.h @@ -25,52 +25,44 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -pair<_BidirectionalIterator, bool> -__prev_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) -{ - using _Result = pair<_BidirectionalIterator, bool>; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_BidirectionalIterator, bool> +__prev_permutation(_BidirectionalIterator __first, _Sentinel __last, _Compare&& __comp) { + using _Result = pair<_BidirectionalIterator, bool>; - _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); - _BidirectionalIterator __i = __last_iter; - if (__first == __last || __first == --__i) - return _Result(std::move(__last_iter), false); + _BidirectionalIterator __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); + _BidirectionalIterator __i = __last_iter; + if (__first == __last || __first == --__i) + return _Result(std::move(__last_iter), false); - while (true) - { - _BidirectionalIterator __ip1 = __i; - if (__comp(*__ip1, *--__i)) - { - _BidirectionalIterator __j = __last_iter; - while (!__comp(*--__j, *__i)) - ; - _IterOps<_AlgPolicy>::iter_swap(__i, __j); - std::__reverse<_AlgPolicy>(__ip1, __last_iter); - return _Result(std::move(__last_iter), true); - } - if (__i == __first) - { - std::__reverse<_AlgPolicy>(__first, __last_iter); - return _Result(std::move(__last_iter), false); - } + while (true) { + _BidirectionalIterator __ip1 = __i; + if (__comp(*__ip1, *--__i)) { + _BidirectionalIterator __j = __last_iter; + while (!__comp(*--__j, *__i)) + ; + _IterOps<_AlgPolicy>::iter_swap(__i, __j); + std::__reverse<_AlgPolicy>(__ip1, __last_iter); + return _Result(std::move(__last_iter), true); } + if (__i == __first) { + std::__reverse<_AlgPolicy>(__first, __last_iter); + return _Result(std::move(__last_iter), false); + } + } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { return std::__prev_permutation<_ClassicAlgPolicy>( - std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp)).second; + std::move(__first), std::move(__last), static_cast<__comp_ref_type<_Compare> >(__comp)) + .second; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -bool -prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) -{ - return std::prev_permutation(__first, __last, __less<>()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool +prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last) { + return std::prev_permutation(__first, __last, __less<>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/push_heap.h b/libcxx/include/__algorithm/push_heap.h index 82b571e44bd4d..7d8720e3a93d4 100644 --- a/libcxx/include/__algorithm/push_heap.h +++ b/libcxx/include/__algorithm/push_heap.h @@ -28,20 +28,22 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __sift_up(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__sift_up(_RandomAccessIterator __first, + _RandomAccessIterator __last, + _Compare&& __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len) { using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; if (__len > 1) { - __len = (__len - 2) / 2; + __len = (__len - 2) / 2; _RandomAccessIterator __ptr = __first + __len; if (__comp(*__ptr, *--__last)) { value_type __t(_IterOps<_AlgPolicy>::__iter_move(__last)); do { *__last = _IterOps<_AlgPolicy>::__iter_move(__ptr); - __last = __ptr; + __last = __ptr; if (__len == 0) break; __len = (__len - 1) / 2; @@ -54,15 +56,15 @@ void __sift_up(_RandomAccessIterator __first, _RandomAccessIterator __last, _Com } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) { typename iterator_traits<_RandomAccessIterator>::difference_type __len = __last - __first; std::__sift_up<_AlgPolicy, __comp_ref_type<_Compare> >(std::move(__first), std::move(__last), __comp, __len); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible."); static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable."); @@ -70,8 +72,8 @@ void push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Com } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::push_heap(std::move(__first), std::move(__last), __less<>()); } diff --git a/libcxx/include/__algorithm/remove.h b/libcxx/include/__algorithm/remove.h index eff22757186e3..2b9d4ff26ed2a 100644 --- a/libcxx/include/__algorithm/remove.h +++ b/libcxx/include/__algorithm/remove.h @@ -22,22 +22,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -remove(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) -{ - __first = std::find(__first, __last, __value); - if (__first != __last) - { - _ForwardIterator __i = __first; - while (++__i != __last) - { - if (!(*__i == __value)) - { - *__first = std::move(*__i); - ++__first; - } - } +remove(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + __first = std::find(__first, __last, __value); + if (__first != __last) { + _ForwardIterator __i = __first; + while (++__i != __last) { + if (!(*__i == __value)) { + *__first = std::move(*__i); + ++__first; + } } - return __first; + } + return __first; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/remove_copy.h b/libcxx/include/__algorithm/remove_copy.h index 060833813d211..7be4c166ce3d7 100644 --- a/libcxx/include/__algorithm/remove_copy.h +++ b/libcxx/include/__algorithm/remove_copy.h @@ -18,19 +18,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -remove_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, const _Tp& __value) -{ - for (; __first != __last; ++__first) - { - if (!(*__first == __value)) - { - *__result = *__first; - ++__result; - } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +remove_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, const _Tp& __value) { + for (; __first != __last; ++__first) { + if (!(*__first == __value)) { + *__result = *__first; + ++__result; } - return __result; + } + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/remove_copy_if.h b/libcxx/include/__algorithm/remove_copy_if.h index 9858d43c043f5..dcafed169157d 100644 --- a/libcxx/include/__algorithm/remove_copy_if.h +++ b/libcxx/include/__algorithm/remove_copy_if.h @@ -18,19 +18,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -remove_copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Predicate __pred) -{ - for (; __first != __last; ++__first) - { - if (!__pred(*__first)) - { - *__result = *__first; - ++__result; - } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +remove_copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Predicate __pred) { + for (; __first != __last; ++__first) { + if (!__pred(*__first)) { + *__result = *__first; + ++__result; } - return __result; + } + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/remove_if.h b/libcxx/include/__algorithm/remove_if.h index 8197e4fc2f8ef..6eceddce8d56b 100644 --- a/libcxx/include/__algorithm/remove_if.h +++ b/libcxx/include/__algorithm/remove_if.h @@ -21,22 +21,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -remove_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) -{ - __first = std::find_if<_ForwardIterator, _Predicate&>(__first, __last, __pred); - if (__first != __last) - { - _ForwardIterator __i = __first; - while (++__i != __last) - { - if (!__pred(*__i)) - { - *__first = std::move(*__i); - ++__first; - } - } +remove_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + __first = std::find_if<_ForwardIterator, _Predicate&>(__first, __last, __pred); + if (__first != __last) { + _ForwardIterator __i = __first; + while (++__i != __last) { + if (!__pred(*__i)) { + *__first = std::move(*__i); + ++__first; + } } - return __first; + } + return __first; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/replace.h b/libcxx/include/__algorithm/replace.h index 09b1dac79b708..8057c78686e11 100644 --- a/libcxx/include/__algorithm/replace.h +++ b/libcxx/include/__algorithm/replace.h @@ -18,13 +18,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -replace(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value, const _Tp& __new_value) -{ - for (; __first != __last; ++__first) - if (*__first == __old_value) - *__first = __new_value; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +replace(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value, const _Tp& __new_value) { + for (; __first != __last; ++__first) + if (*__first == __old_value) + *__first = __new_value; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/replace_copy.h b/libcxx/include/__algorithm/replace_copy.h index 107e92968944f..9a2258d9f58ed 100644 --- a/libcxx/include/__algorithm/replace_copy.h +++ b/libcxx/include/__algorithm/replace_copy.h @@ -18,17 +18,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -replace_copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, - const _Tp& __old_value, const _Tp& __new_value) -{ - for (; __first != __last; ++__first, (void) ++__result) - if (*__first == __old_value) - *__result = __new_value; - else - *__result = *__first; - return __result; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator replace_copy( + _InputIterator __first, + _InputIterator __last, + _OutputIterator __result, + const _Tp& __old_value, + const _Tp& __new_value) { + for (; __first != __last; ++__first, (void)++__result) + if (*__first == __old_value) + *__result = __new_value; + else + *__result = *__first; + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/replace_copy_if.h b/libcxx/include/__algorithm/replace_copy_if.h index a77b1caa805a2..c2ed30f08d598 100644 --- a/libcxx/include/__algorithm/replace_copy_if.h +++ b/libcxx/include/__algorithm/replace_copy_if.h @@ -18,17 +18,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -replace_copy_if(_InputIterator __first, _InputIterator __last, _OutputIterator __result, - _Predicate __pred, const _Tp& __new_value) -{ - for (; __first != __last; ++__first, (void) ++__result) - if (__pred(*__first)) - *__result = __new_value; - else - *__result = *__first; - return __result; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator replace_copy_if( + _InputIterator __first, + _InputIterator __last, + _OutputIterator __result, + _Predicate __pred, + const _Tp& __new_value) { + for (; __first != __last; ++__first, (void)++__result) + if (__pred(*__first)) + *__result = __new_value; + else + *__result = *__first; + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/replace_if.h b/libcxx/include/__algorithm/replace_if.h index 05269065f025a..78487e3deed70 100644 --- a/libcxx/include/__algorithm/replace_if.h +++ b/libcxx/include/__algorithm/replace_if.h @@ -18,13 +18,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -replace_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, const _Tp& __new_value) -{ - for (; __first != __last; ++__first) - if (__pred(*__first)) - *__first = __new_value; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +replace_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, const _Tp& __new_value) { + for (; __first != __last; ++__first) + if (__pred(*__first)) + *__first = __new_value; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/reverse.h b/libcxx/include/__algorithm/reverse.h index 86634520aa261..6bd0aa3932806 100644 --- a/libcxx/include/__algorithm/reverse.h +++ b/libcxx/include/__algorithm/reverse.h @@ -22,41 +22,33 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -__reverse_impl(_BidirectionalIterator __first, _BidirectionalIterator __last, bidirectional_iterator_tag) -{ - while (__first != __last) - { - if (__first == --__last) - break; - _IterOps<_AlgPolicy>::iter_swap(__first, __last); - ++__first; - } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +__reverse_impl(_BidirectionalIterator __first, _BidirectionalIterator __last, bidirectional_iterator_tag) { + while (__first != __last) { + if (__first == --__last) + break; + _IterOps<_AlgPolicy>::iter_swap(__first, __last); + ++__first; + } } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -__reverse_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, random_access_iterator_tag) -{ - if (__first != __last) - for (; __first < --__last; ++__first) - _IterOps<_AlgPolicy>::iter_swap(__first, __last); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +__reverse_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, random_access_iterator_tag) { + if (__first != __last) + for (; __first < --__last; ++__first) + _IterOps<_AlgPolicy>::iter_swap(__first, __last); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void __reverse(_BidirectionalIterator __first, _Sentinel __last) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __reverse(_BidirectionalIterator __first, _Sentinel __last) { using _IterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_BidirectionalIterator>; std::__reverse_impl<_AlgPolicy>(std::move(__first), std::move(__last), _IterCategory()); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void -reverse(_BidirectionalIterator __first, _BidirectionalIterator __last) -{ +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +reverse(_BidirectionalIterator __first, _BidirectionalIterator __last) { std::__reverse<_ClassicAlgPolicy>(std::move(__first), std::move(__last)); } diff --git a/libcxx/include/__algorithm/reverse_copy.h b/libcxx/include/__algorithm/reverse_copy.h index 7672fc86686be..0fcecc3923268 100644 --- a/libcxx/include/__algorithm/reverse_copy.h +++ b/libcxx/include/__algorithm/reverse_copy.h @@ -18,13 +18,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result) -{ - for (; __first != __last; ++__result) - *__result = *--__last; - return __result; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result) { + for (; __first != __last; ++__result) + *__result = *--__last; + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/rotate.h b/libcxx/include/__algorithm/rotate.h index f9236fcaafb2a..d8162b1a94b27 100644 --- a/libcxx/include/__algorithm/rotate.h +++ b/libcxx/include/__algorithm/rotate.h @@ -27,193 +27,165 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -__rotate_left(_ForwardIterator __first, _ForwardIterator __last) -{ - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; - using _Ops = _IterOps<_AlgPolicy>; - - value_type __tmp = _Ops::__iter_move(__first); - _ForwardIterator __lm1 = std::__move<_AlgPolicy>( - _Ops::next(__first), __last, __first).second; - *__lm1 = std::move(__tmp); - return __lm1; +__rotate_left(_ForwardIterator __first, _ForwardIterator __last) { + typedef typename iterator_traits<_ForwardIterator>::value_type value_type; + using _Ops = _IterOps<_AlgPolicy>; + + value_type __tmp = _Ops::__iter_move(__first); + _ForwardIterator __lm1 = std::__move<_AlgPolicy>(_Ops::next(__first), __last, __first).second; + *__lm1 = std::move(__tmp); + return __lm1; } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _BidirectionalIterator -__rotate_right(_BidirectionalIterator __first, _BidirectionalIterator __last) -{ - typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - using _Ops = _IterOps<_AlgPolicy>; - - _BidirectionalIterator __lm1 = _Ops::prev(__last); - value_type __tmp = _Ops::__iter_move(__lm1); - _BidirectionalIterator __fp1 = std::__move_backward<_AlgPolicy>(__first, __lm1, std::move(__last)).second; - *__first = std::move(__tmp); - return __fp1; +__rotate_right(_BidirectionalIterator __first, _BidirectionalIterator __last) { + typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; + using _Ops = _IterOps<_AlgPolicy>; + + _BidirectionalIterator __lm1 = _Ops::prev(__last); + value_type __tmp = _Ops::__iter_move(__lm1); + _BidirectionalIterator __fp1 = std::__move_backward<_AlgPolicy>(__first, __lm1, std::move(__last)).second; + *__first = std::move(__tmp); + return __fp1; } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _ForwardIterator -__rotate_forward(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) -{ - _ForwardIterator __i = __middle; - while (true) - { - _IterOps<_AlgPolicy>::iter_swap(__first, __i); - ++__first; - if (++__i == __last) - break; +__rotate_forward(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) { + _ForwardIterator __i = __middle; + while (true) { + _IterOps<_AlgPolicy>::iter_swap(__first, __i); + ++__first; + if (++__i == __last) + break; + if (__first == __middle) + __middle = __i; + } + _ForwardIterator __r = __first; + if (__first != __middle) { + __i = __middle; + while (true) { + _IterOps<_AlgPolicy>::iter_swap(__first, __i); + ++__first; + if (++__i == __last) { if (__first == __middle) - __middle = __i; - } - _ForwardIterator __r = __first; - if (__first != __middle) - { + break; __i = __middle; - while (true) - { - _IterOps<_AlgPolicy>::iter_swap(__first, __i); - ++__first; - if (++__i == __last) - { - if (__first == __middle) - break; - __i = __middle; - } - else if (__first == __middle) - __middle = __i; - } + } else if (__first == __middle) + __middle = __i; } - return __r; + } + return __r; } -template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX17 _Integral -__algo_gcd(_Integral __x, _Integral __y) -{ - do - { - _Integral __t = __x % __y; - __x = __y; - __y = __t; - } while (__y); - return __x; +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Integral __algo_gcd(_Integral __x, _Integral __y) { + do { + _Integral __t = __x % __y; + __x = __y; + __y = __t; + } while (__y); + return __x; } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _RandomAccessIterator -__rotate_gcd(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) -{ - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - using _Ops = _IterOps<_AlgPolicy>; - - const difference_type __m1 = __middle - __first; - const difference_type __m2 = _Ops::distance(__middle, __last); - if (__m1 == __m2) - { - std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle, __last); - return __middle; - } - const difference_type __g = std::__algo_gcd(__m1, __m2); - for (_RandomAccessIterator __p = __first + __g; __p != __first;) - { - value_type __t(_Ops::__iter_move(--__p)); - _RandomAccessIterator __p1 = __p; - _RandomAccessIterator __p2 = __p1 + __m1; - do - { - *__p1 = _Ops::__iter_move(__p2); - __p1 = __p2; - const difference_type __d = _Ops::distance(__p2, __last); - if (__m1 < __d) - __p2 += __m1; - else - __p2 = __first + (__m1 - __d); - } while (__p2 != __p); - *__p1 = std::move(__t); - } - return __first + __m2; +__rotate_gcd(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) { + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + using _Ops = _IterOps<_AlgPolicy>; + + const difference_type __m1 = __middle - __first; + const difference_type __m2 = _Ops::distance(__middle, __last); + if (__m1 == __m2) { + std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle, __last); + return __middle; + } + const difference_type __g = std::__algo_gcd(__m1, __m2); + for (_RandomAccessIterator __p = __first + __g; __p != __first;) { + value_type __t(_Ops::__iter_move(--__p)); + _RandomAccessIterator __p1 = __p; + _RandomAccessIterator __p2 = __p1 + __m1; + do { + *__p1 = _Ops::__iter_move(__p2); + __p1 = __p2; + const difference_type __d = _Ops::distance(__p2, __last); + if (__m1 < __d) + __p2 += __m1; + else + __p2 = __first + (__m1 - __d); + } while (__p2 != __p); + *__p1 = std::move(__t); + } + return __first + __m2; } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator -__rotate_impl(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, - std::forward_iterator_tag) -{ - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; - if (is_trivially_move_assignable::value) - { - if (_IterOps<_AlgPolicy>::next(__first) == __middle) - return std::__rotate_left<_AlgPolicy>(__first, __last); - } - return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator +__rotate_impl(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, std::forward_iterator_tag) { + typedef typename iterator_traits<_ForwardIterator>::value_type value_type; + if (is_trivially_move_assignable::value) { + if (_IterOps<_AlgPolicy>::next(__first) == __middle) + return std::__rotate_left<_AlgPolicy>(__first, __last); + } + return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 _BidirectionalIterator -__rotate_impl(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last, - bidirectional_iterator_tag) -{ - typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - if (is_trivially_move_assignable::value) - { - if (_IterOps<_AlgPolicy>::next(__first) == __middle) - return std::__rotate_left<_AlgPolicy>(__first, __last); - if (_IterOps<_AlgPolicy>::next(__middle) == __last) - return std::__rotate_right<_AlgPolicy>(__first, __last); - } - return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _BidirectionalIterator __rotate_impl( + _BidirectionalIterator __first, + _BidirectionalIterator __middle, + _BidirectionalIterator __last, + bidirectional_iterator_tag) { + typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; + if (is_trivially_move_assignable::value) { + if (_IterOps<_AlgPolicy>::next(__first) == __middle) + return std::__rotate_left<_AlgPolicy>(__first, __last); + if (_IterOps<_AlgPolicy>::next(__middle) == __last) + return std::__rotate_right<_AlgPolicy>(__first, __last); + } + return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator -__rotate_impl(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last, - random_access_iterator_tag) -{ - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - if (is_trivially_move_assignable::value) - { - if (_IterOps<_AlgPolicy>::next(__first) == __middle) - return std::__rotate_left<_AlgPolicy>(__first, __last); - if (_IterOps<_AlgPolicy>::next(__middle) == __last) - return std::__rotate_right<_AlgPolicy>(__first, __last); - return std::__rotate_gcd<_AlgPolicy>(__first, __middle, __last); - } - return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __rotate_impl( + _RandomAccessIterator __first, + _RandomAccessIterator __middle, + _RandomAccessIterator __last, + random_access_iterator_tag) { + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + if (is_trivially_move_assignable::value) { + if (_IterOps<_AlgPolicy>::next(__first) == __middle) + return std::__rotate_left<_AlgPolicy>(__first, __last); + if (_IterOps<_AlgPolicy>::next(__middle) == __last) + return std::__rotate_right<_AlgPolicy>(__first, __last); + return std::__rotate_gcd<_AlgPolicy>(__first, __middle, __last); + } + return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iterator, _Iterator> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iterator, _Iterator> __rotate(_Iterator __first, _Iterator __middle, _Sentinel __last) { - using _Ret = pair<_Iterator, _Iterator>; + using _Ret = pair<_Iterator, _Iterator>; _Iterator __last_iter = _IterOps<_AlgPolicy>::next(__middle, __last); if (__first == __middle) - return _Ret(__last_iter, __last_iter); + return _Ret(__last_iter, __last_iter); if (__middle == __last) - return _Ret(std::move(__first), std::move(__last_iter)); + return _Ret(std::move(__first), std::move(__last_iter)); using _IterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_Iterator>; - auto __result = std::__rotate_impl<_AlgPolicy>( - std::move(__first), std::move(__middle), __last_iter, _IterCategory()); + auto __result = std::__rotate_impl<_AlgPolicy>(std::move(__first), std::move(__middle), __last_iter, _IterCategory()); return _Ret(std::move(__result), std::move(__last_iter)); } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator -rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) -{ - return std::__rotate<_ClassicAlgPolicy>( - std::move(__first), std::move(__middle), std::move(__last)).first; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator +rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) { + return std::__rotate<_ClassicAlgPolicy>(std::move(__first), std::move(__middle), std::move(__last)).first; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/rotate_copy.h b/libcxx/include/__algorithm/rotate_copy.h index 6e886cb73cfc5..cddcadd237d90 100644 --- a/libcxx/include/__algorithm/rotate_copy.h +++ b/libcxx/include/__algorithm/rotate_copy.h @@ -19,11 +19,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _OutputIterator __result) -{ - return std::copy(__first, __middle, std::copy(__middle, __last, __result)); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _OutputIterator __result) { + return std::copy(__first, __middle, std::copy(__middle, __last, __result)); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/sample.h b/libcxx/include/__algorithm/sample.h index 100b0a466863d..cc29dd686f6be 100644 --- a/libcxx/include/__algorithm/sample.h +++ b/libcxx/include/__algorithm/sample.h @@ -29,20 +29,23 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI -_SampleIterator __sample(_PopulationIterator __first, - _PopulationSentinel __last, _SampleIterator __output_iter, - _Distance __n, - _UniformRandomNumberGenerator& __g, - input_iterator_tag) { - +_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample( + _PopulationIterator __first, + _PopulationSentinel __last, + _SampleIterator __output_iter, + _Distance __n, + _UniformRandomNumberGenerator& __g, + input_iterator_tag) { _Distance __k = 0; - for (; __first != __last && __k < __n; ++__first, (void) ++__k) + for (; __first != __last && __k < __n; ++__first, (void)++__k) __output_iter[__k] = *__first; _Distance __sz = __k; - for (; __first != __last; ++__first, (void) ++__k) { + for (; __first != __last; ++__first, (void)++__k) { _Distance __r = uniform_int_distribution<_Distance>(0, __k)(__g); if (__r < __sz) __output_iter[__r] = *__first; @@ -51,14 +54,18 @@ _SampleIterator __sample(_PopulationIterator __first, } template -_LIBCPP_HIDE_FROM_ABI -_SampleIterator __sample(_PopulationIterator __first, - _PopulationSentinel __last, _SampleIterator __output_iter, - _Distance __n, - _UniformRandomNumberGenerator& __g, - forward_iterator_tag) { +_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample( + _PopulationIterator __first, + _PopulationSentinel __last, + _SampleIterator __output_iter, + _Distance __n, + _UniformRandomNumberGenerator& __g, + forward_iterator_tag) { _Distance __unsampled_sz = _IterOps<_AlgPolicy>::distance(__first, __last); for (__n = std::min(__n, __unsampled_sz); __n != 0; ++__first) { _Distance __r = uniform_int_distribution<_Distance>(0, --__unsampled_sz)(__g); @@ -71,36 +78,40 @@ _SampleIterator __sample(_PopulationIterator __first, } template -_LIBCPP_HIDE_FROM_ABI -_SampleIterator __sample(_PopulationIterator __first, - _PopulationSentinel __last, _SampleIterator __output_iter, - _Distance __n, _UniformRandomNumberGenerator& __g) { +_LIBCPP_HIDE_FROM_ABI _SampleIterator __sample( + _PopulationIterator __first, + _PopulationSentinel __last, + _SampleIterator __output_iter, + _Distance __n, + _UniformRandomNumberGenerator& __g) { _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0, "N must be a positive number."); using _PopIterCategory = typename _IterOps<_AlgPolicy>::template __iterator_category<_PopulationIterator>; - using _Difference = typename _IterOps<_AlgPolicy>::template __difference_type<_PopulationIterator>; - using _CommonType = typename common_type<_Distance, _Difference>::type; + using _Difference = typename _IterOps<_AlgPolicy>::template __difference_type<_PopulationIterator>; + using _CommonType = typename common_type<_Distance, _Difference>::type; return std::__sample<_AlgPolicy>( - std::move(__first), std::move(__last), std::move(__output_iter), _CommonType(__n), - __g, _PopIterCategory()); + std::move(__first), std::move(__last), std::move(__output_iter), _CommonType(__n), __g, _PopIterCategory()); } #if _LIBCPP_STD_VER >= 17 -template -inline _LIBCPP_HIDE_FROM_ABI -_SampleIterator sample(_PopulationIterator __first, - _PopulationIterator __last, _SampleIterator __output_iter, - _Distance __n, _UniformRandomNumberGenerator&& __g) { +template +inline _LIBCPP_HIDE_FROM_ABI _SampleIterator +sample(_PopulationIterator __first, + _PopulationIterator __last, + _SampleIterator __output_iter, + _Distance __n, + _UniformRandomNumberGenerator&& __g) { static_assert(__has_forward_iterator_category<_PopulationIterator>::value || - __has_random_access_iterator_category<_SampleIterator>::value, + __has_random_access_iterator_category<_SampleIterator>::value, "SampleIterator must meet the requirements of RandomAccessIterator"); - return std::__sample<_ClassicAlgPolicy>( - std::move(__first), std::move(__last), std::move(__output_iter), __n, __g); + return std::__sample<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__output_iter), __n, __g); } #endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/include/__algorithm/search.h b/libcxx/include/__algorithm/search.h index 544e127f7d4c1..75f936d0f217e 100644 --- a/libcxx/include/__algorithm/search.h +++ b/libcxx/include/__algorithm/search.h @@ -29,17 +29,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter1, _Iter1> __search_forward_impl(_Iter1 __first1, _Sent1 __last1, - _Iter2 __first2, _Sent2 __last2, - _Pred& __pred, - _Proj1& __proj1, - _Proj2& __proj2) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_forward_impl( + _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { if (__first2 == __last2) return std::make_pair(__first1, __first1); // Everything matches an empty sequence while (true) { @@ -64,8 +62,7 @@ pair<_Iter1, _Iter1> __search_forward_impl(_Iter1 __first1, _Sent1 __last1, } // if there is a mismatch, restart with a new __first1 - if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) - { + if (!std::__invoke(__pred, std::__invoke(__proj1, *__m1), std::__invoke(__proj2, *__m2))) { ++__first1; break; } // else there is a match, check next elements @@ -74,21 +71,25 @@ pair<_Iter1, _Iter1> __search_forward_impl(_Iter1 __first1, _Sent1 __last1, } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter1, _Iter1> __search_random_access_impl(_Iter1 __first1, _Sent1 __last1, - _Iter2 __first2, _Sent2 __last2, - _Pred& __pred, - _Proj1& __proj1, - _Proj2& __proj2, - _DiffT1 __size1, - _DiffT2 __size2) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_random_access_impl( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred& __pred, + _Proj1& __proj1, + _Proj2& __proj2, + _DiffT1 __size1, + _DiffT2 __size2) { const _Iter1 __s = __first1 + __size1 - _DiffT1(__size2 - 1); // Start of pattern match can't go beyond here while (true) { @@ -116,20 +117,17 @@ pair<_Iter1, _Iter1> __search_random_access_impl(_Iter1 __first1, _Sent1 __last1 } } -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter1, _Iter1> __search_impl(_Iter1 __first1, _Sent1 __last1, - _Iter2 __first2, _Sent2 __last2, - _Pred& __pred, - _Proj1& __proj1, - _Proj2& __proj2, - __enable_if_t<__has_random_access_iterator_category<_Iter1>::value - && __has_random_access_iterator_category<_Iter2>::value>* = nullptr) { - +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_impl( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred& __pred, + _Proj1& __proj1, + _Proj2& __proj2, + __enable_if_t<__has_random_access_iterator_category<_Iter1>::value && + __has_random_access_iterator_category<_Iter2>::value>* = nullptr) { auto __size2 = __last2 - __first2; if (__size2 == 0) return std::make_pair(__first1, __first1); @@ -139,42 +137,32 @@ pair<_Iter1, _Iter1> __search_impl(_Iter1 __first1, _Sent1 __last1, return std::make_pair(__last1, __last1); } - return std::__search_random_access_impl<_ClassicAlgPolicy>(__first1, __last1, - __first2, __last2, - __pred, - __proj1, - __proj2, - __size1, - __size2); + return std::__search_random_access_impl<_ClassicAlgPolicy>( + __first1, __last1, __first2, __last2, __pred, __proj1, __proj2, __size1, __size2); } -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter1, _Iter1> __search_impl(_Iter1 __first1, _Sent1 __last1, - _Iter2 __first2, _Sent2 __last2, - _Pred& __pred, - _Proj1& __proj1, - _Proj2& __proj2, - __enable_if_t<__has_forward_iterator_category<_Iter1>::value - && __has_forward_iterator_category<_Iter2>::value - && !(__has_random_access_iterator_category<_Iter1>::value - && __has_random_access_iterator_category<_Iter2>::value)>* = nullptr) { - return std::__search_forward_impl<_ClassicAlgPolicy>(__first1, __last1, - __first2, __last2, - __pred, - __proj1, - __proj2); +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_impl( + _Iter1 __first1, + _Sent1 __last1, + _Iter2 __first2, + _Sent2 __last2, + _Pred& __pred, + _Proj1& __proj1, + _Proj2& __proj2, + __enable_if_t<__has_forward_iterator_category<_Iter1>::value && __has_forward_iterator_category<_Iter2>::value && + !(__has_random_access_iterator_category<_Iter1>::value && + __has_random_access_iterator_category<_Iter2>::value)>* = nullptr) { + return std::__search_forward_impl<_ClassicAlgPolicy>(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2); } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator1 search(_ForwardIterator1 __first1, _ForwardIterator1 __last1, - _ForwardIterator2 __first2, _ForwardIterator2 __last2, - _BinaryPredicate __pred) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 +search(_ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _BinaryPredicate __pred) { static_assert(__is_callable<_BinaryPredicate, decltype(*__first1), decltype(*__first2)>::value, "BinaryPredicate has to be callable"); auto __proj = __identity(); @@ -182,9 +170,8 @@ _ForwardIterator1 search(_ForwardIterator1 __first1, _ForwardIterator1 __last1, } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator1 search(_ForwardIterator1 __first1, _ForwardIterator1 __last1, - _ForwardIterator2 __first2, _ForwardIterator2 __last2) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator1 +search(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) { return std::search(__first1, __last1, __first2, __last2, __equal_to()); } diff --git a/libcxx/include/__algorithm/search_n.h b/libcxx/include/__algorithm/search_n.h index 7e3ddf48a2b6c..c3c01e700bf6a 100644 --- a/libcxx/include/__algorithm/search_n.h +++ b/libcxx/include/__algorithm/search_n.h @@ -31,12 +31,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter, _Iter> __search_n_forward_impl(_Iter __first, _Sent __last, - _SizeT __count, - const _Type& __value, - _Pred& __pred, - _Proj& __proj) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter> __search_n_forward_impl( + _Iter __first, _Sent __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) { if (__count <= 0) return std::make_pair(__first, __first); while (true) { @@ -62,8 +58,7 @@ pair<_Iter, _Iter> __search_n_forward_impl(_Iter __first, _Sent __last, } // if there is a mismatch, restart with a new __first - if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) - { + if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) { __first = __m; ++__first; break; @@ -73,13 +68,8 @@ pair<_Iter, _Iter> __search_n_forward_impl(_Iter __first, _Sent __last, } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -std::pair<_Iter, _Iter> __search_n_random_access_impl(_Iter __first, _Sent __last, - _SizeT __count, - const _Type& __value, - _Pred& __pred, - _Proj& __proj, - _DiffT __size1) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 std::pair<_Iter, _Iter> __search_n_random_access_impl( + _Iter __first, _Sent __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj, _DiffT __size1) { using difference_type = typename iterator_traits<_Iter>::difference_type; if (__count == 0) return std::make_pair(__first, __first); @@ -109,8 +99,7 @@ std::pair<_Iter, _Iter> __search_n_random_access_impl(_Iter __first, _Sent __las ++__m; // no need to check range on __m because __s guarantees we have enough source // if there is a mismatch, restart with a new __first - if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) - { + if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) { __first = __m; ++__first; break; @@ -119,61 +108,44 @@ std::pair<_Iter, _Iter> __search_n_random_access_impl(_Iter __first, _Sent __las } } -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter, _Iter> __search_n_impl(_Iter __first, _Sent __last, - _DiffT __count, - const _Type& __value, - _Pred& __pred, - _Proj& __proj, - __enable_if_t<__has_random_access_iterator_category<_Iter>::value>* = nullptr) { - return std::__search_n_random_access_impl<_ClassicAlgPolicy>(__first, __last, - __count, - __value, - __pred, - __proj, - __last - __first); +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter> __search_n_impl( + _Iter __first, + _Sent __last, + _DiffT __count, + const _Type& __value, + _Pred& __pred, + _Proj& __proj, + __enable_if_t<__has_random_access_iterator_category<_Iter>::value>* = nullptr) { + return std::__search_n_random_access_impl<_ClassicAlgPolicy>( + __first, __last, __count, __value, __pred, __proj, __last - __first); } -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_Iter1, _Iter1> __search_n_impl(_Iter1 __first, _Sent1 __last, - _DiffT __count, - const _Type& __value, - _Pred& __pred, - _Proj& __proj, - __enable_if_t<__has_forward_iterator_category<_Iter1>::value - && !__has_random_access_iterator_category<_Iter1>::value>* = nullptr) { - return std::__search_n_forward_impl<_ClassicAlgPolicy>(__first, __last, - __count, - __value, - __pred, - __proj); +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __search_n_impl( + _Iter1 __first, + _Sent1 __last, + _DiffT __count, + const _Type& __value, + _Pred& __pred, + _Proj& __proj, + __enable_if_t<__has_forward_iterator_category<_Iter1>::value && + !__has_random_access_iterator_category<_Iter1>::value>* = nullptr) { + return std::__search_n_forward_impl<_ClassicAlgPolicy>(__first, __last, __count, __value, __pred, __proj); } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator search_n(_ForwardIterator __first, _ForwardIterator __last, - _Size __count, - const _Tp& __value, - _BinaryPredicate __pred) { - static_assert(__is_callable<_BinaryPredicate, decltype(*__first), const _Tp&>::value, - "BinaryPredicate has to be callable"); +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator search_n( + _ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value, _BinaryPredicate __pred) { + static_assert( + __is_callable<_BinaryPredicate, decltype(*__first), const _Tp&>::value, "BinaryPredicate has to be callable"); auto __proj = __identity(); return std::__search_n_impl(__first, __last, std::__convert_to_integral(__count), __value, __pred, __proj).first; } template -_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_ForwardIterator search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value) { +_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator +search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value) { return std::search_n(__first, __last, std::__convert_to_integral(__count), __value, __equal_to()); } diff --git a/libcxx/include/__algorithm/set_difference.h b/libcxx/include/__algorithm/set_difference.h index 26a300092c91f..a924702ce5f26 100644 --- a/libcxx/include/__algorithm/set_difference.h +++ b/libcxx/include/__algorithm/set_difference.h @@ -55,7 +55,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator set_d _OutputIterator __result, _Compare __comp) { return std::__set_difference<_ClassicAlgPolicy, __comp_ref_type<_Compare> >( - __first1, __last1, __first2, __last2, __result, __comp) + __first1, __last1, __first2, __last2, __result, __comp) .second; } diff --git a/libcxx/include/__algorithm/shift_left.h b/libcxx/include/__algorithm/shift_left.h index c9f2cbb9f7b6f..645c58c291192 100644 --- a/libcxx/include/__algorithm/shift_left.h +++ b/libcxx/include/__algorithm/shift_left.h @@ -22,30 +22,29 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -inline _LIBCPP_HIDE_FROM_ABI constexpr -_ForwardIterator -shift_left(_ForwardIterator __first, _ForwardIterator __last, - typename iterator_traits<_ForwardIterator>::difference_type __n) -{ - if (__n == 0) { - return __last; +inline _LIBCPP_HIDE_FROM_ABI constexpr _ForwardIterator +shift_left(_ForwardIterator __first, + _ForwardIterator __last, + typename iterator_traits<_ForwardIterator>::difference_type __n) { + if (__n == 0) { + return __last; + } + + _ForwardIterator __m = __first; + if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) { + if (__n >= __last - __first) { + return __first; } - - _ForwardIterator __m = __first; - if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) { - if (__n >= __last - __first) { - return __first; - } - __m += __n; - } else { - for (; __n > 0; --__n) { - if (__m == __last) { - return __first; - } - ++__m; - } + __m += __n; + } else { + for (; __n > 0; --__n) { + if (__m == __last) { + return __first; + } + ++__m; } - return std::move(__m, __last, __first); + } + return std::move(__m, __last, __first); } #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__algorithm/shift_right.h b/libcxx/include/__algorithm/shift_right.h index 70983e327bcb9..73ef98bd39ded 100644 --- a/libcxx/include/__algorithm/shift_right.h +++ b/libcxx/include/__algorithm/shift_right.h @@ -25,73 +25,72 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -inline _LIBCPP_HIDE_FROM_ABI constexpr -_ForwardIterator -shift_right(_ForwardIterator __first, _ForwardIterator __last, - typename iterator_traits<_ForwardIterator>::difference_type __n) -{ - if (__n == 0) { - return __first; - } +inline _LIBCPP_HIDE_FROM_ABI constexpr _ForwardIterator +shift_right(_ForwardIterator __first, + _ForwardIterator __last, + typename iterator_traits<_ForwardIterator>::difference_type __n) { + if (__n == 0) { + return __first; + } - if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) { - decltype(__n) __d = __last - __first; - if (__n >= __d) { - return __last; - } - _ForwardIterator __m = __first + (__d - __n); - return std::move_backward(__first, __m, __last); - } else if constexpr (__has_bidirectional_iterator_category<_ForwardIterator>::value) { - _ForwardIterator __m = __last; - for (; __n > 0; --__n) { - if (__m == __first) { - return __last; - } - --__m; - } - return std::move_backward(__first, __m, __last); - } else { - _ForwardIterator __ret = __first; - for (; __n > 0; --__n) { - if (__ret == __last) { - return __last; - } - ++__ret; - } + if constexpr (__has_random_access_iterator_category<_ForwardIterator>::value) { + decltype(__n) __d = __last - __first; + if (__n >= __d) { + return __last; + } + _ForwardIterator __m = __first + (__d - __n); + return std::move_backward(__first, __m, __last); + } else if constexpr (__has_bidirectional_iterator_category<_ForwardIterator>::value) { + _ForwardIterator __m = __last; + for (; __n > 0; --__n) { + if (__m == __first) { + return __last; + } + --__m; + } + return std::move_backward(__first, __m, __last); + } else { + _ForwardIterator __ret = __first; + for (; __n > 0; --__n) { + if (__ret == __last) { + return __last; + } + ++__ret; + } - // We have an __n-element scratch space from __first to __ret. - // Slide an __n-element window [__trail, __lead) from left to right. - // We're essentially doing swap_ranges(__first, __ret, __trail, __lead) - // over and over; but once __lead reaches __last we needn't bother - // to save the values of elements [__trail, __last). + // We have an __n-element scratch space from __first to __ret. + // Slide an __n-element window [__trail, __lead) from left to right. + // We're essentially doing swap_ranges(__first, __ret, __trail, __lead) + // over and over; but once __lead reaches __last we needn't bother + // to save the values of elements [__trail, __last). - auto __trail = __first; - auto __lead = __ret; - while (__trail != __ret) { - if (__lead == __last) { - std::move(__first, __trail, __ret); - return __ret; - } - ++__trail; - ++__lead; - } + auto __trail = __first; + auto __lead = __ret; + while (__trail != __ret) { + if (__lead == __last) { + std::move(__first, __trail, __ret); + return __ret; + } + ++__trail; + ++__lead; + } - _ForwardIterator __mid = __first; - while (true) { - if (__lead == __last) { - __trail = std::move(__mid, __ret, __trail); - std::move(__first, __mid, __trail); - return __ret; - } - swap(*__mid, *__trail); - ++__mid; - ++__trail; - ++__lead; - if (__mid == __ret) { - __mid = __first; - } - } + _ForwardIterator __mid = __first; + while (true) { + if (__lead == __last) { + __trail = std::move(__mid, __ret, __trail); + std::move(__first, __mid, __trail); + return __ret; + } + swap(*__mid, *__trail); + ++__mid; + ++__trail; + ++__lead; + if (__mid == __ret) { + __mid = __first; + } } + } } #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__algorithm/shuffle.h b/libcxx/include/__algorithm/shuffle.h index 6fee88ca6ff25..c9c56ce8c2c0b 100644 --- a/libcxx/include/__algorithm/shuffle.h +++ b/libcxx/include/__algorithm/shuffle.h @@ -32,8 +32,8 @@ class _LIBCPP_EXPORTED_FROM_ABI __libcpp_debug_randomizer { public: _LIBCPP_HIDE_FROM_ABI __libcpp_debug_randomizer() { __state_ = __seed(); - __inc_ = __state_ + 0xda3e39cb94b95bdbULL; - __inc_ = (__inc_ << 1) | 1; + __inc_ = __state_ + 0xda3e39cb94b95bdbULL; + __inc_ = (__inc_ << 1) | 1; } typedef uint_fast32_t result_type; @@ -42,7 +42,7 @@ class _LIBCPP_EXPORTED_FROM_ABI __libcpp_debug_randomizer { _LIBCPP_HIDE_FROM_ABI result_type operator()() { uint_fast64_t __oldstate = __state_; - __state_ = __oldstate * 6364136223846793005ULL + __inc_; + __state_ = __oldstate * 6364136223846793005ULL + __inc_; return __oldstate >> 32; } @@ -62,102 +62,95 @@ class _LIBCPP_EXPORTED_FROM_ABI __libcpp_debug_randomizer { } }; -#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE) \ - || defined(_LIBCPP_BUILDING_LIBRARY) +#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE) || defined(_LIBCPP_BUILDING_LIBRARY) class _LIBCPP_EXPORTED_FROM_ABI __rs_default; _LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get(); -class _LIBCPP_EXPORTED_FROM_ABI __rs_default -{ - static unsigned __c_; +class _LIBCPP_EXPORTED_FROM_ABI __rs_default { + static unsigned __c_; + + __rs_default(); - __rs_default(); public: - typedef uint_fast32_t result_type; + typedef uint_fast32_t result_type; - static const result_type _Min = 0; - static const result_type _Max = 0xFFFFFFFF; + static const result_type _Min = 0; + static const result_type _Max = 0xFFFFFFFF; - __rs_default(const __rs_default&); - ~__rs_default(); + __rs_default(const __rs_default&); + ~__rs_default(); - result_type operator()(); + result_type operator()(); - static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type min() {return _Min;} - static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type max() {return _Max;} + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type min() { return _Min; } + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR result_type max() { return _Max; } - friend _LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get(); + friend _LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get(); }; _LIBCPP_EXPORTED_FROM_ABI __rs_default __rs_get(); template _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX14 void -random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last) -{ - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - typedef uniform_int_distribution _Dp; - typedef typename _Dp::param_type _Pp; - difference_type __d = __last - __first; - if (__d > 1) - { - _Dp __uid; - __rs_default __g = __rs_get(); - for (--__last, (void) --__d; __first < __last; ++__first, (void) --__d) - { - difference_type __i = __uid(__g, _Pp(0, __d)); - if (__i != difference_type(0)) - swap(*__first, *(__first + __i)); - } +random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last) { + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + typedef uniform_int_distribution _Dp; + typedef typename _Dp::param_type _Pp; + difference_type __d = __last - __first; + if (__d > 1) { + _Dp __uid; + __rs_default __g = __rs_get(); + for (--__last, (void)--__d; __first < __last; ++__first, (void)--__d) { + difference_type __i = __uid(__g, _Pp(0, __d)); + if (__i != difference_type(0)) + swap(*__first, *(__first + __i)); } + } } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX14 void -random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last, -#ifndef _LIBCPP_CXX03_LANG +random_shuffle(_RandomAccessIterator __first, + _RandomAccessIterator __last, +# ifndef _LIBCPP_CXX03_LANG _RandomNumberGenerator&& __rand) -#else +# else _RandomNumberGenerator& __rand) -#endif +# endif { - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - difference_type __d = __last - __first; - if (__d > 1) - { - for (--__last; __first < __last; ++__first, (void) --__d) - { - difference_type __i = __rand(__d); - if (__i != difference_type(0)) - swap(*__first, *(__first + __i)); - } + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + difference_type __d = __last - __first; + if (__d > 1) { + for (--__last; __first < __last; ++__first, (void)--__d) { + difference_type __i = __rand(__d); + if (__i != difference_type(0)) + swap(*__first, *(__first + __i)); } + } } #endif template -_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator __shuffle( - _RandomAccessIterator __first, _Sentinel __last_sentinel, _UniformRandomNumberGenerator&& __g) { - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - typedef uniform_int_distribution _Dp; - typedef typename _Dp::param_type _Pp; - - auto __original_last = _IterOps<_AlgPolicy>::next(__first, __last_sentinel); - auto __last = __original_last; - difference_type __d = __last - __first; - if (__d > 1) - { - _Dp __uid; - for (--__last, (void) --__d; __first < __last; ++__first, (void) --__d) - { - difference_type __i = __uid(__g, _Pp(0, __d)); - if (__i != difference_type(0)) - _IterOps<_AlgPolicy>::iter_swap(__first, __first + __i); - } +_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator +__shuffle(_RandomAccessIterator __first, _Sentinel __last_sentinel, _UniformRandomNumberGenerator&& __g) { + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + typedef uniform_int_distribution _Dp; + typedef typename _Dp::param_type _Pp; + + auto __original_last = _IterOps<_AlgPolicy>::next(__first, __last_sentinel); + auto __last = __original_last; + difference_type __d = __last - __first; + if (__d > 1) { + _Dp __uid; + for (--__last, (void)--__d; __first < __last; ++__first, (void)--__d) { + difference_type __i = __uid(__g, _Pp(0, __d)); + if (__i != difference_type(0)) + _IterOps<_AlgPolicy>::iter_swap(__first, __first + __i); } + } - return __original_last; + return __original_last; } template diff --git a/libcxx/include/__algorithm/sift_down.h b/libcxx/include/__algorithm/sift_down.h index 3a222f7c7f1b6..7f152e4dbd7f3 100644 --- a/libcxx/include/__algorithm/sift_down.h +++ b/libcxx/include/__algorithm/sift_down.h @@ -26,90 +26,89 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void -__sift_down(_RandomAccessIterator __first, _Compare&& __comp, +__sift_down(_RandomAccessIterator __first, + _Compare&& __comp, typename iterator_traits<_RandomAccessIterator>::difference_type __len, - _RandomAccessIterator __start) -{ - using _Ops = _IterOps<_AlgPolicy>; + _RandomAccessIterator __start) { + using _Ops = _IterOps<_AlgPolicy>; - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - // left-child of __start is at 2 * __start + 1 - // right-child of __start is at 2 * __start + 2 - difference_type __child = __start - __first; + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + // left-child of __start is at 2 * __start + 1 + // right-child of __start is at 2 * __start + 2 + difference_type __child = __start - __first; - if (__len < 2 || (__len - 2) / 2 < __child) - return; + if (__len < 2 || (__len - 2) / 2 < __child) + return; - __child = 2 * __child + 1; - _RandomAccessIterator __child_i = __first + __child; + __child = 2 * __child + 1; + _RandomAccessIterator __child_i = __first + __child; + + if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) { + // right-child exists and is greater than left-child + ++__child_i; + ++__child; + } + + // check if we are in heap-order + if (__comp(*__child_i, *__start)) + // we are, __start is larger than its largest child + return; + + value_type __top(_Ops::__iter_move(__start)); + do { + // we are not in heap-order, swap the parent with its largest child + *__start = _Ops::__iter_move(__child_i); + __start = __child_i; + + if ((__len - 2) / 2 < __child) + break; + + // recompute the child based off of the updated parent + __child = 2 * __child + 1; + __child_i = __first + __child; if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) { - // right-child exists and is greater than left-child - ++__child_i; - ++__child; + // right-child exists and is greater than left-child + ++__child_i; + ++__child; } // check if we are in heap-order - if (__comp(*__child_i, *__start)) - // we are, __start is larger than its largest child - return; - - value_type __top(_Ops::__iter_move(__start)); - do - { - // we are not in heap-order, swap the parent with its largest child - *__start = _Ops::__iter_move(__child_i); - __start = __child_i; - - if ((__len - 2) / 2 < __child) - break; - - // recompute the child based off of the updated parent - __child = 2 * __child + 1; - __child_i = __first + __child; - - if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) { - // right-child exists and is greater than left-child - ++__child_i; - ++__child; - } - - // check if we are in heap-order - } while (!__comp(*__child_i, __top)); - *__start = std::move(__top); + } while (!__comp(*__child_i, __top)); + *__start = std::move(__top); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator -__floyd_sift_down(_RandomAccessIterator __first, _Compare&& __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len) -{ - using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; - _LIBCPP_ASSERT_UNCATEGORIZED(__len >= 2, "shouldn't be called unless __len >= 2"); - - _RandomAccessIterator __hole = __first; - _RandomAccessIterator __child_i = __first; - difference_type __child = 0; - - while (true) { - __child_i += difference_type(__child + 1); - __child = 2 * __child + 1; - - if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) { - // right-child exists and is greater than left-child - ++__child_i; - ++__child; - } - - // swap __hole with its largest child - *__hole = _IterOps<_AlgPolicy>::__iter_move(__child_i); - __hole = __child_i; - - // if __hole is now a leaf, we're done - if (__child > (__len - 2) / 2) - return __hole; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __floyd_sift_down( + _RandomAccessIterator __first, + _Compare&& __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len) { + using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; + _LIBCPP_ASSERT_UNCATEGORIZED(__len >= 2, "shouldn't be called unless __len >= 2"); + + _RandomAccessIterator __hole = __first; + _RandomAccessIterator __child_i = __first; + difference_type __child = 0; + + while (true) { + __child_i += difference_type(__child + 1); + __child = 2 * __child + 1; + + if ((__child + 1) < __len && __comp(*__child_i, *(__child_i + difference_type(1)))) { + // right-child exists and is greater than left-child + ++__child_i; + ++__child; } + + // swap __hole with its largest child + *__hole = _IterOps<_AlgPolicy>::__iter_move(__child_i); + __hole = __child_i; + + // if __hole is now a leaf, we're done + if (__child > (__len - 2) / 2) + return __hole; + } } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h index 567c988ff0d3c..1b878c33c7a16 100644 --- a/libcxx/include/__algorithm/sort.h +++ b/libcxx/include/__algorithm/sort.h @@ -44,49 +44,47 @@ _LIBCPP_BEGIN_NAMESPACE_STD // stable, 2-3 compares, 0-2 swaps template -_LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 unsigned __sort3(_ForwardIterator __x, _ForwardIterator __y, _ForwardIterator __z, - _Compare __c) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 unsigned +__sort3(_ForwardIterator __x, _ForwardIterator __y, _ForwardIterator __z, _Compare __c) { using _Ops = _IterOps<_AlgPolicy>; unsigned __r = 0; - if (!__c(*__y, *__x)) // if x <= y + if (!__c(*__y, *__x)) // if x <= y { - if (!__c(*__z, *__y)) // if y <= z - return __r; // x <= y && y <= z - // x <= y && y > z - _Ops::iter_swap(__y, __z); // x <= z && y < z + if (!__c(*__z, *__y)) // if y <= z + return __r; // x <= y && y <= z + // x <= y && y > z + _Ops::iter_swap(__y, __z); // x <= z && y < z __r = 1; - if (__c(*__y, *__x)) // if x > y + if (__c(*__y, *__x)) // if x > y { - _Ops::iter_swap(__x, __y); // x < y && y <= z + _Ops::iter_swap(__x, __y); // x < y && y <= z __r = 2; } - return __r; // x <= y && y < z + return __r; // x <= y && y < z } - if (__c(*__z, *__y)) // x > y, if y > z + if (__c(*__z, *__y)) // x > y, if y > z { - _Ops::iter_swap(__x, __z); // x < y && y < z + _Ops::iter_swap(__x, __z); // x < y && y < z __r = 1; return __r; } - _Ops::iter_swap(__x, __y); // x > y && y <= z - __r = 1; // x < y && x <= z - if (__c(*__z, *__y)) // if y > z + _Ops::iter_swap(__x, __y); // x > y && y <= z + __r = 1; // x < y && x <= z + if (__c(*__z, *__y)) // if y > z { - _Ops::iter_swap(__y, __z); // x <= y && y < z + _Ops::iter_swap(__y, __z); // x <= y && y < z __r = 2; } return __r; -} // x <= y && y <= z +} // x <= y && y <= z // stable, 3-6 compares, 0-5 swaps template -_LIBCPP_HIDE_FROM_ABI -void __sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, - _Compare __c) { - using _Ops = _IterOps<_AlgPolicy>; +_LIBCPP_HIDE_FROM_ABI void +__sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, _Compare __c) { + using _Ops = _IterOps<_AlgPolicy>; std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c); if (__c(*__x4, *__x3)) { _Ops::iter_swap(__x3, __x4); @@ -102,8 +100,13 @@ void __sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3 // stable, 4-10 compares, 0-9 swaps template -_LIBCPP_HIDE_FROM_ABI void __sort5(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, - _ForwardIterator __x4, _ForwardIterator __x5, _Comp __comp) { +_LIBCPP_HIDE_FROM_ABI void +__sort5(_ForwardIterator __x1, + _ForwardIterator __x2, + _ForwardIterator __x3, + _ForwardIterator __x4, + _ForwardIterator __x5, + _Comp __comp) { using _Ops = _IterOps<_AlgPolicy>; std::__sort4<_AlgPolicy, _Comp>(__x1, __x2, __x3, __x4, __comp); @@ -139,8 +142,9 @@ struct __is_simple_comparator : true_type {}; template ::value_type> using __use_branchless_sort = - integral_constant::value && sizeof(_Tp) <= sizeof(void*) && - is_arithmetic<_Tp>::value && __is_simple_comparator<_Compare>::value>; + integral_constant::value && sizeof(_Tp) <= sizeof(void*) && + is_arithmetic<_Tp>::value && __is_simple_comparator<_Compare>::value>; namespace __detail { @@ -154,49 +158,56 @@ template inline _LIBCPP_HIDE_FROM_ABI void __cond_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _Compare __c) { // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`). using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; - bool __r = __c(*__x, *__y); + bool __r = __c(*__x, *__y); value_type __tmp = __r ? *__x : *__y; - *__y = __r ? *__y : *__x; - *__x = __tmp; + *__y = __r ? *__y : *__x; + *__x = __tmp; } // Ensures that *__x, *__y and *__z are ordered according to the comparator __c, // under the assumption that *__y and *__z are already ordered. template -inline _LIBCPP_HIDE_FROM_ABI void __partially_sorted_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, - _RandomAccessIterator __z, _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void +__partially_sorted_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _RandomAccessIterator __z, _Compare __c) { // Note: this function behaves correctly even with proxy iterators (because it relies on `value_type`). using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; - bool __r = __c(*__z, *__x); + bool __r = __c(*__z, *__x); value_type __tmp = __r ? *__z : *__x; - *__z = __r ? *__x : *__z; - __r = __c(__tmp, *__y); - *__x = __r ? *__x : *__y; - *__y = __r ? *__y : __tmp; + *__z = __r ? *__x : *__z; + __r = __c(__tmp, *__y); + *__x = __r ? *__x : *__y; + *__y = __r ? *__y : __tmp; } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort3_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, - _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless( + _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) { std::__cond_swap<_Compare>(__x2, __x3, __c); std::__partially_sorted_swap<_Compare>(__x1, __x2, __x3, __c); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort3_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, - _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void __sort3_maybe_branchless( + _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, _Compare __c) { std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort4_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, - _RandomAccessIterator __x4, _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless( + _RandomAccessIterator __x1, + _RandomAccessIterator __x2, + _RandomAccessIterator __x3, + _RandomAccessIterator __x4, + _Compare __c) { std::__cond_swap<_Compare>(__x1, __x3, __c); std::__cond_swap<_Compare>(__x2, __x4, __c); std::__cond_swap<_Compare>(__x1, __x2, __c); @@ -204,18 +215,24 @@ __sort4_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, std::__cond_swap<_Compare>(__x2, __x3, __c); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort4_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, - _RandomAccessIterator __x4, _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void __sort4_maybe_branchless( + _RandomAccessIterator __x1, + _RandomAccessIterator __x2, + _RandomAccessIterator __x3, + _RandomAccessIterator __x4, + _Compare __c) { std::__sort4<_AlgPolicy, _Compare>(__x1, __x2, __x3, __x4, __c); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort5_maybe_branchless( +inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless( _RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, @@ -230,20 +247,25 @@ __sort5_maybe_branchless( std::__partially_sorted_swap<_Compare>(__x2, __x3, __x4, __c); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI void -__sort5_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, - _RandomAccessIterator __x4, _RandomAccessIterator __x5, _Compare __c) { +inline _LIBCPP_HIDE_FROM_ABI void __sort5_maybe_branchless( + _RandomAccessIterator __x1, + _RandomAccessIterator __x2, + _RandomAccessIterator __x3, + _RandomAccessIterator __x4, + _RandomAccessIterator __x5, + _Compare __c) { std::__sort5<_AlgPolicy, _Compare, _RandomAccessIterator>( std::move(__x1), std::move(__x2), std::move(__x3), std::move(__x4), std::move(__x5), __c); } // Assumes size > 0 template -_LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX14 void __selection_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, - _Compare __comp) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__selection_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { _BidirectionalIterator __lm1 = __last; for (--__lm1; __first != __lm1; ++__first) { _BidirectionalIterator __i = std::__min_element<_Compare>(__first, __last, __comp); @@ -255,8 +277,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX14 void __selection_sort(_BidirectionalIterator __fir // Sort the iterator range [__first, __last) using the comparator __comp using // the insertion sort algorithm. template -_LIBCPP_HIDE_FROM_ABI -void __insertion_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { +_LIBCPP_HIDE_FROM_ABI void +__insertion_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { using _Ops = _IterOps<_AlgPolicy>; typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; @@ -292,16 +314,17 @@ __insertion_sort_unguarded(_RandomAccessIterator const __first, _RandomAccessIte typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; if (__first == __last) return; - const _RandomAccessIterator __leftmost = __first - difference_type(1); (void)__leftmost; // can be unused when assertions are disabled + const _RandomAccessIterator __leftmost = __first - difference_type(1); + (void)__leftmost; // can be unused when assertions are disabled for (_RandomAccessIterator __i = __first + difference_type(1); __i != __last; ++__i) { _RandomAccessIterator __j = __i - difference_type(1); if (__comp(*__i, *__j)) { value_type __t(_Ops::__iter_move(__i)); _RandomAccessIterator __k = __j; - __j = __i; + __j = __i; do { *__j = _Ops::__iter_move(__k); - __j = __k; + __j = __k; _LIBCPP_ASSERT_UNCATEGORIZED( __k != __leftmost, "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?"); @@ -312,8 +335,8 @@ __insertion_sort_unguarded(_RandomAccessIterator const __first, _RandomAccessIte } template -_LIBCPP_HIDE_FROM_ABI bool __insertion_sort_incomplete( - _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { +_LIBCPP_HIDE_FROM_ABI bool +__insertion_sort_incomplete(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { using _Ops = _IterOps<_AlgPolicy>; typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; @@ -334,23 +357,27 @@ _LIBCPP_HIDE_FROM_ABI bool __insertion_sort_incomplete( return true; case 5: std::__sort5_maybe_branchless<_AlgPolicy, _Comp>( - __first, __first + difference_type(1), __first + difference_type(2), __first + difference_type(3), - --__last, __comp); + __first, + __first + difference_type(1), + __first + difference_type(2), + __first + difference_type(3), + --__last, + __comp); return true; } typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; _RandomAccessIterator __j = __first + difference_type(2); std::__sort3_maybe_branchless<_AlgPolicy, _Comp>(__first, __first + difference_type(1), __j, __comp); const unsigned __limit = 8; - unsigned __count = 0; + unsigned __count = 0; for (_RandomAccessIterator __i = __j + difference_type(1); __i != __last; ++__i) { if (__comp(*__i, *__j)) { value_type __t(_Ops::__iter_move(__i)); _RandomAccessIterator __k = __j; - __j = __i; + __j = __i; do { *__j = _Ops::__iter_move(__k); - __j = __k; + __j = __k; } while (__j != __first && __comp(__t, *--__k)); *__j = std::move(__t); if (++__count == __limit) @@ -507,8 +534,9 @@ __bitset_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; _LIBCPP_ASSERT_UNCATEGORIZED(__last - __first >= difference_type(3), ""); - const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around - const _RandomAccessIterator __end = __last; (void)__end; // + const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around + const _RandomAccessIterator __end = __last; + (void)__end; // value_type __pivot(_Ops::__iter_move(__first)); // Find the first element greater than the pivot. @@ -598,8 +626,9 @@ __partition_with_equals_on_right(_RandomAccessIterator __first, _RandomAccessIte typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; _LIBCPP_ASSERT_UNCATEGORIZED(__last - __first >= difference_type(3), ""); - const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around - const _RandomAccessIterator __end = __last; (void)__end; // + const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around + const _RandomAccessIterator __end = __last; + (void)__end; // value_type __pivot(_Ops::__iter_move(__first)); // Find the first element greater or equal to the pivot. It will be always // guarded because __introsort will do the median-of-three before calling @@ -665,8 +694,9 @@ __partition_with_equals_on_left(_RandomAccessIterator __first, _RandomAccessIter typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; // TODO(LLVM18): Make __begin const, see https://reviews.llvm.org/D147089#4349748 - _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around - const _RandomAccessIterator __end = __last; (void)__end; // + _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around + const _RandomAccessIterator __end = __last; + (void)__end; // value_type __pivot(_Ops::__iter_move(__first)); if (__comp(__pivot, *(__last - difference_type(1)))) { // Guarded. @@ -753,8 +783,12 @@ void __introsort(_RandomAccessIterator __first, return; case 5: std::__sort5_maybe_branchless<_AlgPolicy, _Compare>( - __first, __first + difference_type(1), __first + difference_type(2), __first + difference_type(3), - --__last, __comp); + __first, + __first + difference_type(1), + __first + difference_type(2), + __first + difference_type(3), + --__last, + __comp); return; } // Use insertion sort if the length of the range is below the specified limit. @@ -803,10 +837,10 @@ void __introsort(_RandomAccessIterator __first, continue; } // Use bitset partition only if asked for. - auto __ret = - _UseBitSetPartition - ? std::__bitset_partition<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp) - : std::__partition_with_equals_on_right<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp); + auto __ret = _UseBitSetPartition + ? std::__bitset_partition<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp) + : std::__partition_with_equals_on_right<_AlgPolicy, _RandomAccessIterator, _Compare>( + __first, __last, __comp); _RandomAccessIterator __i = __ret.first; // [__first, __i) < *__i and *__i <= [__i+1, __last) // If we were given a perfect partition, see if insertion sort is quick... @@ -858,19 +892,27 @@ extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, char*>(char #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, wchar_t*>(wchar_t*, wchar_t*, __less&); #endif -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, signed char*>(signed char*, signed char*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned char*>(unsigned char*, unsigned char*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, signed char*>(signed char*, signed char*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, unsigned char*>(unsigned char*, unsigned char*, __less&); extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, short*>(short*, short*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned short*>(unsigned short*, unsigned short*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, unsigned short*>(unsigned short*, unsigned short*, __less&); extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, int*>(int*, int*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned*>(unsigned*, unsigned*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, unsigned*>(unsigned*, unsigned*, __less&); extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, long*>(long*, long*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned long*>(unsigned long*, unsigned long*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, long long*>(long long*, long long*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned long long*>(unsigned long long*, unsigned long long*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, unsigned long*>(unsigned long*, unsigned long*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, long long*>(long long*, long long*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, unsigned long long*>( + unsigned long long*, unsigned long long*, __less&); extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, float*>(float*, float*, __less&); extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, double*>(double*, double*, __less&); -extern template _LIBCPP_EXPORTED_FROM_ABI void __sort<__less&, long double*>(long double*, long double*, __less&); +extern template _LIBCPP_EXPORTED_FROM_ABI void +__sort<__less&, long double*>(long double*, long double*, __less&); template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void @@ -884,8 +926,7 @@ __sort_dispatch(_RandomAccessIterator __first, _RandomAccessIterator __last, _Co std::__introsort<_AlgPolicy, _Comp&, _RandomAccessIterator, - __use_branchless_sort<_Comp, _RandomAccessIterator>::value>( - __first, __last, __comp, __depth_limit); + __use_branchless_sort<_Comp, _RandomAccessIterator>::value>(__first, __last, __comp, __depth_limit); } template @@ -941,8 +982,8 @@ _LIBCPP_HIDE_FROM_ABI void __sort_dispatch(_Type* __first, _Type* __last, ranges #endif template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void __sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +__sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) { std::__debug_randomize_range<_AlgPolicy>(__first, __last); if (__libcpp_is_constant_evaluated()) { @@ -955,14 +996,14 @@ void __sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _C } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { std::__sort_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void sort(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +sort(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::sort(__first, __last, __less<>()); } diff --git a/libcxx/include/__algorithm/sort_heap.h b/libcxx/include/__algorithm/sort_heap.h index a82926ed1ccac..0a6d992d0090e 100644 --- a/libcxx/include/__algorithm/sort_heap.h +++ b/libcxx/include/__algorithm/sort_heap.h @@ -27,20 +27,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) { - _RandomAccessIterator __saved_last = __last; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void +__sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) { + _RandomAccessIterator __saved_last = __last; __comp_ref_type<_Compare> __comp_ref = __comp; using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; - for (difference_type __n = __last - __first; __n > 1; --__last, (void) --__n) + for (difference_type __n = __last - __first; __n > 1; --__last, (void)--__n) std::__pop_heap<_AlgPolicy>(__first, __last, __comp_ref, __n); std::__check_strict_weak_ordering_sorted(__first, __saved_last, __comp_ref); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { static_assert(std::is_copy_constructible<_RandomAccessIterator>::value, "Iterators must be copy constructible."); static_assert(std::is_copy_assignable<_RandomAccessIterator>::value, "Iterators must be copy assignable."); @@ -48,8 +48,8 @@ void sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Com } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -void sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void +sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::sort_heap(std::move(__first), std::move(__last), __less<>()); } diff --git a/libcxx/include/__algorithm/stable_partition.h b/libcxx/include/__algorithm/stable_partition.h index 0e92a0a5f73d9..8762abcf18e15 100644 --- a/libcxx/include/__algorithm/stable_partition.h +++ b/libcxx/include/__algorithm/stable_partition.h @@ -29,296 +29,269 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _ForwardIterator -__stable_partition_impl(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, - _Distance __len, _Pair __p, forward_iterator_tag __fit) -{ - using _Ops = _IterOps<_AlgPolicy>; +_LIBCPP_HIDE_FROM_ABI _ForwardIterator __stable_partition_impl( + _ForwardIterator __first, + _ForwardIterator __last, + _Predicate __pred, + _Distance __len, + _Pair __p, + forward_iterator_tag __fit) { + using _Ops = _IterOps<_AlgPolicy>; - // *__first is known to be false - // __len >= 1 - if (__len == 1) - return __first; - if (__len == 2) - { - _ForwardIterator __m = __first; - if (__pred(*++__m)) - { - _Ops::iter_swap(__first, __m); - return __m; - } - return __first; + // *__first is known to be false + // __len >= 1 + if (__len == 1) + return __first; + if (__len == 2) { + _ForwardIterator __m = __first; + if (__pred(*++__m)) { + _Ops::iter_swap(__first, __m); + return __m; } - if (__len <= __p.second) - { // The buffer is big enough to use - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; - __destruct_n __d(0); - unique_ptr __h(__p.first, __d); - // Move the falses into the temporary buffer, and the trues to the front of the line - // Update __first to always point to the end of the trues - value_type* __t = __p.first; - ::new ((void*)__t) value_type(_Ops::__iter_move(__first)); + return __first; + } + if (__len <= __p.second) { // The buffer is big enough to use + typedef typename iterator_traits<_ForwardIterator>::value_type value_type; + __destruct_n __d(0); + unique_ptr __h(__p.first, __d); + // Move the falses into the temporary buffer, and the trues to the front of the line + // Update __first to always point to the end of the trues + value_type* __t = __p.first; + ::new ((void*)__t) value_type(_Ops::__iter_move(__first)); + __d.template __incr(); + ++__t; + _ForwardIterator __i = __first; + while (++__i != __last) { + if (__pred(*__i)) { + *__first = _Ops::__iter_move(__i); + ++__first; + } else { + ::new ((void*)__t) value_type(_Ops::__iter_move(__i)); __d.template __incr(); ++__t; - _ForwardIterator __i = __first; - while (++__i != __last) - { - if (__pred(*__i)) - { - *__first = _Ops::__iter_move(__i); - ++__first; - } - else - { - ::new ((void*)__t) value_type(_Ops::__iter_move(__i)); - __d.template __incr(); - ++__t; - } - } - // All trues now at start of range, all falses in buffer - // Move falses back into range, but don't mess up __first which points to first false - __i = __first; - for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void) ++__i) - *__i = _Ops::__iter_move(__t2); - // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer - return __first; - } - // Else not enough buffer, do in place - // __len >= 3 - _ForwardIterator __m = __first; - _Distance __len2 = __len / 2; // __len2 >= 2 - _Ops::advance(__m, __len2); - // recurse on [__first, __m), *__first know to be false - // F????????????????? - // f m l - _ForwardIterator __first_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - __first, __m, __pred, __len2, __p, __fit); - // TTTFFFFF?????????? - // f ff m l - // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true - _ForwardIterator __m1 = __m; - _ForwardIterator __second_false = __last; - _Distance __len_half = __len - __len2; - while (__pred(*__m1)) - { - if (++__m1 == __last) - goto __second_half_done; - --__len_half; + } } - // TTTFFFFFTTTF?????? - // f ff m m1 l - __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - __m1, __last, __pred, __len_half, __p, __fit); + // All trues now at start of range, all falses in buffer + // Move falses back into range, but don't mess up __first which points to first false + __i = __first; + for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void)++__i) + *__i = _Ops::__iter_move(__t2); + // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer + return __first; + } + // Else not enough buffer, do in place + // __len >= 3 + _ForwardIterator __m = __first; + _Distance __len2 = __len / 2; // __len2 >= 2 + _Ops::advance(__m, __len2); + // recurse on [__first, __m), *__first know to be false + // F????????????????? + // f m l + _ForwardIterator __first_false = + std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__first, __m, __pred, __len2, __p, __fit); + // TTTFFFFF?????????? + // f ff m l + // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true + _ForwardIterator __m1 = __m; + _ForwardIterator __second_false = __last; + _Distance __len_half = __len - __len2; + while (__pred(*__m1)) { + if (++__m1 == __last) + goto __second_half_done; + --__len_half; + } + // TTTFFFFFTTTF?????? + // f ff m m1 l + __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__m1, __last, __pred, __len_half, __p, __fit); __second_half_done: - // TTTFFFFFTTTTTFFFFF - // f ff m sf l - return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first; - // TTTTTTTTFFFFFFFFFF - // | + // TTTFFFFFTTTTTFFFFF + // f ff m sf l + return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first; + // TTTTTTTTFFFFFFFFFF + // | } template _LIBCPP_HIDE_FROM_ABI _ForwardIterator -__stable_partition_impl(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, - forward_iterator_tag) -{ - typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type; - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; +__stable_partition_impl(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, forward_iterator_tag) { + typedef typename iterator_traits<_ForwardIterator>::difference_type difference_type; + typedef typename iterator_traits<_ForwardIterator>::value_type value_type; - const difference_type __alloc_limit = 3; // might want to make this a function of trivial assignment - // Either prove all true and return __first or point to first false - while (true) - { - if (__first == __last) - return __first; - if (!__pred(*__first)) - break; - ++__first; - } - // We now have a reduced range [__first, __last) - // *__first is known to be false - difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last); - pair __p(0, 0); - unique_ptr __h; - if (__len >= __alloc_limit) - { -// TODO: Remove the use of std::get_temporary_buffer -_LIBCPP_SUPPRESS_DEPRECATED_PUSH - __p = std::get_temporary_buffer(__len); -_LIBCPP_SUPPRESS_DEPRECATED_POP - __h.reset(__p.first); - } - return std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - std::move(__first), std::move(__last), __pred, __len, __p, forward_iterator_tag()); + const difference_type __alloc_limit = 3; // might want to make this a function of trivial assignment + // Either prove all true and return __first or point to first false + while (true) { + if (__first == __last) + return __first; + if (!__pred(*__first)) + break; + ++__first; + } + // We now have a reduced range [__first, __last) + // *__first is known to be false + difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last); + pair __p(0, 0); + unique_ptr __h; + if (__len >= __alloc_limit) { + // TODO: Remove the use of std::get_temporary_buffer + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + __p = std::get_temporary_buffer(__len); + _LIBCPP_SUPPRESS_DEPRECATED_POP + __h.reset(__p.first); + } + return std::__stable_partition_impl<_AlgPolicy, _Predicate&>( + std::move(__first), std::move(__last), __pred, __len, __p, forward_iterator_tag()); } template -_BidirectionalIterator -__stable_partition_impl(_BidirectionalIterator __first, _BidirectionalIterator __last, _Predicate __pred, - _Distance __len, _Pair __p, bidirectional_iterator_tag __bit) -{ - using _Ops = _IterOps<_AlgPolicy>; +_BidirectionalIterator __stable_partition_impl( + _BidirectionalIterator __first, + _BidirectionalIterator __last, + _Predicate __pred, + _Distance __len, + _Pair __p, + bidirectional_iterator_tag __bit) { + using _Ops = _IterOps<_AlgPolicy>; - // *__first is known to be false - // *__last is known to be true - // __len >= 2 - if (__len == 2) - { - _Ops::iter_swap(__first, __last); - return __last; - } - if (__len == 3) - { - _BidirectionalIterator __m = __first; - if (__pred(*++__m)) - { - _Ops::iter_swap(__first, __m); - _Ops::iter_swap(__m, __last); - return __last; - } - _Ops::iter_swap(__m, __last); - _Ops::iter_swap(__first, __m); - return __m; + // *__first is known to be false + // *__last is known to be true + // __len >= 2 + if (__len == 2) { + _Ops::iter_swap(__first, __last); + return __last; + } + if (__len == 3) { + _BidirectionalIterator __m = __first; + if (__pred(*++__m)) { + _Ops::iter_swap(__first, __m); + _Ops::iter_swap(__m, __last); + return __last; } - if (__len <= __p.second) - { // The buffer is big enough to use - typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - __destruct_n __d(0); - unique_ptr __h(__p.first, __d); - // Move the falses into the temporary buffer, and the trues to the front of the line - // Update __first to always point to the end of the trues - value_type* __t = __p.first; - ::new ((void*)__t) value_type(_Ops::__iter_move(__first)); + _Ops::iter_swap(__m, __last); + _Ops::iter_swap(__first, __m); + return __m; + } + if (__len <= __p.second) { // The buffer is big enough to use + typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; + __destruct_n __d(0); + unique_ptr __h(__p.first, __d); + // Move the falses into the temporary buffer, and the trues to the front of the line + // Update __first to always point to the end of the trues + value_type* __t = __p.first; + ::new ((void*)__t) value_type(_Ops::__iter_move(__first)); + __d.template __incr(); + ++__t; + _BidirectionalIterator __i = __first; + while (++__i != __last) { + if (__pred(*__i)) { + *__first = _Ops::__iter_move(__i); + ++__first; + } else { + ::new ((void*)__t) value_type(_Ops::__iter_move(__i)); __d.template __incr(); ++__t; - _BidirectionalIterator __i = __first; - while (++__i != __last) - { - if (__pred(*__i)) - { - *__first = _Ops::__iter_move(__i); - ++__first; - } - else - { - ::new ((void*)__t) value_type(_Ops::__iter_move(__i)); - __d.template __incr(); - ++__t; - } - } - // move *__last, known to be true - *__first = _Ops::__iter_move(__i); - __i = ++__first; - // All trues now at start of range, all falses in buffer - // Move falses back into range, but don't mess up __first which points to first false - for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void) ++__i) - *__i = _Ops::__iter_move(__t2); - // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer - return __first; - } - // Else not enough buffer, do in place - // __len >= 4 - _BidirectionalIterator __m = __first; - _Distance __len2 = __len / 2; // __len2 >= 2 - _Ops::advance(__m, __len2); - // recurse on [__first, __m-1], except reduce __m-1 until *(__m-1) is true, *__first know to be false - // F????????????????T - // f m l - _BidirectionalIterator __m1 = __m; - _BidirectionalIterator __first_false = __first; - _Distance __len_half = __len2; - while (!__pred(*--__m1)) - { - if (__m1 == __first) - goto __first_half_done; - --__len_half; + } } - // F???TFFF?????????T - // f m1 m l - __first_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - __first, __m1, __pred, __len_half, __p, __bit); + // move *__last, known to be true + *__first = _Ops::__iter_move(__i); + __i = ++__first; + // All trues now at start of range, all falses in buffer + // Move falses back into range, but don't mess up __first which points to first false + for (value_type* __t2 = __p.first; __t2 < __t; ++__t2, (void)++__i) + *__i = _Ops::__iter_move(__t2); + // __h destructs moved-from values out of the temp buffer, but doesn't deallocate buffer + return __first; + } + // Else not enough buffer, do in place + // __len >= 4 + _BidirectionalIterator __m = __first; + _Distance __len2 = __len / 2; // __len2 >= 2 + _Ops::advance(__m, __len2); + // recurse on [__first, __m-1], except reduce __m-1 until *(__m-1) is true, *__first know to be false + // F????????????????T + // f m l + _BidirectionalIterator __m1 = __m; + _BidirectionalIterator __first_false = __first; + _Distance __len_half = __len2; + while (!__pred(*--__m1)) { + if (__m1 == __first) + goto __first_half_done; + --__len_half; + } + // F???TFFF?????????T + // f m1 m l + __first_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__first, __m1, __pred, __len_half, __p, __bit); __first_half_done: - // TTTFFFFF?????????T - // f ff m l - // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true - __m1 = __m; - _BidirectionalIterator __second_false = __last; - ++__second_false; - __len_half = __len - __len2; - while (__pred(*__m1)) - { - if (++__m1 == __last) - goto __second_half_done; - --__len_half; - } - // TTTFFFFFTTTF?????T - // f ff m m1 l - __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - __m1, __last, __pred, __len_half, __p, __bit); + // TTTFFFFF?????????T + // f ff m l + // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true + __m1 = __m; + _BidirectionalIterator __second_false = __last; + ++__second_false; + __len_half = __len - __len2; + while (__pred(*__m1)) { + if (++__m1 == __last) + goto __second_half_done; + --__len_half; + } + // TTTFFFFFTTTF?????T + // f ff m m1 l + __second_false = std::__stable_partition_impl<_AlgPolicy, _Predicate&>(__m1, __last, __pred, __len_half, __p, __bit); __second_half_done: - // TTTFFFFFTTTTTFFFFF - // f ff m sf l - return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first; - // TTTTTTTTFFFFFFFFFF - // | + // TTTFFFFFTTTTTFFFFF + // f ff m sf l + return std::__rotate<_AlgPolicy>(__first_false, __m, __second_false).first; + // TTTTTTTTFFFFFFFFFF + // | } template -_LIBCPP_HIDE_FROM_ABI _BidirectionalIterator -__stable_partition_impl(_BidirectionalIterator __first, _BidirectionalIterator __last, _Predicate __pred, - bidirectional_iterator_tag) -{ - typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; - typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; - const difference_type __alloc_limit = 4; // might want to make this a function of trivial assignment - // Either prove all true and return __first or point to first false - while (true) - { - if (__first == __last) - return __first; - if (!__pred(*__first)) - break; - ++__first; - } - // __first points to first false, everything prior to __first is already set. - // Either prove [__first, __last) is all false and return __first, or point __last to last true - do - { - if (__first == --__last) - return __first; - } while (!__pred(*__last)); - // We now have a reduced range [__first, __last] - // *__first is known to be false - // *__last is known to be true - // __len >= 2 - difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last) + 1; - pair __p(0, 0); - unique_ptr __h; - if (__len >= __alloc_limit) - { -// TODO: Remove the use of std::get_temporary_buffer -_LIBCPP_SUPPRESS_DEPRECATED_PUSH - __p = std::get_temporary_buffer(__len); -_LIBCPP_SUPPRESS_DEPRECATED_POP - __h.reset(__p.first); - } - return std::__stable_partition_impl<_AlgPolicy, _Predicate&>( - std::move(__first), std::move(__last), __pred, __len, __p, bidirectional_iterator_tag()); +_LIBCPP_HIDE_FROM_ABI _BidirectionalIterator __stable_partition_impl( + _BidirectionalIterator __first, _BidirectionalIterator __last, _Predicate __pred, bidirectional_iterator_tag) { + typedef typename iterator_traits<_BidirectionalIterator>::difference_type difference_type; + typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; + const difference_type __alloc_limit = 4; // might want to make this a function of trivial assignment + // Either prove all true and return __first or point to first false + while (true) { + if (__first == __last) + return __first; + if (!__pred(*__first)) + break; + ++__first; + } + // __first points to first false, everything prior to __first is already set. + // Either prove [__first, __last) is all false and return __first, or point __last to last true + do { + if (__first == --__last) + return __first; + } while (!__pred(*__last)); + // We now have a reduced range [__first, __last] + // *__first is known to be false + // *__last is known to be true + // __len >= 2 + difference_type __len = _IterOps<_AlgPolicy>::distance(__first, __last) + 1; + pair __p(0, 0); + unique_ptr __h; + if (__len >= __alloc_limit) { + // TODO: Remove the use of std::get_temporary_buffer + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + __p = std::get_temporary_buffer(__len); + _LIBCPP_SUPPRESS_DEPRECATED_POP + __h.reset(__p.first); + } + return std::__stable_partition_impl<_AlgPolicy, _Predicate&>( + std::move(__first), std::move(__last), __pred, __len, __p, bidirectional_iterator_tag()); } template -_LIBCPP_HIDE_FROM_ABI -_ForwardIterator __stable_partition( +_LIBCPP_HIDE_FROM_ABI _ForwardIterator __stable_partition( _ForwardIterator __first, _ForwardIterator __last, _Predicate&& __pred, _IterCategory __iter_category) { return std::__stable_partition_impl<_AlgPolicy, __remove_cvref_t<_Predicate>&>( std::move(__first), std::move(__last), __pred, __iter_category); } template -inline _LIBCPP_HIDE_FROM_ABI -_ForwardIterator -stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) -{ +inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator +stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { using _IterCategory = typename iterator_traits<_ForwardIterator>::iterator_category; return std::__stable_partition<_ClassicAlgPolicy, _Predicate&>( std::move(__first), std::move(__last), __pred, _IterCategory()); diff --git a/libcxx/include/__algorithm/stable_sort.h b/libcxx/include/__algorithm/stable_sort.h index 175393fc47779..ffc6e4ce28188 100644 --- a/libcxx/include/__algorithm/stable_sort.h +++ b/libcxx/include/__algorithm/stable_sort.h @@ -32,9 +32,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI -void __insertion_sort_move(_BidirectionalIterator __first1, _BidirectionalIterator __last1, - typename iterator_traits<_BidirectionalIterator>::value_type* __first2, _Compare __comp) { +_LIBCPP_HIDE_FROM_ABI void __insertion_sort_move( + _BidirectionalIterator __first1, + _BidirectionalIterator __last1, + typename iterator_traits<_BidirectionalIterator>::value_type* __first2, + _Compare __comp) { using _Ops = _IterOps<_AlgPolicy>; typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; @@ -63,200 +65,187 @@ void __insertion_sort_move(_BidirectionalIterator __first1, _BidirectionalIterat } template -_LIBCPP_HIDE_FROM_ABI void -__merge_move_construct(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, - typename iterator_traits<_InputIterator1>::value_type* __result, _Compare __comp) -{ - using _Ops = _IterOps<_AlgPolicy>; +_LIBCPP_HIDE_FROM_ABI void __merge_move_construct( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + typename iterator_traits<_InputIterator1>::value_type* __result, + _Compare __comp) { + using _Ops = _IterOps<_AlgPolicy>; - typedef typename iterator_traits<_InputIterator1>::value_type value_type; - __destruct_n __d(0); - unique_ptr __h(__result, __d); - for (; true; ++__result) - { - if (__first1 == __last1) - { - for (; __first2 != __last2; ++__first2, (void) ++__result, __d.template __incr()) - ::new ((void*)__result) value_type(_Ops::__iter_move(__first2)); - __h.release(); - return; - } - if (__first2 == __last2) - { - for (; __first1 != __last1; ++__first1, (void) ++__result, __d.template __incr()) - ::new ((void*)__result) value_type(_Ops::__iter_move(__first1)); - __h.release(); - return; - } - if (__comp(*__first2, *__first1)) - { - ::new ((void*)__result) value_type(_Ops::__iter_move(__first2)); - __d.template __incr(); - ++__first2; - } - else - { - ::new ((void*)__result) value_type(_Ops::__iter_move(__first1)); - __d.template __incr(); - ++__first1; - } + typedef typename iterator_traits<_InputIterator1>::value_type value_type; + __destruct_n __d(0); + unique_ptr __h(__result, __d); + for (; true; ++__result) { + if (__first1 == __last1) { + for (; __first2 != __last2; ++__first2, (void)++__result, __d.template __incr()) + ::new ((void*)__result) value_type(_Ops::__iter_move(__first2)); + __h.release(); + return; + } + if (__first2 == __last2) { + for (; __first1 != __last1; ++__first1, (void)++__result, __d.template __incr()) + ::new ((void*)__result) value_type(_Ops::__iter_move(__first1)); + __h.release(); + return; } + if (__comp(*__first2, *__first1)) { + ::new ((void*)__result) value_type(_Ops::__iter_move(__first2)); + __d.template __incr(); + ++__first2; + } else { + ::new ((void*)__result) value_type(_Ops::__iter_move(__first1)); + __d.template __incr(); + ++__first1; + } + } } template -_LIBCPP_HIDE_FROM_ABI void -__merge_move_assign(_InputIterator1 __first1, _InputIterator1 __last1, - _InputIterator2 __first2, _InputIterator2 __last2, - _OutputIterator __result, _Compare __comp) -{ - using _Ops = _IterOps<_AlgPolicy>; +_LIBCPP_HIDE_FROM_ABI void __merge_move_assign( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _OutputIterator __result, + _Compare __comp) { + using _Ops = _IterOps<_AlgPolicy>; - for (; __first1 != __last1; ++__result) - { - if (__first2 == __last2) - { - for (; __first1 != __last1; ++__first1, (void) ++__result) - *__result = _Ops::__iter_move(__first1); - return; - } - if (__comp(*__first2, *__first1)) - { - *__result = _Ops::__iter_move(__first2); - ++__first2; - } - else - { - *__result = _Ops::__iter_move(__first1); - ++__first1; - } + for (; __first1 != __last1; ++__result) { + if (__first2 == __last2) { + for (; __first1 != __last1; ++__first1, (void)++__result) + *__result = _Ops::__iter_move(__first1); + return; } - for (; __first2 != __last2; ++__first2, (void) ++__result) - *__result = _Ops::__iter_move(__first2); + if (__comp(*__first2, *__first1)) { + *__result = _Ops::__iter_move(__first2); + ++__first2; + } else { + *__result = _Ops::__iter_move(__first1); + ++__first1; + } + } + for (; __first2 != __last2; ++__first2, (void)++__result) + *__result = _Ops::__iter_move(__first2); } template -void -__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size); +void __stable_sort(_RandomAccessIterator __first, + _RandomAccessIterator __last, + _Compare __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len, + typename iterator_traits<_RandomAccessIterator>::value_type* __buff, + ptrdiff_t __buff_size); template -void -__stable_sort_move(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Compare __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __first2) -{ - using _Ops = _IterOps<_AlgPolicy>; +void __stable_sort_move(_RandomAccessIterator __first1, + _RandomAccessIterator __last1, + _Compare __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len, + typename iterator_traits<_RandomAccessIterator>::value_type* __first2) { + using _Ops = _IterOps<_AlgPolicy>; - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - switch (__len) - { - case 0: - return; - case 1: - ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); - return; - case 2: - __destruct_n __d(0); - unique_ptr __h2(__first2, __d); - if (__comp(*--__last1, *__first1)) - { - ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1)); - __d.template __incr(); - ++__first2; - ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); - } - else - { - ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); - __d.template __incr(); - ++__first2; - ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1)); - } - __h2.release(); - return; - } - if (__len <= 8) - { - std::__insertion_sort_move<_AlgPolicy, _Compare>(__first1, __last1, __first2, __comp); - return; + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + switch (__len) { + case 0: + return; + case 1: + ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); + return; + case 2: + __destruct_n __d(0); + unique_ptr __h2(__first2, __d); + if (__comp(*--__last1, *__first1)) { + ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1)); + __d.template __incr(); + ++__first2; + ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); + } else { + ::new ((void*)__first2) value_type(_Ops::__iter_move(__first1)); + __d.template __incr(); + ++__first2; + ::new ((void*)__first2) value_type(_Ops::__iter_move(__last1)); } - typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; - _RandomAccessIterator __m = __first1 + __l2; - std::__stable_sort<_AlgPolicy, _Compare>(__first1, __m, __comp, __l2, __first2, __l2); - std::__stable_sort<_AlgPolicy, _Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2); - std::__merge_move_construct<_AlgPolicy, _Compare>(__first1, __m, __m, __last1, __first2, __comp); + __h2.release(); + return; + } + if (__len <= 8) { + std::__insertion_sort_move<_AlgPolicy, _Compare>(__first1, __last1, __first2, __comp); + return; + } + typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; + _RandomAccessIterator __m = __first1 + __l2; + std::__stable_sort<_AlgPolicy, _Compare>(__first1, __m, __comp, __l2, __first2, __l2); + std::__stable_sort<_AlgPolicy, _Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2); + std::__merge_move_construct<_AlgPolicy, _Compare>(__first1, __m, __m, __last1, __first2, __comp); } template -struct __stable_sort_switch -{ - static const unsigned value = 128*is_trivially_copy_assignable<_Tp>::value; +struct __stable_sort_switch { + static const unsigned value = 128 * is_trivially_copy_assignable<_Tp>::value; }; template -void -__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size) -{ - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - switch (__len) - { - case 0: - case 1: - return; - case 2: - if (__comp(*--__last, *__first)) - _IterOps<_AlgPolicy>::iter_swap(__first, __last); - return; - } - if (__len <= static_cast(__stable_sort_switch::value)) - { - std::__insertion_sort<_AlgPolicy, _Compare>(__first, __last, __comp); - return; - } - typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; - _RandomAccessIterator __m = __first + __l2; - if (__len <= __buff_size) - { - __destruct_n __d(0); - unique_ptr __h2(__buff, __d); - std::__stable_sort_move<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff); - __d.__set(__l2, (value_type*)nullptr); - std::__stable_sort_move<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff + __l2); - __d.__set(__len, (value_type*)nullptr); - std::__merge_move_assign<_AlgPolicy, _Compare>( - __buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp); -// std::__merge<_Compare>(move_iterator(__buff), -// move_iterator(__buff + __l2), -// move_iterator<_RandomAccessIterator>(__buff + __l2), -// move_iterator<_RandomAccessIterator>(__buff + __len), -// __first, __comp); - return; - } - std::__stable_sort<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff, __buff_size); - std::__stable_sort<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size); - std::__inplace_merge<_AlgPolicy>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size); +void __stable_sort(_RandomAccessIterator __first, + _RandomAccessIterator __last, + _Compare __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len, + typename iterator_traits<_RandomAccessIterator>::value_type* __buff, + ptrdiff_t __buff_size) { + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + switch (__len) { + case 0: + case 1: + return; + case 2: + if (__comp(*--__last, *__first)) + _IterOps<_AlgPolicy>::iter_swap(__first, __last); + return; + } + if (__len <= static_cast(__stable_sort_switch::value)) { + std::__insertion_sort<_AlgPolicy, _Compare>(__first, __last, __comp); + return; + } + typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; + _RandomAccessIterator __m = __first + __l2; + if (__len <= __buff_size) { + __destruct_n __d(0); + unique_ptr __h2(__buff, __d); + std::__stable_sort_move<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff); + __d.__set(__l2, (value_type*)nullptr); + std::__stable_sort_move<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff + __l2); + __d.__set(__len, (value_type*)nullptr); + std::__merge_move_assign<_AlgPolicy, _Compare>( + __buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp); + // std::__merge<_Compare>(move_iterator(__buff), + // move_iterator(__buff + __l2), + // move_iterator<_RandomAccessIterator>(__buff + __l2), + // move_iterator<_RandomAccessIterator>(__buff + __len), + // __first, __comp); + return; + } + std::__stable_sort<_AlgPolicy, _Compare>(__first, __m, __comp, __l2, __buff, __buff_size); + std::__stable_sort<_AlgPolicy, _Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size); + std::__inplace_merge<_AlgPolicy>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size); } template -inline _LIBCPP_HIDE_FROM_ABI -void __stable_sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) { - using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; +inline _LIBCPP_HIDE_FROM_ABI void +__stable_sort_impl(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp) { + using value_type = typename iterator_traits<_RandomAccessIterator>::value_type; using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; difference_type __len = __last - __first; pair __buf(0, 0); unique_ptr __h; if (__len > static_cast(__stable_sort_switch::value)) { -// TODO: Remove the use of std::get_temporary_buffer -_LIBCPP_SUPPRESS_DEPRECATED_PUSH - __buf = std::get_temporary_buffer(__len); -_LIBCPP_SUPPRESS_DEPRECATED_POP - __h.reset(__buf.first); + // TODO: Remove the use of std::get_temporary_buffer + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + __buf = std::get_temporary_buffer(__len); + _LIBCPP_SUPPRESS_DEPRECATED_POP + __h.reset(__buf.first); } std::__stable_sort<_AlgPolicy, __comp_ref_type<_Compare> >(__first, __last, __comp, __len, __buf.first, __buf.second); @@ -264,14 +253,13 @@ _LIBCPP_SUPPRESS_DEPRECATED_POP } template -inline _LIBCPP_HIDE_FROM_ABI -void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { +inline _LIBCPP_HIDE_FROM_ABI void +stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { std::__stable_sort_impl<_ClassicAlgPolicy>(std::move(__first), std::move(__last), __comp); } template -inline _LIBCPP_HIDE_FROM_ABI -void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last) { +inline _LIBCPP_HIDE_FROM_ABI void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last) { std::stable_sort(__first, __last, __less<>()); } diff --git a/libcxx/include/__algorithm/swap_ranges.h b/libcxx/include/__algorithm/swap_ranges.h index 96eba7484bf42..7fab5c49a656f 100644 --- a/libcxx/include/__algorithm/swap_ranges.h +++ b/libcxx/include/__algorithm/swap_ranges.h @@ -22,8 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD // 2+2 iterators: the shorter size will be used. template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -pair<_ForwardIterator1, _ForwardIterator2> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator1, _ForwardIterator2> __swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _Sentinel2 __last2) { while (__first1 != __last1 && __first2 != __last2) { _IterOps<_AlgPolicy>::iter_swap(__first1, __first2); @@ -36,8 +35,7 @@ __swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 // 2+1 iterators: size2 >= size1. template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -pair<_ForwardIterator1, _ForwardIterator2> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator1, _ForwardIterator2> __swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2) { while (__first1 != __last1) { _IterOps<_AlgPolicy>::iter_swap(__first1, __first2); @@ -51,8 +49,7 @@ __swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator2 swap_ranges(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) { - return std::__swap_ranges<_ClassicAlgPolicy>( - std::move(__first1), std::move(__last1), std::move(__first2)).second; + return std::__swap_ranges<_ClassicAlgPolicy>(std::move(__first1), std::move(__last1), std::move(__first2)).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/transform.h b/libcxx/include/__algorithm/transform.h index 76b9bb780cf94..1b424409591ce 100644 --- a/libcxx/include/__algorithm/transform.h +++ b/libcxx/include/__algorithm/transform.h @@ -18,24 +18,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -transform(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _UnaryOperation __op) -{ - for (; __first != __last; ++__first, (void) ++__result) - *__result = __op(*__first); - return __result; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator +transform(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _UnaryOperation __op) { + for (; __first != __last; ++__first, (void)++__result) + *__result = __op(*__first); + return __result; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_OutputIterator -transform(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, - _OutputIterator __result, _BinaryOperation __binary_op) -{ - for (; __first1 != __last1; ++__first1, (void) ++__first2, ++__result) - *__result = __binary_op(*__first1, *__first2); - return __result; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator transform( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _OutputIterator __result, + _BinaryOperation __binary_op) { + for (; __first1 != __last1; ++__first1, (void)++__first2, ++__result) + *__result = __binary_op(*__first1, *__first2); + return __result; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/uniform_random_bit_generator_adaptor.h b/libcxx/include/__algorithm/uniform_random_bit_generator_adaptor.h index 1f2039949d740..aef0fbfb7c284 100644 --- a/libcxx/include/__algorithm/uniform_random_bit_generator_adaptor.h +++ b/libcxx/include/__algorithm/uniform_random_bit_generator_adaptor.h @@ -20,7 +20,7 @@ #if _LIBCPP_STD_VER >= 20 _LIBCPP_PUSH_MACROS -#include <__undef_macros> +# include <__undef_macros> _LIBCPP_BEGIN_NAMESPACE_STD @@ -41,16 +41,12 @@ class _ClassicGenAdaptor { public: using result_type = invoke_result_t<_Gen&>; - _LIBCPP_HIDE_FROM_ABI - static constexpr auto min() { return __remove_cvref_t<_Gen>::min(); } - _LIBCPP_HIDE_FROM_ABI - static constexpr auto max() { return __remove_cvref_t<_Gen>::max(); } + _LIBCPP_HIDE_FROM_ABI static constexpr auto min() { return __remove_cvref_t<_Gen>::min(); } + _LIBCPP_HIDE_FROM_ABI static constexpr auto max() { return __remove_cvref_t<_Gen>::max(); } - _LIBCPP_HIDE_FROM_ABI - constexpr explicit _ClassicGenAdaptor(_Gen& __g) : __gen_(__g) {} + _LIBCPP_HIDE_FROM_ABI constexpr explicit _ClassicGenAdaptor(_Gen& __g) : __gen_(__g) {} - _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()() const { return __gen_(); } + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()() const { return __gen_(); } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/unwrap_iter.h b/libcxx/include/__algorithm/unwrap_iter.h index 91346e3c1bd2a..a298a2b271056 100644 --- a/libcxx/include/__algorithm/unwrap_iter.h +++ b/libcxx/include/__algorithm/unwrap_iter.h @@ -57,11 +57,11 @@ struct __unwrap_iter_impl<_Iter, true> { } }; -template, - __enable_if_t::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -decltype(_Impl::__unwrap(std::declval<_Iter>())) __unwrap_iter(_Iter __i) _NOEXCEPT { +template , + __enable_if_t::value, int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 decltype(_Impl::__unwrap(std::declval<_Iter>())) +__unwrap_iter(_Iter __i) _NOEXCEPT { return _Impl::__unwrap(__i); } diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h index 449802a2e3040..3dfb6937d0325 100644 --- a/libcxx/include/__atomic/atomic.h +++ b/libcxx/include/__atomic/atomic.h @@ -33,241 +33,220 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -struct atomic - : public __atomic_base<_Tp> -{ +struct atomic : public __atomic_base<_Tp> { using __base = __atomic_base<_Tp>; using value_type = _Tp; using difference_type = value_type; #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI - atomic() = default; + _LIBCPP_HIDE_FROM_ABI atomic() = default; #else - _LIBCPP_HIDE_FROM_ABI - atomic() _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; #endif - _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} - _LIBCPP_HIDE_FROM_ABI - _Tp operator=(_Tp __d) volatile _NOEXCEPT - {__base::store(__d); return __d;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator=(_Tp __d) _NOEXCEPT - {__base::store(__d); return __d;} + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT { + __base::store(__d); + return __d; + } - atomic& operator=(const atomic&) = delete; - atomic& operator=(const atomic&) volatile = delete; + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; }; // atomic template -struct atomic<_Tp*> - : public __atomic_base<_Tp*> -{ - using __base = __atomic_base<_Tp*>; - using value_type = _Tp*; - using difference_type = ptrdiff_t; - - _LIBCPP_HIDE_FROM_ABI - atomic() _NOEXCEPT = default; - - _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} - - _LIBCPP_HIDE_FROM_ABI - _Tp* operator=(_Tp* __d) volatile _NOEXCEPT - {__base::store(__d); return __d;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator=(_Tp* __d) _NOEXCEPT - {__base::store(__d); return __d;} - - _LIBCPP_HIDE_FROM_ABI - _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - // __atomic_fetch_add accepts function pointers, guard against them. - static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); - return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI - _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - // __atomic_fetch_add accepts function pointers, guard against them. - static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); - return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI - _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - // __atomic_fetch_add accepts function pointers, guard against them. - static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); - return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI - _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - // __atomic_fetch_add accepts function pointers, guard against them. - static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); - return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI - _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} - - atomic& operator=(const atomic&) = delete; - atomic& operator=(const atomic&) volatile = delete; +struct atomic<_Tp*> : public __atomic_base<_Tp*> { + using __base = __atomic_base<_Tp*>; + using value_type = _Tp*; + using difference_type = ptrdiff_t; + + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; }; #if _LIBCPP_STD_VER >= 20 template requires is_floating_point_v<_Tp> struct atomic<_Tp> : __atomic_base<_Tp> { - private: - _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() { - // Only x87-fp80 long double has 64-bit mantissa - return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>; - } +private: + _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() { + // Only x87-fp80 long double has 64-bit mantissa + return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>; + } - _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() { + _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() { # ifndef _LIBCPP_COMPILER_CLANG_BASED - return false; + return false; # else - // The builtin __cxx_atomic_fetch_add errors during compilation for - // long double on platforms with fp80 format. - // For more details, see - // lib/Sema/SemaChecking.cpp function IsAllowedValueType - // LLVM Parser does not allow atomicrmw with x86_fp80 type. - // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && - // &Context.getTargetInfo().getLongDoubleFormat() == - // &llvm::APFloat::x87DoubleExtended()) - // For more info - // https://github.com/llvm/llvm-project/issues/68602 - // https://reviews.llvm.org/D53965 - return !__is_fp80_long_double(); + // The builtin __cxx_atomic_fetch_add errors during compilation for + // long double on platforms with fp80 format. + // For more details, see + // lib/Sema/SemaChecking.cpp function IsAllowedValueType + // LLVM Parser does not allow atomicrmw with x86_fp80 type. + // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && + // &Context.getTargetInfo().getLongDoubleFormat() == + // &llvm::APFloat::x87DoubleExtended()) + // For more info + // https://github.com/llvm/llvm-project/issues/68602 + // https://reviews.llvm.org/D53965 + return !__is_fp80_long_double(); # endif - } - - template - _LIBCPP_HIDE_FROM_ABI static _Tp - __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) { - if constexpr (__has_rmw_builtin()) { - return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m); - } else { - _Tp __old = __self.load(memory_order_relaxed); - _Tp __new = __operation(__old, __operand); - while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) { + } + + template + _LIBCPP_HIDE_FROM_ABI static _Tp + __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) { + if constexpr (__has_rmw_builtin()) { + return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m); + } else { + _Tp __old = __self.load(memory_order_relaxed); + _Tp __new = __operation(__old, __operand); + while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) { # ifdef _LIBCPP_COMPILER_CLANG_BASED - if constexpr (__is_fp80_long_double()) { - // https://github.com/llvm/llvm-project/issues/47978 - // clang bug: __old is not updated on failure for atomic::compare_exchange_weak - // Note __old = __self.load(memory_order_relaxed) will not work - std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed); - } -# endif - __new = __operation(__old, __operand); - } - return __old; + if constexpr (__is_fp80_long_double()) { + // https://github.com/llvm/llvm-project/issues/47978 + // clang bug: __old is not updated on failure for atomic::compare_exchange_weak + // Note __old = __self.load(memory_order_relaxed) will not work + std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed); } +# endif + __new = __operation(__old, __operand); + } + return __old; } + } + + template + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order); + }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op); + } + + template + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order); + }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op); + } + +public: + using __base = __atomic_base<_Tp>; + using value_type = _Tp; + using difference_type = value_type; - template - _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) { - auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { - return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order); - }; - return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op); - } - - template - _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) { - auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { - return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order); - }; - return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op); - } - - public: - using __base = __atomic_base<_Tp>; - using value_type = _Tp; - using difference_type = value_type; - - _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default; - _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {} - - atomic(const atomic&) = delete; - atomic& operator=(const atomic&) = delete; - atomic& operator=(const atomic&) volatile = delete; - - _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept - requires __base::is_always_lock_free - { - __base::store(__d); - return __d; - } - _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept { - __base::store(__d); - return __d; - } - - _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept - requires __base::is_always_lock_free - { - return __fetch_add(*this, __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { - return __fetch_add(*this, __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept - requires __base::is_always_lock_free - { - return __fetch_sub(*this, __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { - return __fetch_sub(*this, __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept - requires __base::is_always_lock_free - { - return fetch_add(__op) + __op; - } - - _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; } - - _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept - requires __base::is_always_lock_free - { - return fetch_sub(__op) - __op; - } - - _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default; + _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {} + + atomic(const atomic&) = delete; + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept + requires __base::is_always_lock_free + { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_add(__op) + __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_sub(__op) - __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; } }; #endif // _LIBCPP_STD_VER >= 20 @@ -275,476 +254,363 @@ struct atomic<_Tp> : __atomic_base<_Tp> { // atomic_is_lock_free template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT -{ - return __o->is_lock_free(); +_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); } template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT -{ - return __o->is_lock_free(); +_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); } // atomic_init template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI -void -atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI -void -atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); } // atomic_store template -_LIBCPP_HIDE_FROM_ABI -void -atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - __o->store(__d); +_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); } template -_LIBCPP_HIDE_FROM_ABI -void -atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - __o->store(__d); +_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); } // atomic_store_explicit template -_LIBCPP_HIDE_FROM_ABI -void +_LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) -{ - __o->store(__d, __m); + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); } template -_LIBCPP_HIDE_FROM_ABI -void +_LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) -{ - __o->store(__d, __m); + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); } // atomic_load template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT -{ - return __o->load(); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); } template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_load(const atomic<_Tp>* __o) _NOEXCEPT -{ - return __o->load(); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); } // atomic_load_explicit template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) -{ - return __o->load(__m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); } template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) -{ - return __o->load(__m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); } // atomic_exchange template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->exchange(__d); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); } template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->exchange(__d); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); } // atomic_exchange_explicit template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT -{ - return __o->exchange(__d, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); } template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT -{ - return __o->exchange(__d, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); } // atomic_compare_exchange_weak template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->compare_exchange_weak(*__e, __d); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); } template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->compare_exchange_weak(*__e, __d); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); } // atomic_compare_exchange_strong template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->compare_exchange_strong(*__e, __d); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); } template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT -{ - return __o->compare_exchange_strong(*__e, __d); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); } // atomic_compare_exchange_weak_explicit template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, - typename atomic<_Tp>::value_type __d, - memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) -{ - return __o->compare_exchange_weak(*__e, __d, __s, __f); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); } template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, - memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) -{ - return __o->compare_exchange_weak(*__e, __d, __s, __f); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); } // atomic_compare_exchange_strong_explicit template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, - typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, - memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) -{ - return __o->compare_exchange_strong(*__e, __d, __s, __f); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); } template -_LIBCPP_HIDE_FROM_ABI -bool -atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, - typename atomic<_Tp>::value_type __d, - memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) -{ - return __o->compare_exchange_strong(*__e, __d, __s, __f); +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); } // atomic_wait template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_wait(const volatile atomic<_Tp>* __o, - typename atomic<_Tp>::value_type __v) _NOEXCEPT -{ - return __o->wait(__v); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); } template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_wait(const atomic<_Tp>* __o, - typename atomic<_Tp>::value_type __v) _NOEXCEPT -{ - return __o->wait(__v); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); } // atomic_wait_explicit template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_wait_explicit(const volatile atomic<_Tp>* __o, - typename atomic<_Tp>::value_type __v, - memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) -{ - return __o->wait(__v, __m); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); } template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_wait_explicit(const atomic<_Tp>* __o, - typename atomic<_Tp>::value_type __v, - memory_order __m) _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) -{ - return __o->wait(__v, __m); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); } // atomic_notify_one template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT -{ - __o->notify_one(); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); } template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT -{ - __o->notify_one(); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); } // atomic_notify_all template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT -{ - __o->notify_all(); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); } template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI -void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT -{ - __o->notify_all(); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); } // atomic_fetch_add template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT -{ - return __o->fetch_add(__op); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); } template -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT -{ - return __o->fetch_add(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); } // atomic_fetch_add_explicit template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_add(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); } template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_add(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); } // atomic_fetch_sub template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT -{ - return __o->fetch_sub(__op); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); } template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT -{ - return __o->fetch_sub(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); } // atomic_fetch_sub_explicit template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_sub(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); } template -_LIBCPP_HIDE_FROM_ABI -_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_sub(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); } // atomic_fetch_and template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_and(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_and(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); } // atomic_fetch_and_explicit template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_and(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_and(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); } // atomic_fetch_or template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_or(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_or(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); } // atomic_fetch_or_explicit template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_or(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_or(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); } // atomic_fetch_xor template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_xor(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT -{ - return __o->fetch_xor(__op); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); } // atomic_fetch_xor_explicit template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_xor(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); } template ::value && !is_same<_Tp, bool>::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -_Tp -atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT -{ - return __o->fetch_xor(__op, __m); +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/atomic_base.h b/libcxx/include/__atomic/atomic_base.h index 775d06d757018..3ad3b562c5980 100644 --- a/libcxx/include/__atomic/atomic_base.h +++ b/libcxx/include/__atomic/atomic_base.h @@ -29,114 +29,109 @@ _LIBCPP_BEGIN_NAMESPACE_STD template ::value && !is_same<_Tp, bool>::value> -struct __atomic_base // false +struct __atomic_base // false { - mutable __cxx_atomic_impl<_Tp> __a_; + mutable __cxx_atomic_impl<_Tp> __a_; #if _LIBCPP_STD_VER >= 17 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; #endif - _LIBCPP_HIDE_FROM_ABI - bool is_lock_free() const volatile _NOEXCEPT - {return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));} - _LIBCPP_HIDE_FROM_ABI - bool is_lock_free() const _NOEXCEPT - {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} - _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { - std::__cxx_atomic_store(std::addressof(__a_), __d, __m); - } - _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { - std::__cxx_atomic_store(std::addressof(__a_), __d, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { - return std::__cxx_atomic_load(std::addressof(__a_), __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT - _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { - return std::__cxx_atomic_load(std::addressof(__a_), __m); - } - _LIBCPP_HIDE_FROM_ABI - operator _Tp() const volatile _NOEXCEPT {return load();} - _LIBCPP_HIDE_FROM_ABI - operator _Tp() const _NOEXCEPT {return load();} - _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { - return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); - } - _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { - return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { - return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); - } - _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT - _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { - return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); - } - _LIBCPP_HIDE_FROM_ABI bool - compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); - } - - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const - volatile _NOEXCEPT { - std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void - wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { - std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { - std::__cxx_atomic_notify_one(std::addressof(__a_)); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { - std::__cxx_atomic_notify_one(std::addressof(__a_)); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { - std::__cxx_atomic_notify_all(std::addressof(__a_)); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { - std::__cxx_atomic_notify_all(std::addressof(__a_)); - } + _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT { + return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>)); + } + _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { + return static_cast<__atomic_base const volatile*>(this)->is_lock_free(); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const + volatile _NOEXCEPT { + std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void + wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { + std::__cxx_atomic_notify_one(std::addressof(__a_)); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { + std::__cxx_atomic_notify_one(std::addressof(__a_)); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { + std::__cxx_atomic_notify_all(std::addressof(__a_)); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { + std::__cxx_atomic_notify_all(std::addressof(__a_)); + } #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI constexpr - __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} + _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} #else - _LIBCPP_HIDE_FROM_ABI - __atomic_base() _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default; #endif - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} - __atomic_base(const __atomic_base&) = delete; + __atomic_base(const __atomic_base&) = delete; }; #if _LIBCPP_STD_VER >= 17 @@ -147,84 +142,62 @@ _LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; // atomic template -struct __atomic_base<_Tp, true> - : public __atomic_base<_Tp, false> -{ - using __base = __atomic_base<_Tp, false>; - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - __atomic_base() _NOEXCEPT = default; - - _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} - - _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { - return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); - } - _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { - return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); - } - - _LIBCPP_HIDE_FROM_ABI - _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} - _LIBCPP_HIDE_FROM_ABI - _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} - _LIBCPP_HIDE_FROM_ABI - _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} - _LIBCPP_HIDE_FROM_ABI - _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} - _LIBCPP_HIDE_FROM_ABI - _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} - _LIBCPP_HIDE_FROM_ABI - _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} - _LIBCPP_HIDE_FROM_ABI - _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} +struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { + using __base = __atomic_base<_Tp, false>; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h index edfa978f99357..d76e5e45c01a0 100644 --- a/libcxx/include/__atomic/atomic_flag.h +++ b/libcxx/include/__atomic/atomic_flag.h @@ -24,205 +24,135 @@ _LIBCPP_BEGIN_NAMESPACE_STD -struct atomic_flag -{ - __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; - - _LIBCPP_HIDE_FROM_ABI - bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT - {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} - _LIBCPP_HIDE_FROM_ABI - bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT - {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} - - _LIBCPP_HIDE_FROM_ABI - bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} - _LIBCPP_HIDE_FROM_ABI - bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} - _LIBCPP_HIDE_FROM_ABI - void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} - _LIBCPP_HIDE_FROM_ABI - void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} - - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT - {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT - {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void notify_one() volatile _NOEXCEPT - {__cxx_atomic_notify_one(&__a_);} - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void notify_one() _NOEXCEPT - {__cxx_atomic_notify_one(&__a_);} - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void notify_all() volatile _NOEXCEPT - {__cxx_atomic_notify_all(&__a_);} - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI - void notify_all() _NOEXCEPT - {__cxx_atomic_notify_all(&__a_);} +struct atomic_flag { + __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; + + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const + volatile _NOEXCEPT { + __cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void + wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + __cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { + __cxx_atomic_notify_one(&__a_); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { __cxx_atomic_notify_one(&__a_); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { + __cxx_atomic_notify_all(&__a_); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { __cxx_atomic_notify_all(&__a_); } #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI constexpr - atomic_flag() _NOEXCEPT : __a_(false) {} + _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {} #else - atomic_flag() _NOEXCEPT = default; + atomic_flag() _NOEXCEPT = default; #endif - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION - - atomic_flag(const atomic_flag&) = delete; - atomic_flag& operator=(const atomic_flag&) = delete; - atomic_flag& operator=(const atomic_flag&) volatile = delete; + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION + atomic_flag(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) volatile = delete; }; -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT -{ - return __o->test(); -} +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test(const atomic_flag* __o) _NOEXCEPT -{ - return __o->test(); -} +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - return __o->test(__m); +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - return __o->test(__m); +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT -{ - return __o->test_and_set(); +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT { + return __o->test_and_set(); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT -{ - return __o->test_and_set(); -} +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT { return __o->test_and_set(); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - return __o->test_and_set(__m); +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); } -inline _LIBCPP_HIDE_FROM_ABI -bool -atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - return __o->test_and_set(__m); +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); } -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT -{ - __o->clear(); -} +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT { __o->clear(); } -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_flag_clear(atomic_flag* __o) _NOEXCEPT -{ - __o->clear(); -} +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(atomic_flag* __o) _NOEXCEPT { __o->clear(); } -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - __o->clear(__m); +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); } -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT -{ - __o->clear(__m); +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT -{ - __o->wait(__v); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT { + __o->wait(__v); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT -{ - __o->wait(__v); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT { + __o->wait(__v); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_wait_explicit(const volatile atomic_flag* __o, - bool __v, memory_order __m) _NOEXCEPT -{ - __o->wait(__v, __m); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_wait_explicit(const atomic_flag* __o, - bool __v, memory_order __m) _NOEXCEPT -{ - __o->wait(__v, __m); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT -{ - __o->notify_one(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT { + __o->notify_one(); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT -{ - __o->notify_one(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT { + __o->notify_one(); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT -{ - __o->notify_all(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void +atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT { + __o->notify_all(); } -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC -void -atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT -{ - __o->notify_all(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT { + __o->notify_all(); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/atomic_lock_free.h b/libcxx/include/__atomic/atomic_lock_free.h index d607569ed07da..0715439db4503 100644 --- a/libcxx/include/__atomic/atomic_lock_free.h +++ b/libcxx/include/__atomic/atomic_lock_free.h @@ -16,33 +16,33 @@ #endif #if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) -# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE -# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE -#ifndef _LIBCPP_HAS_NO_CHAR8_T -# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE -#endif -# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE -# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE -# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE -# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE -# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE -# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE -# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE -# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE +# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE +# ifndef _LIBCPP_HAS_NO_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE #elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) -# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE -# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE -#ifndef _LIBCPP_HAS_NO_CHAR8_T -# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE -#endif -# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE -# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE -# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE -# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE -# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE -# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE -# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE -# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE +# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE +# ifndef _LIBCPP_HAS_NO_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE #endif #endif // _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index d55450bb5f9c5..3d20d6a8ce251 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -34,77 +34,73 @@ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t +__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); template struct __libcpp_atomic_wait_backoff_impl { - _Atp* __a; - _Fn __test_fn; - _LIBCPP_AVAILABILITY_SYNC - _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const - { - if(__elapsed > chrono::microseconds(64)) - { - auto const __monitor = std::__libcpp_atomic_monitor(__a); - if(__test_fn()) - return true; - std::__libcpp_atomic_wait(__a, __monitor); - } - else if(__elapsed > chrono::microseconds(4)) - __libcpp_thread_yield(); - else - {} // poll - return false; - } + _Atp* __a; + _Fn __test_fn; + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(64)) { + auto const __monitor = std::__libcpp_atomic_monitor(__a); + if (__test_fn()) + return true; + std::__libcpp_atomic_wait(__a, __monitor); + } else if (__elapsed > chrono::microseconds(4)) + __libcpp_thread_yield(); + else { + } // poll + return false; + } }; template -_LIBCPP_AVAILABILITY_SYNC -_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) -{ - __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Fn> > __backoff_fn = {__a, __test_fn}; - return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Fn&& __test_fn) { + __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Fn> > __backoff_fn = {__a, __test_fn}; + return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); } #else // _LIBCPP_HAS_NO_THREADS template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) {} template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} template -_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) -{ - return std::__libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp*, _Fn&& __test_fn) { + return std::__libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); } #endif // _LIBCPP_HAS_NO_THREADS -template _LIBCPP_HIDE_FROM_ABI -bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { - return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; +template +_LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { + return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; } template struct __cxx_atomic_wait_test_fn_impl { - _Atp* __a; - _Tp __val; - memory_order __order; - _LIBCPP_HIDE_FROM_ABI bool operator()() const - { - return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); - } + _Atp* __a; + _Tp __val; + memory_order __order; + _LIBCPP_HIDE_FROM_ABI bool operator()() const { + return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); + } }; template -_LIBCPP_AVAILABILITY_SYNC -_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) -{ - __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; - return std::__cxx_atomic_wait(__a, __test_fn); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool +__cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { + __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; + return std::__cxx_atomic_wait(__a, __test_fn); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/check_memory_order.h b/libcxx/include/__atomic/check_memory_order.h index d744312122bd4..3012aec0521b3 100644 --- a/libcxx/include/__atomic/check_memory_order.h +++ b/libcxx/include/__atomic/check_memory_order.h @@ -15,20 +15,16 @@ # pragma GCC system_header #endif -#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ - _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ - __m == memory_order_acquire || \ - __m == memory_order_acq_rel, \ - "memory order argument to atomic operation is invalid") +#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || __m == memory_order_acquire || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") -#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ - _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ - __m == memory_order_acq_rel, \ - "memory order argument to atomic operation is invalid") +#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") -#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ - _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ - __f == memory_order_acq_rel, \ - "memory order argument to atomic operation is invalid") +#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ + _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || __f == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") #endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H diff --git a/libcxx/include/__atomic/contention_t.h b/libcxx/include/__atomic/contention_t.h index 1d8d02430b4f0..65890f338ce99 100644 --- a/libcxx/include/__atomic/contention_t.h +++ b/libcxx/include/__atomic/contention_t.h @@ -20,9 +20,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__)) - using __cxx_contention_t = int32_t; +using __cxx_contention_t = int32_t; #else - using __cxx_contention_t = int64_t; +using __cxx_contention_t = int64_t; #endif // __linux__ || (_AIX && !__64BIT__) using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h index 5d724669fee87..1a0b808a0cb1c 100644 --- a/libcxx/include/__atomic/cxx_atomic_impl.h +++ b/libcxx/include/__atomic/cxx_atomic_impl.h @@ -26,20 +26,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ - defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) +#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because // the default operator= in an object is not volatile, a byte-by-byte copy // is required. -template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { +template ::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { __a_value = __val; } -template ::value, int> = 0> _LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { +template ::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { volatile char* __to = reinterpret_cast(std::addressof(__a_value)); - volatile char* __end = __to + sizeof(_Tp); + volatile char* __end = __to + sizeof(_Tp); volatile const char* __from = reinterpret_cast(std::addressof(__val)); while (__to != __end) *__to++ = *__from++; @@ -51,78 +50,74 @@ void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& _ template struct __cxx_atomic_base_impl { - _LIBCPP_HIDE_FROM_ABI -#ifndef _LIBCPP_CXX03_LANG - __cxx_atomic_base_impl() _NOEXCEPT = default; -#else - __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} -#endif // _LIBCPP_CXX03_LANG - _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT - : __a_value(value) {} +# ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +# else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +# endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} _Tp __a_value; }; _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { // Avoid switch statement to make this a constexpr. - return __order == memory_order_relaxed ? __ATOMIC_RELAXED: - (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: - (__order == memory_order_release ? __ATOMIC_RELEASE: - (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: - (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: - __ATOMIC_CONSUME)))); + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELEASE + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME)))); } _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { // Avoid switch statement to make this a constexpr. - return __order == memory_order_relaxed ? __ATOMIC_RELAXED: - (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: - (__order == memory_order_release ? __ATOMIC_RELAXED: - (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: - (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: - __ATOMIC_CONSUME)))); + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELAXED + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME)))); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { __cxx_atomic_assign_volatile(__a->__a_value, __val); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { __a->__a_value = __val; } -_LIBCPP_HIDE_FROM_ABI inline -void __cxx_atomic_thread_fence(memory_order __order) { +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) { __atomic_thread_fence(__to_gcc_order(__order)); } -_LIBCPP_HIDE_FROM_ABI inline -void __cxx_atomic_signal_fence(memory_order __order) { +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) { __atomic_signal_fence(__to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { _Tp __ret; __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); return __ret; @@ -141,17 +136,15 @@ __cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, me } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { _Tp __ret; __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); return __ret; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __value, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { _Tp __ret; __atomic_exchange( std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); @@ -159,9 +152,7 @@ _Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { _Tp __ret; __atomic_exchange( std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); @@ -169,10 +160,12 @@ _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong( - volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, - memory_order __success, memory_order __failure) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { return __atomic_compare_exchange( std::addressof(__a->__a_value), __expected, @@ -183,10 +176,8 @@ bool __cxx_atomic_compare_exchange_strong( } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong( - __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, - memory_order __failure) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange( std::addressof(__a->__a_value), __expected, @@ -197,10 +188,12 @@ bool __cxx_atomic_compare_exchange_strong( } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak( - volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, - memory_order __success, memory_order __failure) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { return __atomic_compare_exchange( std::addressof(__a->__a_value), __expected, @@ -211,10 +204,8 @@ bool __cxx_atomic_compare_exchange_weak( } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak( - __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, - memory_order __failure) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange( std::addressof(__a->__a_value), __expected, @@ -225,193 +216,182 @@ bool __cxx_atomic_compare_exchange_weak( } template -struct __skip_amt { enum {value = 1}; }; +struct __skip_amt { + enum { value = 1 }; +}; template -struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; +struct __skip_amt<_Tp*> { + enum { value = sizeof(_Tp) }; +}; // FIXME: Haven't figured out what the spec says about using arrays with // atomic_fetch_add. Force a failure rather than creating bad behavior. template -struct __skip_amt<_Tp[]> { }; +struct __skip_amt<_Tp[]> {}; template -struct __skip_amt<_Tp[n]> { }; +struct __skip_amt<_Tp[n]> {}; template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Td __delta, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Td __delta, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, - memory_order __order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); } -#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) +# define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) #elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) template struct __cxx_atomic_base_impl { - _LIBCPP_HIDE_FROM_ABI -#ifndef _LIBCPP_CXX03_LANG - __cxx_atomic_base_impl() _NOEXCEPT = default; -#else - __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} -#endif // _LIBCPP_CXX03_LANG - _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT - : __a_value(__value) {} +# ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +# else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +# endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {} _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; }; -#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) +# define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) -_LIBCPP_HIDE_FROM_ABI inline -void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { - __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); } -_LIBCPP_HIDE_FROM_ABI inline -void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { - __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { - __c11_atomic_init(std::addressof(__a->__a_value), __val); +template +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); } -template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { - __c11_atomic_init(std::addressof(__a->__a_value), __val); +template +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); } -template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { - __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { - __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { - using __ptr_type = __remove_const_t__a_value)>*; - return __c11_atomic_load( - const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { - using __ptr_type = __remove_const_t__a_value)>*; - return __c11_atomic_load( - const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); } template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { - using __ptr_type = __remove_const_t__a_value)>*; - *__dst = __c11_atomic_load( - const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); + using __ptr_type = __remove_const_t__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); } template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { - using __ptr_type = __remove_const_t__a_value)>*; - *__dst = __c11_atomic_load( - const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); + using __ptr_type = __remove_const_t__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { - return __c11_atomic_exchange( - std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { - return __c11_atomic_exchange( - std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); } _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { // Avoid switch statement to make this a constexpr. - return __order == memory_order_release ? memory_order_relaxed: - (__order == memory_order_acq_rel ? memory_order_acquire: - __order); + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); } -template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { return __c11_atomic_compare_exchange_strong( std::addressof(__a->__a_value), __expected, @@ -419,9 +399,10 @@ bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); } -template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { return __c11_atomic_compare_exchange_strong( std::addressof(__a->__a_value), __expected, @@ -430,9 +411,13 @@ bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); } -template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { return __c11_atomic_compare_exchange_weak( std::addressof(__a->__a_value), __expected, @@ -440,9 +425,10 @@ bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __ static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); } -template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { return __c11_atomic_compare_exchange_weak( std::addressof(__a->__a_value), __expected, @@ -451,92 +437,92 @@ bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_add( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_add( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_add( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_add( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_sub( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_sub( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_sub( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_sub( std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_and( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_and( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_or( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_or( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_xor( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } -template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { +template +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { return __c11_atomic_fetch_xor( std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); } @@ -545,26 +531,23 @@ _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, mem #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS -template +template struct __cxx_atomic_lock_impl { - - _LIBCPP_HIDE_FROM_ABI - __cxx_atomic_lock_impl() _NOEXCEPT - : __a_value(), __a_lock(0) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit - __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT - : __a_value(value), __a_lock(0) {} + _LIBCPP_HIDE_FROM_ABI __cxx_atomic_lock_impl() _NOEXCEPT : __a_value(), __a_lock(0) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT + : __a_value(value), + __a_lock(0) {} _Tp __a_value; mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; _LIBCPP_HIDE_FROM_ABI void __lock() const volatile { - while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) - /*spin*/; + while (1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) + /*spin*/; } _LIBCPP_HIDE_FROM_ABI void __lock() const { - while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) - /*spin*/; + while (1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) + /*spin*/; } _LIBCPP_HIDE_FROM_ABI void __unlock() const volatile { __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); @@ -598,39 +581,33 @@ struct __cxx_atomic_lock_impl { }; template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { __cxx_atomic_assign_volatile(__a->__a_value, __val); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { __a->__a_value = __val; } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { __a->__lock(); __cxx_atomic_assign_volatile(__a->__a_value, __val); __a->__unlock(); } template -_LIBCPP_HIDE_FROM_ABI -void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { __a->__lock(); __a->__a_value = __val; __a->__unlock(); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { return __a->__read(); } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { return __a->__read(); } @@ -645,8 +622,7 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -655,24 +631,22 @@ _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { __a->__lock(); - _Tp __old = __a->__a_value; + _Tp __old = __a->__a_value; __a->__a_value = __value; __a->__unlock(); return __old; } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Tp* __expected, _Tp __value, memory_order, memory_order) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) { _Tp __temp; __a->__lock(); __cxx_atomic_assign_volatile(__temp, __a->__a_value); bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); - if(__ret) + if (__ret) __cxx_atomic_assign_volatile(__a->__a_value, __value); else __cxx_atomic_assign_volatile(*__expected, __a->__a_value); @@ -680,12 +654,11 @@ bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* return __ret; } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, - _Tp* __expected, _Tp __value, memory_order, memory_order) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) { __a->__lock(); bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); - if(__ret) + if (__ret) std::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); else std::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); @@ -694,14 +667,13 @@ bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Tp* __expected, _Tp __value, memory_order, memory_order) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) { _Tp __temp; __a->__lock(); __cxx_atomic_assign_volatile(__temp, __a->__a_value); bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); - if(__ret) + if (__ret) __cxx_atomic_assign_volatile(__a->__a_value, __value); else __cxx_atomic_assign_volatile(*__expected, __a->__a_value); @@ -709,12 +681,11 @@ bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __ return __ret; } template -_LIBCPP_HIDE_FROM_ABI -bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, - _Tp* __expected, _Tp __value, memory_order, memory_order) { +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) { __a->__lock(); bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); - if(__ret) + if (__ret) std::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); else std::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); @@ -723,9 +694,7 @@ bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Td __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -734,9 +703,7 @@ _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, - _Td __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) { __a->__lock(); _Tp __old = __a->__a_value; __a->__a_value += __delta; @@ -745,9 +712,8 @@ _Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, - ptrdiff_t __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order) { __a->__lock(); _Tp* __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -756,9 +722,7 @@ _Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, - ptrdiff_t __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order) { __a->__lock(); _Tp* __old = __a->__a_value; __a->__a_value += __delta; @@ -767,9 +731,7 @@ _Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Td __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -778,9 +740,7 @@ _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, - _Td __delta, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) { __a->__lock(); _Tp __old = __a->__a_value; __a->__a_value -= __delta; @@ -789,9 +749,8 @@ _Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -800,9 +759,7 @@ _Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old = __a->__a_value; __a->__a_value &= __pattern; @@ -811,9 +768,8 @@ _Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -822,9 +778,7 @@ _Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old = __a->__a_value; __a->__a_value |= __pattern; @@ -833,9 +787,8 @@ _Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old; __cxx_atomic_assign_volatile(__old, __a->__a_value); @@ -844,9 +797,7 @@ _Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, return __old; } template -_LIBCPP_HIDE_FROM_ABI -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, - _Tp __pattern, memory_order) { +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) { __a->__lock(); _Tp __old = __a->__a_value; __a->__a_value ^= __pattern; @@ -859,16 +810,13 @@ template , __cxx_atomic_lock_impl<_Tp> >::type> #else -template > +template > #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS struct __cxx_atomic_impl : public _Base { - static_assert(is_trivially_copyable<_Tp>::value, - "std::atomic requires that 'T' be a trivially copyable type"); + static_assert(is_trivially_copyable<_Tp>::value, "std::atomic requires that 'T' be a trivially copyable type"); _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default; - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT - : _Base(__value) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {} }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/fence.h b/libcxx/include/__atomic/fence.h index c62f38f21157b..8c27ea54d62dd 100644 --- a/libcxx/include/__atomic/fence.h +++ b/libcxx/include/__atomic/fence.h @@ -19,19 +19,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_thread_fence(memory_order __m) _NOEXCEPT -{ - __cxx_atomic_thread_fence(__m); -} - -inline _LIBCPP_HIDE_FROM_ABI -void -atomic_signal_fence(memory_order __m) _NOEXCEPT -{ - __cxx_atomic_signal_fence(__m); -} +inline _LIBCPP_HIDE_FROM_ABI void atomic_thread_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_thread_fence(__m); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_signal_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_signal_fence(__m); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/kill_dependency.h b/libcxx/include/__atomic/kill_dependency.h index 1bd5c8ca765a7..103d52d35787f 100644 --- a/libcxx/include/__atomic/kill_dependency.h +++ b/libcxx/include/__atomic/kill_dependency.h @@ -18,10 +18,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI -_Tp kill_dependency(_Tp __y) _NOEXCEPT -{ - return __y; +_LIBCPP_HIDE_FROM_ABI _Tp kill_dependency(_Tp __y) _NOEXCEPT { + return __y; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__atomic/memory_order.h b/libcxx/include/__atomic/memory_order.h index 3671dc3cf9be0..16fd1867698fa 100644 --- a/libcxx/include/__atomic/memory_order.h +++ b/libcxx/include/__atomic/memory_order.h @@ -22,14 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD // Figure out what the underlying type for `memory_order` would be if it were // declared as an unscoped enum (accounting for -fshort-enums). Use this result // to pin the underlying type in C++20. -enum __legacy_memory_order { - __mo_relaxed, - __mo_consume, - __mo_acquire, - __mo_release, - __mo_acq_rel, - __mo_seq_cst -}; +enum __legacy_memory_order { __mo_relaxed, __mo_consume, __mo_acquire, __mo_release, __mo_acq_rel, __mo_seq_cst }; using __memory_order_underlying_t = underlying_type<__legacy_memory_order>::type; @@ -45,7 +38,7 @@ enum class memory_order : __memory_order_underlying_t { }; static_assert((is_same::type, __memory_order_underlying_t>::value), - "unexpected underlying type for std::memory_order"); + "unexpected underlying type for std::memory_order"); inline constexpr auto memory_order_relaxed = memory_order::relaxed; inline constexpr auto memory_order_consume = memory_order::consume; diff --git a/libcxx/include/__availability b/libcxx/include/__availability index b5230b3f56b8d..e9904275e7341 100644 --- a/libcxx/include/__availability +++ b/libcxx/include/__availability @@ -64,25 +64,22 @@ // // [1]: https://clang.llvm.org/docs/AttributeReference.html#availability - // For backwards compatibility, allow users to define _LIBCPP_DISABLE_AVAILABILITY // for a while. #if defined(_LIBCPP_DISABLE_AVAILABILITY) -# if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) -# define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS -# endif +# if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) +# define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS +# endif #endif // Availability markup is disabled when building the library, or when the compiler // doesn't support the proper attributes. -#if defined(_LIBCPP_BUILDING_LIBRARY) || \ - defined(_LIBCXXABI_BUILDING_LIBRARY) || \ - !__has_feature(attribute_availability_with_strict) || \ - !__has_feature(attribute_availability_in_templates) || \ +#if defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCXXABI_BUILDING_LIBRARY) || \ + !__has_feature(attribute_availability_with_strict) || !__has_feature(attribute_availability_in_templates) || \ !__has_extension(pragma_clang_attribute_external_declaration) -# if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) -# define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS -# endif +# if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) +# define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS +# endif #endif #if defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) @@ -160,7 +157,7 @@ # define _LIBCPP_AVAILABILITY_HAS_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS # define _LIBCPP_AVAILABILITY_HAS_BAD_ANY_CAST _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS -# define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS __attribute__((availability(watchos,strict,introduced=5.0))) +# define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS __attribute__((availability(watchos, strict, introduced = 5.0))) # define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS # define _LIBCPP_AVAILABILITY_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS @@ -175,11 +172,11 @@ # else # define _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY 1 # endif -# define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY \ - __attribute__((availability(macos,strict,introduced=10.15))) \ - __attribute__((availability(ios,strict,introduced=13.0))) \ - __attribute__((availability(tvos,strict,introduced=13.0))) \ - __attribute__((availability(watchos,strict,introduced=6.0))) +# define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY \ + __attribute__((availability(macos, strict, introduced = 10.15))) \ + __attribute__((availability(ios, strict, introduced = 13.0))) \ + __attribute__((availability(tvos, strict, introduced = 13.0))) \ + __attribute__((availability(watchos, strict, introduced = 6.0))) // clang-format off # define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH \ _Pragma("clang attribute push(__attribute__((availability(macos,strict,introduced=10.15))), apply_to=any(function,record))") \ @@ -204,13 +201,13 @@ # else # define _LIBCPP_AVAILABILITY_HAS_TO_CHARS_FLOATING_POINT 1 # endif -# define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT \ - __attribute__((availability(macos,strict,introduced=13.3))) \ - __attribute__((availability(ios,strict,introduced=16.3))) \ - __attribute__((availability(tvos,strict,introduced=16.3))) \ - __attribute__((availability(watchos,strict,introduced=9.3))) +# define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT \ + __attribute__((availability(macos, strict, introduced = 13.3))) \ + __attribute__((availability(ios, strict, introduced = 16.3))) \ + __attribute__((availability(tvos, strict, introduced = 16.3))) \ + __attribute__((availability(watchos, strict, introduced = 9.3))) - // c++20 synchronization library +// c++20 synchronization library // clang-format off # if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000) || \ (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 140000) || \ @@ -220,19 +217,18 @@ # define _LIBCPP_AVAILABILITY_HAS_SYNC 0 # else # define _LIBCPP_AVAILABILITY_HAS_SYNC 1 -# endif -# define _LIBCPP_AVAILABILITY_SYNC \ - __attribute__((availability(macos,strict,introduced=11.0))) \ - __attribute__((availability(ios,strict,introduced=14.0))) \ - __attribute__((availability(tvos,strict,introduced=14.0))) \ - __attribute__((availability(watchos,strict,introduced=7.0))) +# endif +# define _LIBCPP_AVAILABILITY_SYNC \ + __attribute__((availability(macos, strict, introduced = 11.0))) \ + __attribute__((availability(ios, strict, introduced = 14.0))) \ + __attribute__((availability(tvos, strict, introduced = 14.0))) \ + __attribute__((availability(watchos, strict, introduced = 7.0))) // __libcpp_verbose_abort // TODO: Update once this is released # define _LIBCPP_AVAILABILITY_HAS_VERBOSE_ABORT 0 -# define _LIBCPP_AVAILABILITY_VERBOSE_ABORT \ - __attribute__((unavailable)) +# define _LIBCPP_AVAILABILITY_VERBOSE_ABORT __attribute__((unavailable)) // std::pmr // clang-format off @@ -276,7 +272,8 @@ // ...New vendors can add availability markup here... -# error "It looks like you're trying to enable vendor availability markup, but you haven't defined the corresponding macros yet!" +# error \ + "It looks like you're trying to enable vendor availability markup, but you haven't defined the corresponding macros yet!" #endif @@ -284,13 +281,13 @@ // Those are defined in terms of the availability attributes above, and // should not be vendor-specific. #if defined(_LIBCPP_HAS_NO_EXCEPTIONS) -# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST -# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS -# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST +# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS #else -# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_ANY_CAST -# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS -# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_ANY_CAST +# define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS +# define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS #endif #endif // _LIBCPP___AVAILABILITY diff --git a/libcxx/include/__bit/bit_cast.h b/libcxx/include/__bit/bit_cast.h index 39842465e12a4..f20b39ae748b1 100644 --- a/libcxx/include/__bit/bit_cast.h +++ b/libcxx/include/__bit/bit_cast.h @@ -22,8 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template - requires(sizeof(_ToType) == sizeof(_FromType) && - is_trivially_copyable_v<_ToType> && + requires(sizeof(_ToType) == sizeof(_FromType) && is_trivially_copyable_v<_ToType> && is_trivially_copyable_v<_FromType>) _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _ToType bit_cast(const _FromType& __from) noexcept { return __builtin_bit_cast(_ToType, __from); diff --git a/libcxx/include/__bit/bit_ceil.h b/libcxx/include/__bit/bit_ceil.h index 9eae5c391a835..17fe06aa41ccd 100644 --- a/libcxx/include/__bit/bit_ceil.h +++ b/libcxx/include/__bit/bit_ceil.h @@ -33,7 +33,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp __bit_ceil(_Tp __t) no if constexpr (sizeof(_Tp) >= sizeof(unsigned)) return _Tp{1} << __n; else { - const unsigned __extra = numeric_limits::digits - numeric_limits<_Tp>::digits; + const unsigned __extra = numeric_limits::digits - numeric_limits<_Tp>::digits; const unsigned __ret_val = 1u << (__n + __extra); return (_Tp)(__ret_val >> __extra); } diff --git a/libcxx/include/__bit/byteswap.h b/libcxx/include/__bit/byteswap.h index a1e1b530975e3..20045d6fd43cb 100644 --- a/libcxx/include/__bit/byteswap.h +++ b/libcxx/include/__bit/byteswap.h @@ -24,7 +24,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp byteswap(_Tp __val) noexcept { - if constexpr (sizeof(_Tp) == 1) { return __val; } else if constexpr (sizeof(_Tp) == 2) { @@ -33,15 +32,15 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp byteswap(_Tp __val) no return __builtin_bswap32(__val); } else if constexpr (sizeof(_Tp) == 8) { return __builtin_bswap64(__val); -#ifndef _LIBCPP_HAS_NO_INT128 +# ifndef _LIBCPP_HAS_NO_INT128 } else if constexpr (sizeof(_Tp) == 16) { -#if __has_builtin(__builtin_bswap128) +# if __has_builtin(__builtin_bswap128) return __builtin_bswap128(__val); -#else +# else return static_cast<_Tp>(byteswap(static_cast(__val))) << 64 | static_cast<_Tp>(byteswap(static_cast(__val >> 64))); -#endif // __has_builtin(__builtin_bswap128) -#endif // _LIBCPP_HAS_NO_INT128 +# endif // __has_builtin(__builtin_bswap128) +# endif // _LIBCPP_HAS_NO_INT128 } else { static_assert(sizeof(_Tp) == 0, "byteswap is unimplemented for integral types of this size"); } diff --git a/libcxx/include/__bit/countl.h b/libcxx/include/__bit/countl.h index efc9e200b13a8..396cfc2c3f406 100644 --- a/libcxx/include/__bit/countl.h +++ b/libcxx/include/__bit/countl.h @@ -24,18 +24,20 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_clz(unsigned __x) _NOEXCEPT { return __builtin_clz(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned __x) _NOEXCEPT { + return __builtin_clz(__x); +} -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_clz(unsigned long __x) _NOEXCEPT { return __builtin_clzl(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long __x) _NOEXCEPT { + return __builtin_clzl(__x); +} -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_clz(unsigned long long __x) _NOEXCEPT { return __builtin_clzll(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long long __x) _NOEXCEPT { + return __builtin_clzll(__x); +} -# ifndef _LIBCPP_HAS_NO_INT128 -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_clz(__uint128_t __x) _NOEXCEPT { +#ifndef _LIBCPP_HAS_NO_INT128 +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(__uint128_t __x) _NOEXCEPT { // The function is written in this form due to C++ constexpr limitations. // The algorithm: // - Test whether any bit in the high 64-bits is set @@ -45,42 +47,38 @@ int __libcpp_clz(__uint128_t __x) _NOEXCEPT { // - Any bits set: // - The number of leading zeros of the input is the number of leading // zeros in the high 64-bits. - return ((__x >> 64) == 0) - ? (64 + __builtin_clzll(static_cast(__x))) - : __builtin_clzll(static_cast(__x >> 64)); + return ((__x >> 64) == 0) ? (64 + __builtin_clzll(static_cast(__x))) + : __builtin_clzll(static_cast(__x >> 64)); } -# endif // _LIBCPP_HAS_NO_INT128 - -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -int __countl_zero(_Tp __t) _NOEXCEPT -{ - static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__countl_zero requires an unsigned integer type"); - if (__t == 0) - return numeric_limits<_Tp>::digits; - - if (sizeof(_Tp) <= sizeof(unsigned int)) - return std::__libcpp_clz(static_cast(__t)) - - (numeric_limits::digits - numeric_limits<_Tp>::digits); - else if (sizeof(_Tp) <= sizeof(unsigned long)) - return std::__libcpp_clz(static_cast(__t)) - - (numeric_limits::digits - numeric_limits<_Tp>::digits); - else if (sizeof(_Tp) <= sizeof(unsigned long long)) - return std::__libcpp_clz(static_cast(__t)) - - (numeric_limits::digits - numeric_limits<_Tp>::digits); - else - { - int __ret = 0; - int __iter = 0; - const unsigned int __ulldigits = numeric_limits::digits; - while (true) { - __t = std::__rotl(__t, __ulldigits); - if ((__iter = std::__countl_zero(static_cast(__t))) != __ulldigits) - break; - __ret += __iter; - } - return __ret + __iter; +#endif // _LIBCPP_HAS_NO_INT128 + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT { + static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__countl_zero requires an unsigned integer type"); + if (__t == 0) + return numeric_limits<_Tp>::digits; + + if (sizeof(_Tp) <= sizeof(unsigned int)) + return std::__libcpp_clz(static_cast(__t)) - + (numeric_limits::digits - numeric_limits<_Tp>::digits); + else if (sizeof(_Tp) <= sizeof(unsigned long)) + return std::__libcpp_clz(static_cast(__t)) - + (numeric_limits::digits - numeric_limits<_Tp>::digits); + else if (sizeof(_Tp) <= sizeof(unsigned long long)) + return std::__libcpp_clz(static_cast(__t)) - + (numeric_limits::digits - numeric_limits<_Tp>::digits); + else { + int __ret = 0; + int __iter = 0; + const unsigned int __ulldigits = numeric_limits::digits; + while (true) { + __t = std::__rotl(__t, __ulldigits); + if ((__iter = std::__countl_zero(static_cast(__t))) != __ulldigits) + break; + __ret += __iter; } + return __ret + __iter; + } } #if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h index 66ca5e7e66f2b..0cc679f87a99d 100644 --- a/libcxx/include/__bit/countr.h +++ b/libcxx/include/__bit/countr.h @@ -23,14 +23,17 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_ctz(unsigned __x) _NOEXCEPT { return __builtin_ctz(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned __x) _NOEXCEPT { + return __builtin_ctz(__x); +} -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_ctz(unsigned long __x) _NOEXCEPT { return __builtin_ctzl(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long __x) _NOEXCEPT { + return __builtin_ctzl(__x); +} -_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_ctz(unsigned long long __x) _NOEXCEPT { return __builtin_ctzll(__x); } +_LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long long __x) _NOEXCEPT { + return __builtin_ctzll(__x); +} #if _LIBCPP_STD_VER >= 20 @@ -46,7 +49,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n else if (sizeof(_Tp) <= sizeof(unsigned long long)) return std::__libcpp_ctz(static_cast(__t)); else { - int __ret = 0; + int __ret = 0; const unsigned int __ulldigits = numeric_limits::digits; while (static_cast(__t) == 0uLL) { __ret += __ulldigits; diff --git a/libcxx/include/__bit/endian.h b/libcxx/include/__bit/endian.h index 52635f2d24950..2d31e5ddff4f1 100644 --- a/libcxx/include/__bit/endian.h +++ b/libcxx/include/__bit/endian.h @@ -21,7 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD enum class endian { little = 0xDEAD, - big = 0xFACE, + big = 0xFACE, # if defined(_LIBCPP_LITTLE_ENDIAN) native = little # elif defined(_LIBCPP_BIG_ENDIAN) diff --git a/libcxx/include/__bit/popcount.h b/libcxx/include/__bit/popcount.h index f0bb87ea717ee..b0319cef25189 100644 --- a/libcxx/include/__bit/popcount.h +++ b/libcxx/include/__bit/popcount.h @@ -23,14 +23,17 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_popcount(unsigned __x) _NOEXCEPT { return __builtin_popcount(__x); } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned __x) _NOEXCEPT { + return __builtin_popcount(__x); +} -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_popcount(unsigned long __x) _NOEXCEPT { return __builtin_popcountl(__x); } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned long __x) _NOEXCEPT { + return __builtin_popcountl(__x); +} -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -int __libcpp_popcount(unsigned long long __x) _NOEXCEPT { return __builtin_popcountll(__x); } +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_popcount(unsigned long long __x) _NOEXCEPT { + return __builtin_popcountll(__x); +} #if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__chrono/calendar.h b/libcxx/include/__chrono/calendar.h index 91aaf63253890..bb1c5e7ebc8d0 100644 --- a/libcxx/include/__chrono/calendar.h +++ b/libcxx/include/__chrono/calendar.h @@ -22,19 +22,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { struct local_t {}; -template -using local_time = time_point; +template +using local_time = time_point; using local_seconds = local_time; using local_days = local_time; -struct last_spec { explicit last_spec() = default; }; +struct last_spec { + explicit last_spec() = default; +}; inline constexpr last_spec last{}; - } // namespace chrono _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__chrono/convert_to_timespec.h b/libcxx/include/__chrono/convert_to_timespec.h index fab07f2567d46..11e0b826d05b4 100644 --- a/libcxx/include/__chrono/convert_to_timespec.h +++ b/libcxx/include/__chrono/convert_to_timespec.h @@ -26,23 +26,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD // Convert a nanoseconds duration to the given TimeSpec type, which must have // the same properties as std::timespec. template -_LIBCPP_HIDE_FROM_ABI inline -_TimeSpec __convert_to_timespec(const chrono::nanoseconds& __ns) -{ +_LIBCPP_HIDE_FROM_ABI inline _TimeSpec __convert_to_timespec(const chrono::nanoseconds& __ns) { using namespace chrono; seconds __s = duration_cast(__ns); _TimeSpec __ts; typedef decltype(__ts.tv_sec) __ts_sec; const __ts_sec __ts_sec_max = numeric_limits<__ts_sec>::max(); - if (__s.count() < __ts_sec_max) - { - __ts.tv_sec = static_cast<__ts_sec>(__s.count()); + if (__s.count() < __ts_sec_max) { + __ts.tv_sec = static_cast<__ts_sec>(__s.count()); __ts.tv_nsec = static_cast((__ns - __s).count()); - } - else - { - __ts.tv_sec = __ts_sec_max; + } else { + __ts.tv_sec = __ts_sec_max; __ts.tv_nsec = 999999999; // (10^9 - 1) } diff --git a/libcxx/include/__chrono/day.h b/libcxx/include/__chrono/day.h index d908453d5b082..7342084b08c88 100644 --- a/libcxx/include/__chrono/day.h +++ b/libcxx/include/__chrono/day.h @@ -22,58 +22,73 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class day { private: - unsigned char __d_; + unsigned char __d_; + public: - day() = default; - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr day(unsigned __val) noexcept : __d_(static_cast(__val)) {} - _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator++() noexcept { ++__d_; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr day operator++(int) noexcept { day __tmp = *this; ++(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator--() noexcept { --__d_; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr day operator--(int) noexcept { day __tmp = *this; --(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI constexpr day& operator+=(const days& __dd) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr day& operator-=(const days& __dd) noexcept; - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __d_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __d_ >= 1 && __d_ <= 31; } - }; - - -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const day& __lhs, const day& __rhs) noexcept -{ return static_cast(__lhs) == static_cast(__rhs); } + day() = default; + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr day(unsigned __val) noexcept + : __d_(static_cast(__val)) {} + _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator++() noexcept { + ++__d_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr day operator++(int) noexcept { + day __tmp = *this; + ++(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr day& operator--() noexcept { + --__d_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr day operator--(int) noexcept { + day __tmp = *this; + --(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI constexpr day& operator+=(const days& __dd) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr day& operator-=(const days& __dd) noexcept; + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __d_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __d_ >= 1 && __d_ <= 31; } +}; + +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const day& __lhs, const day& __rhs) noexcept { + return static_cast(__lhs) == static_cast(__rhs); +} _LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const day& __lhs, const day& __rhs) noexcept { return static_cast(__lhs) <=> static_cast(__rhs); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -day operator+ (const day& __lhs, const days& __rhs) noexcept -{ return day(static_cast(__lhs) + __rhs.count()); } +_LIBCPP_HIDE_FROM_ABI inline constexpr day operator+(const day& __lhs, const days& __rhs) noexcept { + return day(static_cast(__lhs) + __rhs.count()); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -day operator+ (const days& __lhs, const day& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr day operator+(const days& __lhs, const day& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -day operator- (const day& __lhs, const days& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr day operator-(const day& __lhs, const days& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -days operator-(const day& __lhs, const day& __rhs) noexcept -{ return days(static_cast(static_cast(__lhs)) - - static_cast(static_cast(__rhs))); } +_LIBCPP_HIDE_FROM_ABI inline constexpr days operator-(const day& __lhs, const day& __rhs) noexcept { + return days(static_cast(static_cast(__lhs)) - static_cast(static_cast(__rhs))); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -day& day::operator+=(const days& __dd) noexcept -{ *this = *this + __dd; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr day& day::operator+=(const days& __dd) noexcept { + *this = *this + __dd; + return *this; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -day& day::operator-=(const days& __dd) noexcept -{ *this = *this - __dd; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr day& day::operator-=(const days& __dd) noexcept { + *this = *this - __dd; + return *this; +} } // namespace chrono diff --git a/libcxx/include/__chrono/duration.h b/libcxx/include/__chrono/duration.h index a19d55fc66829..5693ee6440916 100644 --- a/libcxx/include/__chrono/duration.h +++ b/libcxx/include/__chrono/duration.h @@ -29,100 +29,82 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { -template > class _LIBCPP_TEMPLATE_VIS duration; +template > +class _LIBCPP_TEMPLATE_VIS duration; template struct __is_duration : false_type {}; template -struct __is_duration > : true_type {}; +struct __is_duration > : true_type {}; template -struct __is_duration > : true_type {}; +struct __is_duration > : true_type {}; template -struct __is_duration > : true_type {}; +struct __is_duration > : true_type {}; template -struct __is_duration > : true_type {}; +struct __is_duration > : true_type {}; } // namespace chrono template -struct _LIBCPP_TEMPLATE_VIS common_type, - chrono::duration<_Rep2, _Period2> > -{ - typedef chrono::duration::type, - typename __ratio_gcd<_Period1, _Period2>::type> type; +struct _LIBCPP_TEMPLATE_VIS common_type, chrono::duration<_Rep2, _Period2> > { + typedef chrono::duration::type, typename __ratio_gcd<_Period1, _Period2>::type> + type; }; namespace chrono { // duration_cast -template ::type, - bool = _Period::num == 1, - bool = _Period::den == 1> + bool = _Period::num == 1, + bool = _Period::den == 1> struct __duration_cast; template -struct __duration_cast<_FromDuration, _ToDuration, _Period, true, true> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - _ToDuration operator()(const _FromDuration& __fd) const - { - return _ToDuration(static_cast(__fd.count())); - } +struct __duration_cast<_FromDuration, _ToDuration, _Period, true, true> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const { + return _ToDuration(static_cast(__fd.count())); + } }; template -struct __duration_cast<_FromDuration, _ToDuration, _Period, true, false> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - _ToDuration operator()(const _FromDuration& __fd) const - { - typedef typename common_type::type _Ct; - return _ToDuration(static_cast( - static_cast<_Ct>(__fd.count()) / static_cast<_Ct>(_Period::den))); - } +struct __duration_cast<_FromDuration, _ToDuration, _Period, true, false> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const { + typedef typename common_type::type _Ct; + return _ToDuration( + static_cast(static_cast<_Ct>(__fd.count()) / static_cast<_Ct>(_Period::den))); + } }; template -struct __duration_cast<_FromDuration, _ToDuration, _Period, false, true> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - _ToDuration operator()(const _FromDuration& __fd) const - { - typedef typename common_type::type _Ct; - return _ToDuration(static_cast( - static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num))); - } +struct __duration_cast<_FromDuration, _ToDuration, _Period, false, true> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const { + typedef typename common_type::type _Ct; + return _ToDuration( + static_cast(static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num))); + } }; template -struct __duration_cast<_FromDuration, _ToDuration, _Period, false, false> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - _ToDuration operator()(const _FromDuration& __fd) const - { - typedef typename common_type::type _Ct; - return _ToDuration(static_cast( - static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num) - / static_cast<_Ct>(_Period::den))); - } +struct __duration_cast<_FromDuration, _ToDuration, _Period, false, false> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration operator()(const _FromDuration& __fd) const { + typedef typename common_type::type _Ct; + return _ToDuration(static_cast( + static_cast<_Ct>(__fd.count()) * static_cast<_Ct>(_Period::num) / static_cast<_Ct>(_Period::den))); + } }; template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -_ToDuration -duration_cast(const duration<_Rep, _Period>& __fd) -{ - return __duration_cast, _ToDuration>()(__fd); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration duration_cast(const duration<_Rep, _Period>& __fd) { + return __duration_cast, _ToDuration>()(__fd); } template @@ -134,190 +116,204 @@ inline constexpr bool treat_as_floating_point_v = treat_as_floating_point<_Rep>: #endif template -struct _LIBCPP_TEMPLATE_VIS duration_values -{ +struct _LIBCPP_TEMPLATE_VIS duration_values { public: - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT {return _Rep(0);} - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT {return numeric_limits<_Rep>::max();} - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT {return numeric_limits<_Rep>::lowest();} + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT { return _Rep(0); } + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT { return numeric_limits<_Rep>::max(); } + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT { return numeric_limits<_Rep>::lowest(); } }; #if _LIBCPP_STD_VER >= 17 template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -_ToDuration -floor(const duration<_Rep, _Period>& __d) -{ - _ToDuration __t = chrono::duration_cast<_ToDuration>(__d); - if (__t > __d) - __t = __t - _ToDuration{1}; - return __t; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration floor(const duration<_Rep, _Period>& __d) { + _ToDuration __t = chrono::duration_cast<_ToDuration>(__d); + if (__t > __d) + __t = __t - _ToDuration{1}; + return __t; } template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -_ToDuration -ceil(const duration<_Rep, _Period>& __d) -{ - _ToDuration __t = chrono::duration_cast<_ToDuration>(__d); - if (__t < __d) - __t = __t + _ToDuration{1}; - return __t; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration ceil(const duration<_Rep, _Period>& __d) { + _ToDuration __t = chrono::duration_cast<_ToDuration>(__d); + if (__t < __d) + __t = __t + _ToDuration{1}; + return __t; } template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -_ToDuration -round(const duration<_Rep, _Period>& __d) -{ - _ToDuration __lower = chrono::floor<_ToDuration>(__d); - _ToDuration __upper = __lower + _ToDuration{1}; - auto __lower_diff = __d - __lower; - auto __upper_diff = __upper - __d; - if (__lower_diff < __upper_diff) - return __lower; - if (__lower_diff > __upper_diff) - return __upper; - return __lower.count() & 1 ? __upper : __lower; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration round(const duration<_Rep, _Period>& __d) { + _ToDuration __lower = chrono::floor<_ToDuration>(__d); + _ToDuration __upper = __lower + _ToDuration{1}; + auto __lower_diff = __d - __lower; + auto __upper_diff = __upper - __d; + if (__lower_diff < __upper_diff) + return __lower; + if (__lower_diff > __upper_diff) + return __upper; + return __lower.count() & 1 ? __upper : __lower; } #endif // duration template -class _LIBCPP_TEMPLATE_VIS duration -{ - static_assert(!__is_duration<_Rep>::value, "A duration representation can not be a duration"); - static_assert(__is_ratio<_Period>::value, "Second template parameter of duration must be a std::ratio"); - static_assert(_Period::num > 0, "duration period must be positive"); - - template - struct __no_overflow +class _LIBCPP_TEMPLATE_VIS duration { + static_assert(!__is_duration<_Rep>::value, "A duration representation can not be a duration"); + static_assert(__is_ratio<_Period>::value, "Second template parameter of duration must be a std::ratio"); + static_assert(_Period::num > 0, "duration period must be positive"); + + template + struct __no_overflow { + private: + static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value; + static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value; + static const intmax_t __n1 = _R1::num / __gcd_n1_n2; + static const intmax_t __d1 = _R1::den / __gcd_d1_d2; + static const intmax_t __n2 = _R2::num / __gcd_n1_n2; + static const intmax_t __d2 = _R2::den / __gcd_d1_d2; + static const intmax_t max = -((intmax_t(1) << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1); + + template + struct __mul // __overflow == false { - private: - static const intmax_t __gcd_n1_n2 = __static_gcd<_R1::num, _R2::num>::value; - static const intmax_t __gcd_d1_d2 = __static_gcd<_R1::den, _R2::den>::value; - static const intmax_t __n1 = _R1::num / __gcd_n1_n2; - static const intmax_t __d1 = _R1::den / __gcd_d1_d2; - static const intmax_t __n2 = _R2::num / __gcd_n1_n2; - static const intmax_t __d2 = _R2::den / __gcd_d1_d2; - static const intmax_t max = -((intmax_t(1) << (sizeof(intmax_t) * CHAR_BIT - 1)) + 1); - - template - struct __mul // __overflow == false - { - static const intmax_t value = _Xp * _Yp; - }; - - template - struct __mul<_Xp, _Yp, true> - { - static const intmax_t value = 1; - }; - - public: - static const bool value = (__n1 <= max / __d2) && (__n2 <= max / __d1); - typedef ratio<__mul<__n1, __d2, !value>::value, - __mul<__n2, __d1, !value>::value> type; + static const intmax_t value = _Xp * _Yp; + }; + + template + struct __mul<_Xp, _Yp, true> { + static const intmax_t value = 1; }; + public: + static const bool value = (__n1 <= max / __d2) && (__n2 <= max / __d1); + typedef ratio<__mul<__n1, __d2, !value>::value, __mul<__n2, __d1, !value>::value> type; + }; + public: - typedef _Rep rep; - typedef typename _Period::type period; + typedef _Rep rep; + typedef typename _Period::type period; + private: - rep __rep_; -public: + rep __rep_; +public: #ifndef _LIBCPP_CXX03_LANG - constexpr duration() = default; + constexpr duration() = default; #else - _LIBCPP_HIDE_FROM_ABI duration() {} + _LIBCPP_HIDE_FROM_ABI duration() {} #endif - template ::value && - (treat_as_floating_point::value || - !treat_as_floating_point<_Rep2>::value), int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - explicit duration(const _Rep2& __r) - : __rep_(__r) {} - - // conversions - template ::value && ( - treat_as_floating_point::value || - (__no_overflow<_Period2, period>::type::den == 1 && - !treat_as_floating_point<_Rep2>::value)), int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - duration(const duration<_Rep2, _Period2>& __d) - : __rep_(chrono::duration_cast(__d).count()) {} - - // observer - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR rep count() const {return __rep_;} - - // arithmetic - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type::type operator+() const {return typename common_type::type(*this);} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type::type operator-() const {return typename common_type::type(-__rep_);} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator++() {++__rep_; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator++(int) {return duration(__rep_++);} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator--() {--__rep_; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator--(int) {return duration(__rep_--);} - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator+=(const duration& __d) {__rep_ += __d.count(); return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator-=(const duration& __d) {__rep_ -= __d.count(); return *this;} - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator*=(const rep& __rhs) {__rep_ *= __rhs; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator/=(const rep& __rhs) {__rep_ /= __rhs; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const rep& __rhs) {__rep_ %= __rhs; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const duration& __rhs) {__rep_ %= __rhs.count(); return *this;} - - // special values - - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT {return duration(duration_values::zero());} - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT {return duration(duration_values::min());} - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT {return duration(duration_values::max());} + template ::value && + (treat_as_floating_point::value || !treat_as_floating_point<_Rep2>::value), + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit duration(const _Rep2& __r) : __rep_(__r) {} + + // conversions + template ::value && (treat_as_floating_point::value || + (__no_overflow<_Period2, period>::type::den == 1 && + !treat_as_floating_point<_Rep2>::value)), + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration(const duration<_Rep2, _Period2>& __d) + : __rep_(chrono::duration_cast(__d).count()) {} + + // observer + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR rep count() const { return __rep_; } + + // arithmetic + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type::type operator+() const { + return typename common_type::type(*this); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type::type operator-() const { + return typename common_type::type(-__rep_); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator++() { + ++__rep_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator++(int) { return duration(__rep_++); } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator--() { + --__rep_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration operator--(int) { return duration(__rep_--); } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator+=(const duration& __d) { + __rep_ += __d.count(); + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator-=(const duration& __d) { + __rep_ -= __d.count(); + return *this; + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator*=(const rep& __rhs) { + __rep_ *= __rhs; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator/=(const rep& __rhs) { + __rep_ /= __rhs; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const rep& __rhs) { + __rep_ %= __rhs; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator%=(const duration& __rhs) { + __rep_ %= __rhs.count(); + return *this; + } + + // special values + + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT { + return duration(duration_values::zero()); + } + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT { + return duration(duration_values::min()); + } + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT { + return duration(duration_values::max()); + } }; -typedef duration nanoseconds; -typedef duration microseconds; -typedef duration milliseconds; -typedef duration seconds; -typedef duration< long, ratio< 60> > minutes; -typedef duration< long, ratio<3600> > hours; +typedef duration nanoseconds; +typedef duration microseconds; +typedef duration milliseconds; +typedef duration seconds; +typedef duration< long, ratio< 60> > minutes; +typedef duration< long, ratio<3600> > hours; #if _LIBCPP_STD_VER >= 20 -typedef duration< int, ratio_multiply, hours::period>> days; -typedef duration< int, ratio_multiply, days::period>> weeks; -typedef duration< int, ratio_multiply, days::period>> years; -typedef duration< int, ratio_divide>> months; +typedef duration< int, ratio_multiply, hours::period>> days; +typedef duration< int, ratio_multiply, days::period>> weeks; +typedef duration< int, ratio_multiply, days::period>> years; +typedef duration< int, ratio_divide>> months; #endif // Duration == template -struct __duration_eq -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const - { - typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct; - return _Ct(__lhs).count() == _Ct(__rhs).count(); - } +struct __duration_eq { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const { + typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct; + return _Ct(__lhs).count() == _Ct(__rhs).count(); + } }; template -struct __duration_eq<_LhsDuration, _LhsDuration> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const - {return __lhs.count() == __rhs.count();} +struct __duration_eq<_LhsDuration, _LhsDuration> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const { + return __lhs.count() == __rhs.count(); + } }; template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator==(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return __duration_eq, duration<_Rep2, _Period2> >()(__lhs, __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator==(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return __duration_eq, duration<_Rep2, _Period2> >()(__lhs, __rhs); } #if _LIBCPP_STD_VER <= 17 @@ -325,12 +321,9 @@ operator==(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period // Duration != template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator!=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return !(__lhs == __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator!=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return !(__lhs == __rhs); } #endif // _LIBCPP_STD_VER <= 17 @@ -338,76 +331,58 @@ operator!=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period // Duration < template -struct __duration_lt -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const - { - typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct; - return _Ct(__lhs).count() < _Ct(__rhs).count(); - } +struct __duration_lt { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _RhsDuration& __rhs) const { + typedef typename common_type<_LhsDuration, _RhsDuration>::type _Ct; + return _Ct(__lhs).count() < _Ct(__rhs).count(); + } }; template -struct __duration_lt<_LhsDuration, _LhsDuration> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR - bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const - {return __lhs.count() < __rhs.count();} +struct __duration_lt<_LhsDuration, _LhsDuration> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(const _LhsDuration& __lhs, const _LhsDuration& __rhs) const { + return __lhs.count() < __rhs.count(); + } }; template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator< (const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return __duration_lt, duration<_Rep2, _Period2> >()(__lhs, __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator<(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return __duration_lt, duration<_Rep2, _Period2> >()(__lhs, __rhs); } // Duration > template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator> (const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return __rhs < __lhs; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator>(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return __rhs < __lhs; } // Duration <= template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator<=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return !(__rhs < __lhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator<=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return !(__rhs < __lhs); } // Duration >= template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -bool -operator>=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - return !(__lhs < __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool +operator>=(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + return !(__lhs < __rhs); } #if _LIBCPP_STD_VER >= 20 -template +template requires three_way_comparable> -_LIBCPP_HIDE_FROM_ABI -constexpr auto operator<=>(const duration<_Rep1, _Period1>& __lhs, - const duration<_Rep2, _Period2>& __rhs) -{ - using _Ct = common_type_t, duration<_Rep2, _Period2>>; - return _Ct(__lhs).count() <=> _Ct(__rhs).count(); +_LIBCPP_HIDE_FROM_ABI constexpr auto +operator<=>(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + using _Ct = common_type_t, duration<_Rep2, _Period2>>; + return _Ct(__lhs).count() <=> _Ct(__rhs).count(); } #endif // _LIBCPP_STD_VER >= 20 @@ -415,179 +390,151 @@ constexpr auto operator<=>(const duration<_Rep1, _Period1>& __lhs, // Duration + template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -typename common_type, duration<_Rep2, _Period2> >::type -operator+(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; - return _Cd(_Cd(__lhs).count() + _Cd(__rhs).count()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR + typename common_type, duration<_Rep2, _Period2> >::type + operator+(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; + return _Cd(_Cd(__lhs).count() + _Cd(__rhs).count()); } // Duration - template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -typename common_type, duration<_Rep2, _Period2> >::type -operator-(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; - return _Cd(_Cd(__lhs).count() - _Cd(__rhs).count()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR + typename common_type, duration<_Rep2, _Period2> >::type + operator-(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; + return _Cd(_Cd(__lhs).count() - _Cd(__rhs).count()); } // Duration * -template ::type>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -duration::type, _Period> -operator*(const duration<_Rep1, _Period>& __d, const _Rep2& __s) -{ - typedef typename common_type<_Rep1, _Rep2>::type _Cr; - typedef duration<_Cr, _Period> _Cd; - return _Cd(_Cd(__d).count() * static_cast<_Cr>(__s)); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration::type, _Period> +operator*(const duration<_Rep1, _Period>& __d, const _Rep2& __s) { + typedef typename common_type<_Rep1, _Rep2>::type _Cr; + typedef duration<_Cr, _Period> _Cd; + return _Cd(_Cd(__d).count() * static_cast<_Cr>(__s)); } -template ::type>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -duration::type, _Period> -operator*(const _Rep1& __s, const duration<_Rep2, _Period>& __d) -{ - return __d * __s; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration::type, _Period> +operator*(const _Rep1& __s, const duration<_Rep2, _Period>& __d) { + return __d * __s; } // Duration / -template ::value && is_convertible<_Rep2, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -duration::type, _Period> -operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s) -{ - typedef typename common_type<_Rep1, _Rep2>::type _Cr; - typedef duration<_Cr, _Period> _Cd; - return _Cd(_Cd(__d).count() / static_cast<_Cr>(__s)); +template ::value && + is_convertible<_Rep2, typename common_type<_Rep1, _Rep2>::type>::value, + int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration::type, _Period> +operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s) { + typedef typename common_type<_Rep1, _Rep2>::type _Cr; + typedef duration<_Cr, _Period> _Cd; + return _Cd(_Cd(__d).count() / static_cast<_Cr>(__s)); } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -typename common_type<_Rep1, _Rep2>::type -operator/(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef typename common_type, duration<_Rep2, _Period2> >::type _Ct; - return _Ct(__lhs).count() / _Ct(__rhs).count(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<_Rep1, _Rep2>::type +operator/(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef typename common_type, duration<_Rep2, _Period2> >::type _Ct; + return _Ct(__lhs).count() / _Ct(__rhs).count(); } // Duration % -template ::value && is_convertible<_Rep2, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -duration::type, _Period> -operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s) -{ - typedef typename common_type<_Rep1, _Rep2>::type _Cr; - typedef duration<_Cr, _Period> _Cd; - return _Cd(_Cd(__d).count() % static_cast<_Cr>(__s)); +template ::value && + is_convertible<_Rep2, typename common_type<_Rep1, _Rep2>::type>::value, + int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration::type, _Period> +operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s) { + typedef typename common_type<_Rep1, _Rep2>::type _Cr; + typedef duration<_Cr, _Period> _Cd; + return _Cd(_Cd(__d).count() % static_cast<_Cr>(__s)); } template -inline _LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR -typename common_type, duration<_Rep2, _Period2> >::type -operator%(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef typename common_type<_Rep1, _Rep2>::type _Cr; - typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; - return _Cd(static_cast<_Cr>(_Cd(__lhs).count()) % static_cast<_Cr>(_Cd(__rhs).count())); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR + typename common_type, duration<_Rep2, _Period2> >::type + operator%(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef typename common_type<_Rep1, _Rep2>::type _Cr; + typedef typename common_type, duration<_Rep2, _Period2> >::type _Cd; + return _Cd(static_cast<_Cr>(_Cd(__lhs).count()) % static_cast<_Cr>(_Cd(__rhs).count())); } } // namespace chrono #if _LIBCPP_STD_VER >= 14 // Suffixes for duration literals [time.duration.literals] -inline namespace literals -{ - inline namespace chrono_literals - { - - _LIBCPP_HIDE_FROM_ABI constexpr chrono::hours operator""h(unsigned long long __h) - { - return chrono::hours(static_cast(__h)); - } - - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration> operator""h(long double __h) - { - return chrono::duration>(__h); - } - - - _LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes operator""min(unsigned long long __m) - { - return chrono::minutes(static_cast(__m)); - } - - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration> operator""min(long double __m) - { - return chrono::duration> (__m); - } +inline namespace literals { +inline namespace chrono_literals { +_LIBCPP_HIDE_FROM_ABI constexpr chrono::hours operator""h(unsigned long long __h) { + return chrono::hours(static_cast(__h)); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds operator""s(unsigned long long __s) - { - return chrono::seconds(static_cast(__s)); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration> operator""h(long double __h) { + return chrono::duration>(__h); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""s(long double __s) - { - return chrono::duration (__s); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes operator""min(unsigned long long __m) { + return chrono::minutes(static_cast(__m)); +} +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration> operator""min(long double __m) { + return chrono::duration>(__m); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::milliseconds operator""ms(unsigned long long __ms) - { - return chrono::milliseconds(static_cast(__ms)); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds operator""s(unsigned long long __s) { + return chrono::seconds(static_cast(__s)); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""ms(long double __ms) - { - return chrono::duration(__ms); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""s(long double __s) { + return chrono::duration(__s); +} +_LIBCPP_HIDE_FROM_ABI constexpr chrono::milliseconds operator""ms(unsigned long long __ms) { + return chrono::milliseconds(static_cast(__ms)); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::microseconds operator""us(unsigned long long __us) - { - return chrono::microseconds(static_cast(__us)); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""ms(long double __ms) { + return chrono::duration(__ms); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""us(long double __us) - { - return chrono::duration (__us); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::microseconds operator""us(unsigned long long __us) { + return chrono::microseconds(static_cast(__us)); +} +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""us(long double __us) { + return chrono::duration(__us); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) - { - return chrono::nanoseconds(static_cast(__ns)); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) { + return chrono::nanoseconds(static_cast(__ns)); +} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""ns(long double __ns) - { - return chrono::duration (__ns); - } +_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration operator""ns(long double __ns) { + return chrono::duration(__ns); +} } // namespace chrono_literals } // namespace literals namespace chrono { // hoist the literals into namespace std::chrono - using namespace literals::chrono_literals; +using namespace literals::chrono_literals; } // namespace chrono #endif // _LIBCPP_STD_VER >= 14 diff --git a/libcxx/include/__chrono/file_clock.h b/libcxx/include/__chrono/file_clock.h index 9ea2fcb979355..7d25729fec013 100644 --- a/libcxx/include/__chrono/file_clock.h +++ b/libcxx/include/__chrono/file_clock.h @@ -31,13 +31,12 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { // [time.clock.file], type file_clock using file_clock = filesystem::_FilesystemClock; -template +template using file_time = time_point; } // namespace chrono @@ -49,35 +48,32 @@ _LIBCPP_END_NAMESPACE_STD #ifndef _LIBCPP_CXX03_LANG _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM struct _FilesystemClock { -#if !defined(_LIBCPP_HAS_NO_INT128) +# if !defined(_LIBCPP_HAS_NO_INT128) typedef __int128_t rep; typedef nano period; -#else +# else typedef long long rep; typedef nano period; -#endif +# endif typedef chrono::duration duration; typedef chrono::time_point<_FilesystemClock> time_point; - _LIBCPP_EXPORTED_FROM_ABI - static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false; + _LIBCPP_EXPORTED_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false; _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_EXPORTED_FROM_ABI static time_point now() noexcept; -#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_STD_VER >= 20 template - _LIBCPP_HIDE_FROM_ABI - static chrono::sys_time<_Duration> to_sys(const chrono::file_time<_Duration>& __t) { + _LIBCPP_HIDE_FROM_ABI static chrono::sys_time<_Duration> to_sys(const chrono::file_time<_Duration>& __t) { return chrono::sys_time<_Duration>(__t.time_since_epoch()); } template - _LIBCPP_HIDE_FROM_ABI - static chrono::file_time<_Duration> from_sys(const chrono::sys_time<_Duration>& __t) { + _LIBCPP_HIDE_FROM_ABI static chrono::file_time<_Duration> from_sys(const chrono::sys_time<_Duration>& __t) { return chrono::file_time<_Duration>(__t.time_since_epoch()); } -#endif // _LIBCPP_STD_VER >= 20 +# endif // _LIBCPP_STD_VER >= 20 }; _LIBCPP_END_NAMESPACE_FILESYSTEM #endif // !_LIBCPP_CXX03_LANG diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index 30ed360d4ef50..4ad59382a4148 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -198,7 +198,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( if (__year < 1000 || __year > 9999) __formatter::__format_century(__year, __sstr); else - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); } break; case _CharT('j'): @@ -209,7 +210,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( // an intemediate step. __sstr << chrono::duration_cast(chrono::duration_cast(__value)).count(); else - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); break; case _CharT('q'): @@ -237,7 +239,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( case _CharT('S'): case _CharT('T'): - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); if constexpr (__use_fraction<_Tp>()) __formatter::__format_sub_seconds(__value, __sstr); break; @@ -289,7 +292,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( __formatter::__format_year(__year, __sstr); __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday); } else - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); } break; case _CharT('Z'): @@ -304,7 +308,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( // fractional part should be formatted. if (*(__it + 1) == 'S') { ++__it; - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); __formatter::__format_sub_seconds(__value, __sstr); break; } @@ -314,7 +319,8 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( ++__it; [[fallthrough]]; default: - __facet.put({__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); + __facet.put( + {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); break; } } else { @@ -634,8 +640,7 @@ struct formatter, _CharT> : public __formatter_c }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -646,8 +651,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -658,8 +662,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -670,8 +673,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -682,8 +684,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -694,8 +695,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -706,8 +706,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -718,8 +717,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -730,8 +728,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -742,8 +739,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -754,8 +750,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -766,8 +761,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -778,8 +772,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -790,8 +783,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; @@ -802,8 +794,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_chrono<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_chrono<_CharT> { public: using _Base = __formatter_chrono<_CharT>; diff --git a/libcxx/include/__chrono/hh_mm_ss.h b/libcxx/include/__chrono/hh_mm_ss.h index 0adee2d60db8a..57d2247fe6a3c 100644 --- a/libcxx/include/__chrono/hh_mm_ss.h +++ b/libcxx/include/__chrono/hh_mm_ss.h @@ -24,85 +24,84 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { template -class hh_mm_ss -{ +class hh_mm_ss { private: - static_assert(__is_duration<_Duration>::value, "template parameter of hh_mm_ss must be a std::chrono::duration"); - using __CommonType = common_type_t<_Duration, chrono::seconds>; - - _LIBCPP_HIDE_FROM_ABI static constexpr uint64_t __pow10(unsigned __exp) - { - uint64_t __ret = 1; - for (unsigned __i = 0; __i < __exp; ++__i) - __ret *= 10U; - return __ret; - } - - _LIBCPP_HIDE_FROM_ABI static constexpr unsigned __width(uint64_t __n, uint64_t __d = 10, unsigned __w = 0) - { - if (__n >= 2 && __d != 0 && __w < 19) - return 1 + __width(__n, __d % __n * 10, __w+1); - return 0; - } + static_assert(__is_duration<_Duration>::value, "template parameter of hh_mm_ss must be a std::chrono::duration"); + using __CommonType = common_type_t<_Duration, chrono::seconds>; + + _LIBCPP_HIDE_FROM_ABI static constexpr uint64_t __pow10(unsigned __exp) { + uint64_t __ret = 1; + for (unsigned __i = 0; __i < __exp; ++__i) + __ret *= 10U; + return __ret; + } + + _LIBCPP_HIDE_FROM_ABI static constexpr unsigned __width(uint64_t __n, uint64_t __d = 10, unsigned __w = 0) { + if (__n >= 2 && __d != 0 && __w < 19) + return 1 + __width(__n, __d % __n * 10, __w + 1); + return 0; + } public: - _LIBCPP_HIDE_FROM_ABI static unsigned constexpr fractional_width = __width(__CommonType::period::den) < 19 ? - __width(__CommonType::period::den) : 6u; - using precision = duration>; + _LIBCPP_HIDE_FROM_ABI static unsigned constexpr fractional_width = + __width(__CommonType::period::den) < 19 ? __width(__CommonType::period::den) : 6u; + using precision = duration>; - _LIBCPP_HIDE_FROM_ABI constexpr hh_mm_ss() noexcept : hh_mm_ss{_Duration::zero()} {} + _LIBCPP_HIDE_FROM_ABI constexpr hh_mm_ss() noexcept : hh_mm_ss{_Duration::zero()} {} - _LIBCPP_HIDE_FROM_ABI constexpr explicit hh_mm_ss(_Duration __d) noexcept : - __is_neg_(__d < _Duration(0)), - __h_(chrono::duration_cast (chrono::abs(__d))), + _LIBCPP_HIDE_FROM_ABI constexpr explicit hh_mm_ss(_Duration __d) noexcept + : __is_neg_(__d < _Duration(0)), + __h_(chrono::duration_cast(chrono::abs(__d))), __m_(chrono::duration_cast(chrono::abs(__d) - hours())), __s_(chrono::duration_cast(chrono::abs(__d) - hours() - minutes())), - __f_(chrono::duration_cast (chrono::abs(__d) - hours() - minutes() - seconds())) - {} + __f_(chrono::duration_cast(chrono::abs(__d) - hours() - minutes() - seconds())) {} - _LIBCPP_HIDE_FROM_ABI constexpr bool is_negative() const noexcept { return __is_neg_; } - _LIBCPP_HIDE_FROM_ABI constexpr chrono::hours hours() const noexcept { return __h_; } - _LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes minutes() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds seconds() const noexcept { return __s_; } - _LIBCPP_HIDE_FROM_ABI constexpr precision subseconds() const noexcept { return __f_; } + _LIBCPP_HIDE_FROM_ABI constexpr bool is_negative() const noexcept { return __is_neg_; } + _LIBCPP_HIDE_FROM_ABI constexpr chrono::hours hours() const noexcept { return __h_; } + _LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes minutes() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds seconds() const noexcept { return __s_; } + _LIBCPP_HIDE_FROM_ABI constexpr precision subseconds() const noexcept { return __f_; } - _LIBCPP_HIDE_FROM_ABI constexpr precision to_duration() const noexcept - { - auto __dur = __h_ + __m_ + __s_ + __f_; - return __is_neg_ ? -__dur : __dur; - } + _LIBCPP_HIDE_FROM_ABI constexpr precision to_duration() const noexcept { + auto __dur = __h_ + __m_ + __s_ + __f_; + return __is_neg_ ? -__dur : __dur; + } - _LIBCPP_HIDE_FROM_ABI constexpr explicit operator precision() const noexcept { return to_duration(); } + _LIBCPP_HIDE_FROM_ABI constexpr explicit operator precision() const noexcept { return to_duration(); } private: - bool __is_neg_; - chrono::hours __h_; - chrono::minutes __m_; - chrono::seconds __s_; - precision __f_; + bool __is_neg_; + chrono::hours __h_; + chrono::minutes __m_; + chrono::seconds __s_; + precision __f_; }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(hh_mm_ss); -_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_am(const hours& __h) noexcept { return __h >= hours( 0) && __h < hours(12); } -_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_pm(const hours& __h) noexcept { return __h >= hours(12) && __h < hours(24); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_am(const hours& __h) noexcept { + return __h >= hours(0) && __h < hours(12); +} +_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_pm(const hours& __h) noexcept { + return __h >= hours(12) && __h < hours(24); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr hours make12(const hours& __h) noexcept -{ - if (__h == hours( 0)) return hours(12); - else if (__h <= hours(12)) return __h; - else return __h - hours(12); +_LIBCPP_HIDE_FROM_ABI inline constexpr hours make12(const hours& __h) noexcept { + if (__h == hours(0)) + return hours(12); + else if (__h <= hours(12)) + return __h; + else + return __h - hours(12); } -_LIBCPP_HIDE_FROM_ABI inline constexpr hours make24(const hours& __h, bool __is_pm) noexcept -{ - if (__is_pm) - return __h == hours(12) ? __h : __h + hours(12); - else - return __h == hours(12) ? hours(0) : __h; +_LIBCPP_HIDE_FROM_ABI inline constexpr hours make24(const hours& __h, bool __is_pm) noexcept { + if (__is_pm) + return __h == hours(12) ? __h : __h + hours(12); + else + return __h == hours(12) ? hours(0) : __h; } } // namespace chrono diff --git a/libcxx/include/__chrono/high_resolution_clock.h b/libcxx/include/__chrono/high_resolution_clock.h index 778ff44f3d09b..0697fd2de9b4d 100644 --- a/libcxx/include/__chrono/high_resolution_clock.h +++ b/libcxx/include/__chrono/high_resolution_clock.h @@ -20,8 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { #ifndef _LIBCPP_HAS_NO_MONOTONIC_CLOCK typedef steady_clock high_resolution_clock; diff --git a/libcxx/include/__chrono/literals.h b/libcxx/include/__chrono/literals.h index 28ddc43a2c0c6..89800440edf43 100644 --- a/libcxx/include/__chrono/literals.h +++ b/libcxx/include/__chrono/literals.h @@ -22,24 +22,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD -inline namespace literals -{ - inline namespace chrono_literals - { - _LIBCPP_HIDE_FROM_ABI constexpr chrono::day operator ""d(unsigned long long __d) noexcept - { - return chrono::day(static_cast(__d)); - } - - _LIBCPP_HIDE_FROM_ABI constexpr chrono::year operator ""y(unsigned long long __y) noexcept - { - return chrono::year(static_cast(__y)); - } +inline namespace literals { +inline namespace chrono_literals { +_LIBCPP_HIDE_FROM_ABI constexpr chrono::day operator""d(unsigned long long __d) noexcept { + return chrono::day(static_cast(__d)); +} + +_LIBCPP_HIDE_FROM_ABI constexpr chrono::year operator""y(unsigned long long __y) noexcept { + return chrono::year(static_cast(__y)); +} } // namespace chrono_literals } // namespace literals namespace chrono { // hoist the literals into namespace std::chrono - using namespace literals::chrono_literals; +using namespace literals::chrono_literals; } // namespace chrono _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__chrono/month.h b/libcxx/include/__chrono/month.h index 2dee5d8c6c70d..ce5cc21aab7d1 100644 --- a/libcxx/include/__chrono/month.h +++ b/libcxx/include/__chrono/month.h @@ -22,64 +22,76 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class month { private: - unsigned char __m_; + unsigned char __m_; + public: - month() = default; - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr month(unsigned __val) noexcept : __m_(static_cast(__val)) {} - _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator++() noexcept { *this += months{1}; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr month operator++(int) noexcept { month __tmp = *this; ++(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator--() noexcept { *this -= months{1}; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr month operator--(int) noexcept { month __tmp = *this; --(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI constexpr month& operator+=(const months& __m1) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr month& operator-=(const months& __m1) noexcept; - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_ >= 1 && __m_ <= 12; } + month() = default; + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr month(unsigned __val) noexcept + : __m_(static_cast(__val)) {} + _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator++() noexcept { + *this += months{1}; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr month operator++(int) noexcept { + month __tmp = *this; + ++(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr month& operator--() noexcept { + *this -= months{1}; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr month operator--(int) noexcept { + month __tmp = *this; + --(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI constexpr month& operator+=(const months& __m1) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr month& operator-=(const months& __m1) noexcept; + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator unsigned() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_ >= 1 && __m_ <= 12; } }; - -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const month& __lhs, const month& __rhs) noexcept -{ return static_cast(__lhs) == static_cast(__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const month& __lhs, const month& __rhs) noexcept { + return static_cast(__lhs) == static_cast(__rhs); +} _LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month& __lhs, const month& __rhs) noexcept { - return static_cast(__lhs) <=> static_cast(__rhs); + return static_cast(__lhs) <=> static_cast(__rhs); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -month operator+ (const month& __lhs, const months& __rhs) noexcept -{ - auto const __mu = static_cast(static_cast(__lhs)) + (__rhs.count() - 1); - auto const __yr = (__mu >= 0 ? __mu : __mu - 11) / 12; - return month{static_cast(__mu - __yr * 12 + 1)}; +_LIBCPP_HIDE_FROM_ABI inline constexpr month operator+(const month& __lhs, const months& __rhs) noexcept { + auto const __mu = static_cast(static_cast(__lhs)) + (__rhs.count() - 1); + auto const __yr = (__mu >= 0 ? __mu : __mu - 11) / 12; + return month{static_cast(__mu - __yr * 12 + 1)}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -month operator+ (const months& __lhs, const month& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month operator+(const months& __lhs, const month& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month operator- (const month& __lhs, const months& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month operator-(const month& __lhs, const months& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -months operator-(const month& __lhs, const month& __rhs) noexcept -{ - auto const __dm = static_cast(__lhs) - static_cast(__rhs); - return months(__dm <= 11 ? __dm : __dm + 12); +_LIBCPP_HIDE_FROM_ABI inline constexpr months operator-(const month& __lhs, const month& __rhs) noexcept { + auto const __dm = static_cast(__lhs) - static_cast(__rhs); + return months(__dm <= 11 ? __dm : __dm + 12); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -month& month::operator+=(const months& __dm) noexcept -{ *this = *this + __dm; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month& month::operator+=(const months& __dm) noexcept { + *this = *this + __dm; + return *this; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month& month::operator-=(const months& __dm) noexcept -{ *this = *this - __dm; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month& month::operator-=(const months& __dm) noexcept { + *this = *this - __dm; + return *this; +} inline constexpr month January{1}; inline constexpr month February{2}; diff --git a/libcxx/include/__chrono/month_weekday.h b/libcxx/include/__chrono/month_weekday.h index eb9e42d01f853..7919879655214 100644 --- a/libcxx/include/__chrono/month_weekday.h +++ b/libcxx/include/__chrono/month_weekday.h @@ -22,72 +22,80 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class month_weekday { private: - chrono::month __m_; - chrono::weekday_indexed __wdi_; + chrono::month __m_; + chrono::weekday_indexed __wdi_; + public: - _LIBCPP_HIDE_FROM_ABI constexpr month_weekday(const chrono::month& __mval, const chrono::weekday_indexed& __wdival) noexcept - : __m_{__mval}, __wdi_{__wdival} {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdi_.ok(); } + _LIBCPP_HIDE_FROM_ABI constexpr month_weekday(const chrono::month& __mval, + const chrono::weekday_indexed& __wdival) noexcept + : __m_{__mval}, __wdi_{__wdival} {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdi_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const month_weekday& __lhs, const month_weekday& __rhs) noexcept -{ return __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed(); } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday operator/(const month& __lhs, const weekday_indexed& __rhs) noexcept -{ return month_weekday{__lhs, __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const month_weekday& __lhs, const month_weekday& __rhs) noexcept { + return __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday operator/(int __lhs, const weekday_indexed& __rhs) noexcept -{ return month_weekday{month(__lhs), __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday +operator/(const month& __lhs, const weekday_indexed& __rhs) noexcept { + return month_weekday{__lhs, __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday operator/(const weekday_indexed& __lhs, const month& __rhs) noexcept -{ return month_weekday{__rhs, __lhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday operator/(int __lhs, const weekday_indexed& __rhs) noexcept { + return month_weekday{month(__lhs), __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday operator/(const weekday_indexed& __lhs, int __rhs) noexcept -{ return month_weekday{month(__rhs), __lhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday +operator/(const weekday_indexed& __lhs, const month& __rhs) noexcept { + return month_weekday{__rhs, __lhs}; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday operator/(const weekday_indexed& __lhs, int __rhs) noexcept { + return month_weekday{month(__rhs), __lhs}; +} class month_weekday_last { - chrono::month __m_; - chrono::weekday_last __wdl_; - public: - _LIBCPP_HIDE_FROM_ABI constexpr month_weekday_last(const chrono::month& __mval, const chrono::weekday_last& __wdlval) noexcept - : __m_{__mval}, __wdl_{__wdlval} {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdl_.ok(); } + chrono::month __m_; + chrono::weekday_last __wdl_; + +public: + _LIBCPP_HIDE_FROM_ABI constexpr month_weekday_last(const chrono::month& __mval, + const chrono::weekday_last& __wdlval) noexcept + : __m_{__mval}, __wdl_{__wdlval} {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok() && __wdl_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const month_weekday_last& __lhs, const month_weekday_last& __rhs) noexcept -{ return __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const month_weekday_last& __lhs, const month_weekday_last& __rhs) noexcept { + return __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday_last operator/(const month& __lhs, const weekday_last& __rhs) noexcept -{ return month_weekday_last{__lhs, __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last +operator/(const month& __lhs, const weekday_last& __rhs) noexcept { + return month_weekday_last{__lhs, __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday_last operator/(int __lhs, const weekday_last& __rhs) noexcept -{ return month_weekday_last{month(__lhs), __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last operator/(int __lhs, const weekday_last& __rhs) noexcept { + return month_weekday_last{month(__lhs), __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday_last operator/(const weekday_last& __lhs, const month& __rhs) noexcept -{ return month_weekday_last{__rhs, __lhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last +operator/(const weekday_last& __lhs, const month& __rhs) noexcept { + return month_weekday_last{__rhs, __lhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_weekday_last operator/(const weekday_last& __lhs, int __rhs) noexcept -{ return month_weekday_last{month(__rhs), __lhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last operator/(const weekday_last& __lhs, int __rhs) noexcept { + return month_weekday_last{month(__rhs), __lhs}; +} } // namespace chrono _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__chrono/monthday.h b/libcxx/include/__chrono/monthday.h index 8403d9ec4eebe..a89d16e518618 100644 --- a/libcxx/include/__chrono/monthday.h +++ b/libcxx/include/__chrono/monthday.h @@ -24,101 +24,105 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class month_day { private: - chrono::month __m_; - chrono::day __d_; + chrono::month __m_; + chrono::day __d_; + public: - month_day() = default; - _LIBCPP_HIDE_FROM_ABI constexpr month_day(const chrono::month& __mval, const chrono::day& __dval) noexcept - : __m_{__mval}, __d_{__dval} {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; } - _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; + month_day() = default; + _LIBCPP_HIDE_FROM_ABI constexpr month_day(const chrono::month& __mval, const chrono::day& __dval) noexcept + : __m_{__mval}, __d_{__dval} {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; } + _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool month_day::ok() const noexcept -{ - if (!__m_.ok()) return false; - const unsigned __dval = static_cast(__d_); - if (__dval < 1 || __dval > 31) return false; - if (__dval <= 29) return true; -// Now we've got either 30 or 31 - const unsigned __mval = static_cast(__m_); - if (__mval == 2) return false; - if (__mval == 4 || __mval == 6 || __mval == 9 || __mval == 11) - return __dval == 30; +_LIBCPP_HIDE_FROM_ABI inline constexpr bool month_day::ok() const noexcept { + if (!__m_.ok()) + return false; + const unsigned __dval = static_cast(__d_); + if (__dval < 1 || __dval > 31) + return false; + if (__dval <= 29) return true; + // Now we've got either 30 or 31 + const unsigned __mval = static_cast(__m_); + if (__mval == 2) + return false; + if (__mval == 4 || __mval == 6 || __mval == 9 || __mval == 11) + return __dval == 30; + return true; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const month_day& __lhs, const month_day& __rhs) noexcept -{ return __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const month_day& __lhs, const month_day& __rhs) noexcept { + return __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month_day& __lhs, const month_day& __rhs) noexcept { - if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0) - return __c; - return __lhs.day() <=> __rhs.day(); +_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering +operator<=>(const month_day& __lhs, const month_day& __rhs) noexcept { + if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0) + return __c; + return __lhs.day() <=> __rhs.day(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day operator/(const month& __lhs, const day& __rhs) noexcept -{ return month_day{__lhs, __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const month& __lhs, const day& __rhs) noexcept { + return month_day{__lhs, __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day operator/(const day& __lhs, const month& __rhs) noexcept -{ return __rhs / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const day& __lhs, const month& __rhs) noexcept { + return __rhs / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day operator/(const month& __lhs, int __rhs) noexcept -{ return __lhs / day(__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const month& __lhs, int __rhs) noexcept { + return __lhs / day(__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day operator/(int __lhs, const day& __rhs) noexcept -{ return month(__lhs) / __rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(int __lhs, const day& __rhs) noexcept { + return month(__lhs) / __rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day operator/(const day& __lhs, int __rhs) noexcept -{ return month(__rhs) / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day operator/(const day& __lhs, int __rhs) noexcept { + return month(__rhs) / __lhs; +} class month_day_last { private: - chrono::month __m_; + chrono::month __m_; + public: - _LIBCPP_HIDE_FROM_ABI explicit constexpr month_day_last(const chrono::month& __val) noexcept - : __m_{__val} {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok(); } + _LIBCPP_HIDE_FROM_ABI explicit constexpr month_day_last(const chrono::month& __val) noexcept : __m_{__val} {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __m_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const month_day_last& __lhs, const month_day_last& __rhs) noexcept -{ return __lhs.month() == __rhs.month(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const month_day_last& __lhs, const month_day_last& __rhs) noexcept { + return __lhs.month() == __rhs.month(); +} _LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month_day_last& __lhs, const month_day_last& __rhs) noexcept { - return __lhs.month() <=> __rhs.month(); + return __lhs.month() <=> __rhs.month(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day_last operator/(const month& __lhs, last_spec) noexcept -{ return month_day_last{__lhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(const month& __lhs, last_spec) noexcept { + return month_day_last{__lhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day_last operator/(last_spec, const month& __rhs) noexcept -{ return month_day_last{__rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(last_spec, const month& __rhs) noexcept { + return month_day_last{__rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day_last operator/(int __lhs, last_spec) noexcept -{ return month_day_last{month(__lhs)}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(int __lhs, last_spec) noexcept { + return month_day_last{month(__lhs)}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -month_day_last operator/(last_spec, int __rhs) noexcept -{ return month_day_last{month(__rhs)}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(last_spec, int __rhs) noexcept { + return month_day_last{month(__rhs)}; +} } // namespace chrono diff --git a/libcxx/include/__chrono/steady_clock.h b/libcxx/include/__chrono/steady_clock.h index cdcd2fa0d9709..612a7f156e634 100644 --- a/libcxx/include/__chrono/steady_clock.h +++ b/libcxx/include/__chrono/steady_clock.h @@ -20,20 +20,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { #ifndef _LIBCPP_HAS_NO_MONOTONIC_CLOCK -class _LIBCPP_EXPORTED_FROM_ABI steady_clock -{ +class _LIBCPP_EXPORTED_FROM_ABI steady_clock { public: - typedef nanoseconds duration; - typedef duration::rep rep; - typedef duration::period period; - typedef chrono::time_point time_point; - static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = true; + typedef nanoseconds duration; + typedef duration::rep rep; + typedef duration::period period; + typedef chrono::time_point time_point; + static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = true; - static time_point now() _NOEXCEPT; + static time_point now() _NOEXCEPT; }; #endif diff --git a/libcxx/include/__chrono/system_clock.h b/libcxx/include/__chrono/system_clock.h index 06fe071df2b60..5a9eb65bdae7a 100644 --- a/libcxx/include/__chrono/system_clock.h +++ b/libcxx/include/__chrono/system_clock.h @@ -21,21 +21,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { -class _LIBCPP_EXPORTED_FROM_ABI system_clock -{ +class _LIBCPP_EXPORTED_FROM_ABI system_clock { public: - typedef microseconds duration; - typedef duration::rep rep; - typedef duration::period period; - typedef chrono::time_point time_point; - static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false; - - static time_point now() _NOEXCEPT; - static time_t to_time_t (const time_point& __t) _NOEXCEPT; - static time_point from_time_t(time_t __t) _NOEXCEPT; + typedef microseconds duration; + typedef duration::rep rep; + typedef duration::period period; + typedef chrono::time_point time_point; + static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false; + + static time_point now() _NOEXCEPT; + static time_t to_time_t(const time_point& __t) _NOEXCEPT; + static time_point from_time_t(time_t __t) _NOEXCEPT; }; #if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__chrono/time_point.h b/libcxx/include/__chrono/time_point.h index 5060d9b55539c..e65253ddb98ee 100644 --- a/libcxx/include/__chrono/time_point.h +++ b/libcxx/include/__chrono/time_point.h @@ -28,108 +28,99 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { template -class _LIBCPP_TEMPLATE_VIS time_point -{ - static_assert(__is_duration<_Duration>::value, - "Second template parameter of time_point must be a std::chrono::duration"); +class _LIBCPP_TEMPLATE_VIS time_point { + static_assert(__is_duration<_Duration>::value, + "Second template parameter of time_point must be a std::chrono::duration"); + public: - typedef _Clock clock; - typedef _Duration duration; - typedef typename duration::rep rep; - typedef typename duration::period period; + typedef _Clock clock; + typedef _Duration duration; + typedef typename duration::rep rep; + typedef typename duration::period period; + private: - duration __d_; + duration __d_; public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point() : __d_(duration::zero()) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit time_point(const duration& __d) : __d_(__d) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point() : __d_(duration::zero()) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit time_point(const duration& __d) : __d_(__d) {} - // conversions - template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 - time_point(const time_point& __t) - : __d_(__t.time_since_epoch()) {} + // conversions + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point(const time_point& __t) + : __d_(__t.time_since_epoch()) {} - // observer + // observer - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 duration time_since_epoch() const {return __d_;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 duration time_since_epoch() const { return __d_; } - // arithmetic + // arithmetic - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator+=(const duration& __d) {__d_ += __d; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator-=(const duration& __d) {__d_ -= __d; return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator+=(const duration& __d) { + __d_ += __d; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator-=(const duration& __d) { + __d_ -= __d; + return *this; + } - // special values + // special values - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT {return time_point(duration::min());} - _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT {return time_point(duration::max());} + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT { return time_point(duration::min()); } + _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT { return time_point(duration::max()); } }; } // namespace chrono template -struct _LIBCPP_TEMPLATE_VIS common_type, - chrono::time_point<_Clock, _Duration2> > -{ - typedef chrono::time_point<_Clock, typename common_type<_Duration1, _Duration2>::type> type; +struct _LIBCPP_TEMPLATE_VIS + common_type, chrono::time_point<_Clock, _Duration2> > { + typedef chrono::time_point<_Clock, typename common_type<_Duration1, _Duration2>::type> type; }; namespace chrono { template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -time_point<_Clock, _ToDuration> -time_point_cast(const time_point<_Clock, _Duration>& __t) -{ - return time_point<_Clock, _ToDuration>(chrono::duration_cast<_ToDuration>(__t.time_since_epoch())); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, _ToDuration> +time_point_cast(const time_point<_Clock, _Duration>& __t) { + return time_point<_Clock, _ToDuration>(chrono::duration_cast<_ToDuration>(__t.time_since_epoch())); } #if _LIBCPP_STD_VER >= 17 template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -time_point<_Clock, _ToDuration> -floor(const time_point<_Clock, _Duration>& __t) -{ - return time_point<_Clock, _ToDuration>{chrono::floor<_ToDuration>(__t.time_since_epoch())}; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR time_point<_Clock, _ToDuration> +floor(const time_point<_Clock, _Duration>& __t) { + return time_point<_Clock, _ToDuration>{chrono::floor<_ToDuration>(__t.time_since_epoch())}; } template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -time_point<_Clock, _ToDuration> -ceil(const time_point<_Clock, _Duration>& __t) -{ - return time_point<_Clock, _ToDuration>{chrono::ceil<_ToDuration>(__t.time_since_epoch())}; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR time_point<_Clock, _ToDuration> +ceil(const time_point<_Clock, _Duration>& __t) { + return time_point<_Clock, _ToDuration>{chrono::ceil<_ToDuration>(__t.time_since_epoch())}; } template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -time_point<_Clock, _ToDuration> -round(const time_point<_Clock, _Duration>& __t) -{ - return time_point<_Clock, _ToDuration>{chrono::round<_ToDuration>(__t.time_since_epoch())}; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR time_point<_Clock, _ToDuration> +round(const time_point<_Clock, _Duration>& __t) { + return time_point<_Clock, _ToDuration>{chrono::round<_ToDuration>(__t.time_since_epoch())}; } template ::is_signed, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR -duration<_Rep, _Period> -abs(duration<_Rep, _Period> __d) -{ - return __d >= __d.zero() ? +__d : -__d; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<_Rep, _Period> abs(duration<_Rep, _Period> __d) { + return __d >= __d.zero() ? +__d : -__d; } #endif // _LIBCPP_STD_VER >= 17 // time_point == template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator==(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return __lhs.time_since_epoch() == __rhs.time_since_epoch(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator==(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return __lhs.time_since_epoch() == __rhs.time_since_epoch(); } #if _LIBCPP_STD_VER <= 17 @@ -137,11 +128,9 @@ operator==(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, // time_point != template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator!=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return !(__lhs == __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator!=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return !(__lhs == __rhs); } #endif // _LIBCPP_STD_VER <= 17 @@ -149,41 +138,33 @@ operator!=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, // time_point < template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator<(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return __lhs.time_since_epoch() < __rhs.time_since_epoch(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator<(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return __lhs.time_since_epoch() < __rhs.time_since_epoch(); } // time_point > template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return __rhs < __lhs; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return __rhs < __lhs; } // time_point <= template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator<=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return !(__rhs < __lhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator<=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return !(__rhs < __lhs); } // time_point >= template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -bool -operator>=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return !(__lhs < __rhs); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool +operator>=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return !(__lhs < __rhs); } #if _LIBCPP_STD_VER >= 20 @@ -191,7 +172,7 @@ operator>=(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, template _Duration2> _LIBCPP_HIDE_FROM_ABI constexpr auto operator<=>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { - return __lhs.time_since_epoch() <=> __rhs.time_since_epoch(); + return __lhs.time_since_epoch() <=> __rhs.time_since_epoch(); } #endif // _LIBCPP_STD_VER >= 20 @@ -200,42 +181,37 @@ operator<=>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> -operator+(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Tr; - return _Tr (__lhs.time_since_epoch() + __rhs); + time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> + operator+(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Tr; + return _Tr(__lhs.time_since_epoch() + __rhs); } // time_point operator+(duration x, time_point y); template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -time_point<_Clock, typename common_type, _Duration2>::type> -operator+(const duration<_Rep1, _Period1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return __rhs + __lhs; + time_point<_Clock, typename common_type, _Duration2>::type> + operator+(const duration<_Rep1, _Period1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return __rhs + __lhs; } // time_point operator-(time_point x, duration y); template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> -operator-(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) -{ - typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Ret; - return _Ret(__lhs.time_since_epoch() -__rhs); + time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> + operator-(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) { + typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Ret; + return _Ret(__lhs.time_since_epoch() - __rhs); } // duration operator-(time_point x, time_point y); template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -typename common_type<_Duration1, _Duration2>::type -operator-(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) -{ - return __lhs.time_since_epoch() - __rhs.time_since_epoch(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename common_type<_Duration1, _Duration2>::type +operator-(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) { + return __lhs.time_since_epoch() - __rhs.time_since_epoch(); } } // namespace chrono diff --git a/libcxx/include/__chrono/weekday.h b/libcxx/include/__chrono/weekday.h index 292fcb40dc306..5a7dedc6e3a16 100644 --- a/libcxx/include/__chrono/weekday.h +++ b/libcxx/include/__chrono/weekday.h @@ -24,144 +24,155 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class weekday_indexed; class weekday_last; class weekday { private: - unsigned char __wd_; - _LIBCPP_HIDE_FROM_ABI static constexpr unsigned char __weekday_from_days(int __days) noexcept; + unsigned char __wd_; + _LIBCPP_HIDE_FROM_ABI static constexpr unsigned char __weekday_from_days(int __days) noexcept; + public: weekday() = default; - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr weekday(unsigned __val) noexcept : __wd_(static_cast(__val == 7 ? 0 : __val)) {} - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday(const sys_days& __sysd) noexcept - : __wd_(__weekday_from_days(__sysd.time_since_epoch().count())) {} + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr weekday(unsigned __val) noexcept + : __wd_(static_cast(__val == 7 ? 0 : __val)) {} + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday(const sys_days& __sysd) noexcept + : __wd_(__weekday_from_days(__sysd.time_since_epoch().count())) {} _LIBCPP_HIDE_FROM_ABI inline explicit constexpr weekday(const local_days& __locd) noexcept - : __wd_(__weekday_from_days(__locd.time_since_epoch().count())) {} - - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator++() noexcept { __wd_ = (__wd_ == 6 ? 0 : __wd_ + 1); return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator++(int) noexcept { weekday __tmp = *this; ++(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator--() noexcept { __wd_ = (__wd_ == 0 ? 6 : __wd_ - 1); return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator--(int) noexcept { weekday __tmp = *this; --(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator+=(const days& __dd) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator-=(const days& __dd) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned c_encoding() const noexcept { return __wd_; } + : __wd_(__weekday_from_days(__locd.time_since_epoch().count())) {} + + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator++() noexcept { + __wd_ = (__wd_ == 6 ? 0 : __wd_ + 1); + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator++(int) noexcept { + weekday __tmp = *this; + ++(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday& operator--() noexcept { + __wd_ = (__wd_ == 0 ? 6 : __wd_ - 1); + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator--(int) noexcept { + weekday __tmp = *this; + --(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator+=(const days& __dd) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr weekday& operator-=(const days& __dd) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned c_encoding() const noexcept { return __wd_; } _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned iso_encoding() const noexcept { return __wd_ == 0u ? 7 : __wd_; } _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __wd_ <= 6; } - _LIBCPP_HIDE_FROM_ABI constexpr weekday_indexed operator[](unsigned __index) const noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr weekday_last operator[](last_spec) const noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr weekday_indexed operator[](unsigned __index) const noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr weekday_last operator[](last_spec) const noexcept; }; - // https://howardhinnant.github.io/date_algorithms.html#weekday_from_days -_LIBCPP_HIDE_FROM_ABI inline constexpr -unsigned char weekday::__weekday_from_days(int __days) noexcept -{ - return static_cast( - static_cast(__days >= -4 ? (__days+4) % 7 : (__days+5) % 7 + 6) - ); +_LIBCPP_HIDE_FROM_ABI inline constexpr unsigned char weekday::__weekday_from_days(int __days) noexcept { + return static_cast(static_cast(__days >= -4 ? (__days + 4) % 7 : (__days + 5) % 7 + 6)); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const weekday& __lhs, const weekday& __rhs) noexcept -{ return __lhs.c_encoding() == __rhs.c_encoding(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const weekday& __lhs, const weekday& __rhs) noexcept { + return __lhs.c_encoding() == __rhs.c_encoding(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator< (const weekday& __lhs, const weekday& __rhs) noexcept -{ return __lhs.c_encoding() < __rhs.c_encoding(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<(const weekday& __lhs, const weekday& __rhs) noexcept { + return __lhs.c_encoding() < __rhs.c_encoding(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator> (const weekday& __lhs, const weekday& __rhs) noexcept -{ return __rhs < __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>(const weekday& __lhs, const weekday& __rhs) noexcept { + return __rhs < __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator<=(const weekday& __lhs, const weekday& __rhs) noexcept -{ return !(__rhs < __lhs);} +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<=(const weekday& __lhs, const weekday& __rhs) noexcept { + return !(__rhs < __lhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept -{ return !(__lhs < __rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept { + return !(__lhs < __rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday operator+(const weekday& __lhs, const days& __rhs) noexcept -{ - auto const __mu = static_cast(__lhs.c_encoding()) + __rhs.count(); - auto const __yr = (__mu >= 0 ? __mu : __mu - 6) / 7; - return weekday{static_cast(__mu - __yr * 7)}; +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator+(const weekday& __lhs, const days& __rhs) noexcept { + auto const __mu = static_cast(__lhs.c_encoding()) + __rhs.count(); + auto const __yr = (__mu >= 0 ? __mu : __mu - 6) / 7; + return weekday{static_cast(__mu - __yr * 7)}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday operator+(const days& __lhs, const weekday& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator+(const days& __lhs, const weekday& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday operator-(const weekday& __lhs, const days& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday operator-(const weekday& __lhs, const days& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -days operator-(const weekday& __lhs, const weekday& __rhs) noexcept -{ - const int __wdu = __lhs.c_encoding() - __rhs.c_encoding(); - const int __wk = (__wdu >= 0 ? __wdu : __wdu-6) / 7; - return days{__wdu - __wk * 7}; +_LIBCPP_HIDE_FROM_ABI inline constexpr days operator-(const weekday& __lhs, const weekday& __rhs) noexcept { + const int __wdu = __lhs.c_encoding() - __rhs.c_encoding(); + const int __wk = (__wdu >= 0 ? __wdu : __wdu - 6) / 7; + return days{__wdu - __wk * 7}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday& weekday::operator+=(const days& __dd) noexcept -{ *this = *this + __dd; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday& weekday::operator+=(const days& __dd) noexcept { + *this = *this + __dd; + return *this; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday& weekday::operator-=(const days& __dd) noexcept -{ *this = *this - __dd; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday& weekday::operator-=(const days& __dd) noexcept { + *this = *this - __dd; + return *this; +} class weekday_indexed { private: - chrono::weekday __wd_; - unsigned char __idx_; + chrono::weekday __wd_; + unsigned char __idx_; + public: - weekday_indexed() = default; - _LIBCPP_HIDE_FROM_ABI inline constexpr weekday_indexed(const chrono::weekday& __wdval, unsigned __idxval) noexcept - : __wd_{__wdval}, __idx_(__idxval) {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wd_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __idx_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __wd_.ok() && __idx_ >= 1 && __idx_ <= 5; } + weekday_indexed() = default; + _LIBCPP_HIDE_FROM_ABI inline constexpr weekday_indexed(const chrono::weekday& __wdval, unsigned __idxval) noexcept + : __wd_{__wdval}, __idx_(__idxval) {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wd_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __idx_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __wd_.ok() && __idx_ >= 1 && __idx_ <= 5; } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const weekday_indexed& __lhs, const weekday_indexed& __rhs) noexcept -{ return __lhs.weekday() == __rhs.weekday() && __lhs.index() == __rhs.index(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const weekday_indexed& __lhs, const weekday_indexed& __rhs) noexcept { + return __lhs.weekday() == __rhs.weekday() && __lhs.index() == __rhs.index(); +} class weekday_last { private: - chrono::weekday __wd_; + chrono::weekday __wd_; + public: - _LIBCPP_HIDE_FROM_ABI explicit constexpr weekday_last(const chrono::weekday& __val) noexcept - : __wd_{__val} {} - _LIBCPP_HIDE_FROM_ABI constexpr chrono::weekday weekday() const noexcept { return __wd_; } - _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept { return __wd_.ok(); } + _LIBCPP_HIDE_FROM_ABI explicit constexpr weekday_last(const chrono::weekday& __val) noexcept : __wd_{__val} {} + _LIBCPP_HIDE_FROM_ABI constexpr chrono::weekday weekday() const noexcept { return __wd_; } + _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept { return __wd_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const weekday_last& __lhs, const weekday_last& __rhs) noexcept -{ return __lhs.weekday() == __rhs.weekday(); } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday_indexed weekday::operator[](unsigned __index) const noexcept { return weekday_indexed{*this, __index}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const weekday_last& __lhs, const weekday_last& __rhs) noexcept { + return __lhs.weekday() == __rhs.weekday(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -weekday_last weekday::operator[](last_spec) const noexcept { return weekday_last{*this}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday_indexed weekday::operator[](unsigned __index) const noexcept { + return weekday_indexed{*this, __index}; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr weekday_last weekday::operator[](last_spec) const noexcept { + return weekday_last{*this}; +} -inline constexpr weekday Sunday{0}; -inline constexpr weekday Monday{1}; -inline constexpr weekday Tuesday{2}; -inline constexpr weekday Wednesday{3}; -inline constexpr weekday Thursday{4}; -inline constexpr weekday Friday{5}; -inline constexpr weekday Saturday{6}; +inline constexpr weekday Sunday{0}; +inline constexpr weekday Monday{1}; +inline constexpr weekday Tuesday{2}; +inline constexpr weekday Wednesday{3}; +inline constexpr weekday Thursday{4}; +inline constexpr weekday Friday{5}; +inline constexpr weekday Saturday{6}; } // namespace chrono diff --git a/libcxx/include/__chrono/year.h b/libcxx/include/__chrono/year.h index 14bcbdafdd8a5..1899d09f38dbd 100644 --- a/libcxx/include/__chrono/year.h +++ b/libcxx/include/__chrono/year.h @@ -26,65 +26,81 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class year { private: - short __y_; + short __y_; + public: - year() = default; - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr year(int __val) noexcept : __y_(static_cast(__val)) {} - - _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator++() noexcept { ++__y_; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr year operator++(int) noexcept { year __tmp = *this; ++(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator--() noexcept { --__y_; return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr year operator--(int) noexcept { year __tmp = *this; --(*this); return __tmp; } - _LIBCPP_HIDE_FROM_ABI constexpr year& operator+=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year& operator-=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr year operator+() const noexcept { return *this; } - _LIBCPP_HIDE_FROM_ABI inline constexpr year operator-() const noexcept { return year{-__y_}; } - - _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_leap() const noexcept { return __y_ % 4 == 0 && (__y_ % 100 != 0 || __y_ % 400 == 0); } - _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator int() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; - _LIBCPP_HIDE_FROM_ABI static inline constexpr year min() noexcept { return year{-32767}; } - _LIBCPP_HIDE_FROM_ABI static inline constexpr year max() noexcept { return year{ 32767}; } + year() = default; + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr year(int __val) noexcept : __y_(static_cast(__val)) {} + + _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator++() noexcept { + ++__y_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr year operator++(int) noexcept { + year __tmp = *this; + ++(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr year& operator--() noexcept { + --__y_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr year operator--(int) noexcept { + year __tmp = *this; + --(*this); + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI constexpr year& operator+=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year& operator-=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr year operator+() const noexcept { return *this; } + _LIBCPP_HIDE_FROM_ABI inline constexpr year operator-() const noexcept { return year{-__y_}; } + + _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_leap() const noexcept { + return __y_ % 4 == 0 && (__y_ % 100 != 0 || __y_ % 400 == 0); + } + _LIBCPP_HIDE_FROM_ABI explicit inline constexpr operator int() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; + _LIBCPP_HIDE_FROM_ABI static inline constexpr year min() noexcept { return year{-32767}; } + _LIBCPP_HIDE_FROM_ABI static inline constexpr year max() noexcept { return year{32767}; } }; - -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year& __lhs, const year& __rhs) noexcept -{ return static_cast(__lhs) == static_cast(__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const year& __lhs, const year& __rhs) noexcept { + return static_cast(__lhs) == static_cast(__rhs); +} _LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const year& __lhs, const year& __rhs) noexcept { - return static_cast(__lhs) <=> static_cast(__rhs); + return static_cast(__lhs) <=> static_cast(__rhs); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -year operator+ (const year& __lhs, const years& __rhs) noexcept -{ return year(static_cast(__lhs) + __rhs.count()); } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year operator+ (const years& __lhs, const year& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year operator+(const year& __lhs, const years& __rhs) noexcept { + return year(static_cast(__lhs) + __rhs.count()); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year operator- (const year& __lhs, const years& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year operator+(const years& __lhs, const year& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -years operator-(const year& __lhs, const year& __rhs) noexcept -{ return years{static_cast(__lhs) - static_cast(__rhs)}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year operator-(const year& __lhs, const years& __rhs) noexcept { + return __lhs + -__rhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr years operator-(const year& __lhs, const year& __rhs) noexcept { + return years{static_cast(__lhs) - static_cast(__rhs)}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year& year::operator+=(const years& __dy) noexcept -{ *this = *this + __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year& year::operator+=(const years& __dy) noexcept { + *this = *this + __dy; + return *this; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year& year::operator-=(const years& __dy) noexcept -{ *this = *this - __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year& year::operator-=(const years& __dy) noexcept { + *this = *this - __dy; + return *this; +} _LIBCPP_HIDE_FROM_ABI constexpr bool year::ok() const noexcept { static_assert(static_cast(std::numeric_limits::max()) == static_cast(max())); diff --git a/libcxx/include/__chrono/year_month.h b/libcxx/include/__chrono/year_month.h index 320cf588ccd30..369ea38f7560d 100644 --- a/libcxx/include/__chrono/year_month.h +++ b/libcxx/include/__chrono/year_month.h @@ -24,73 +24,75 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class year_month { - chrono::year __y_; - chrono::month __m_; + chrono::year __y_; + chrono::month __m_; + public: - year_month() = default; - _LIBCPP_HIDE_FROM_ABI constexpr year_month(const chrono::year& __yval, const chrono::month& __mval) noexcept - : __y_{__yval}, __m_{__mval} {} - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok(); } + year_month() = default; + _LIBCPP_HIDE_FROM_ABI constexpr year_month(const chrono::year& __yval, const chrono::month& __mval) noexcept + : __y_{__yval}, __m_{__mval} {} + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator+=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& operator-=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator/(const year& __y, const month& __m) noexcept { return year_month{__y, __m}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator/(const year& __y, const month& __m) noexcept { + return year_month{__y, __m}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator/(const year& __y, int __m) noexcept { return year_month{__y, month(__m)}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator/(const year& __y, int __m) noexcept { + return year_month{__y, month(__m)}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year_month& __lhs, const year_month& __rhs) noexcept -{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(const year_month& __lhs, const year_month& __rhs) noexcept { + return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const year_month& __lhs, const year_month& __rhs) noexcept { - if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0) - return __c; - return __lhs.month() <=> __rhs.month(); +_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering +operator<=>(const year_month& __lhs, const year_month& __rhs) noexcept { + if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0) + return __c; + return __lhs.month() <=> __rhs.month(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator+(const year_month& __lhs, const months& __rhs) noexcept -{ - int __dmi = static_cast(static_cast(__lhs.month())) - 1 + __rhs.count(); - const int __dy = (__dmi >= 0 ? __dmi : __dmi-11) / 12; - __dmi = __dmi - __dy * 12 + 1; - return (__lhs.year() + years(__dy)) / month(static_cast(__dmi)); +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const year_month& __lhs, const months& __rhs) noexcept { + int __dmi = static_cast(static_cast(__lhs.month())) - 1 + __rhs.count(); + const int __dy = (__dmi >= 0 ? __dmi : __dmi - 11) / 12; + __dmi = __dmi - __dy * 12 + 1; + return (__lhs.year() + years(__dy)) / month(static_cast(__dmi)); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator+(const months& __lhs, const year_month& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const months& __lhs, const year_month& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator+(const year_month& __lhs, const years& __rhs) noexcept -{ return (__lhs.year() + __rhs) / __lhs.month(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const year_month& __lhs, const years& __rhs) noexcept { + return (__lhs.year() + __rhs) / __lhs.month(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator+(const years& __lhs, const year_month& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator+(const years& __lhs, const year_month& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -months operator-(const year_month& __lhs, const year_month& __rhs) noexcept -{ return (__lhs.year() - __rhs.year()) + months(static_cast(__lhs.month()) - static_cast(__rhs.month())); } +_LIBCPP_HIDE_FROM_ABI inline constexpr months operator-(const year_month& __lhs, const year_month& __rhs) noexcept { + return (__lhs.year() - __rhs.year()) + + months(static_cast(__lhs.month()) - static_cast(__rhs.month())); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator-(const year_month& __lhs, const months& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator-(const year_month& __lhs, const months& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month operator-(const year_month& __lhs, const years& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month operator-(const year_month& __lhs, const years& __rhs) noexcept { + return __lhs + -__rhs; +} _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator+=(const months& __dm) noexcept { *this = *this + __dm; diff --git a/libcxx/include/__chrono/year_month_day.h b/libcxx/include/__chrono/year_month_day.h index e84d2f8a838b4..75884f3654d87 100644 --- a/libcxx/include/__chrono/year_month_day.h +++ b/libcxx/include/__chrono/year_month_day.h @@ -31,271 +31,323 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class year_month_day_last; class year_month_day { private: - chrono::year __y_; - chrono::month __m_; - chrono::day __d_; + chrono::year __y_; + chrono::month __m_; + chrono::day __d_; + public: - year_month_day() = default; - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day( - const chrono::year& __yval, const chrono::month& __mval, const chrono::day& __dval) noexcept - : __y_{__yval}, __m_{__mval}, __d_{__dval} {} - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day(const year_month_day_last& __ymdl) noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day(const sys_days& __sysd) noexcept - : year_month_day(__from_days(__sysd.time_since_epoch())) {} - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_day(const local_days& __locd) noexcept - : year_month_day(__from_days(__locd.time_since_epoch())) {} - - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const years& __dy) noexcept; - - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } - - _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; - - _LIBCPP_HIDE_FROM_ABI static constexpr year_month_day __from_days(days __d) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; + year_month_day() = default; + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day( + const chrono::year& __yval, const chrono::month& __mval, const chrono::day& __dval) noexcept + : __y_{__yval}, __m_{__mval}, __d_{__dval} {} + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day(const year_month_day_last& __ymdl) noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day(const sys_days& __sysd) noexcept + : year_month_day(__from_days(__sysd.time_since_epoch())) {} + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_day(const local_days& __locd) noexcept + : year_month_day(__from_days(__locd.time_since_epoch())) {} + + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator+=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day& operator-=(const years& __dy) noexcept; + + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day day() const noexcept { return __d_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { + return local_days{__to_days()}; + } + + _LIBCPP_HIDE_FROM_ABI constexpr bool ok() const noexcept; + + _LIBCPP_HIDE_FROM_ABI static constexpr year_month_day __from_days(days __d) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; }; - // https://howardhinnant.github.io/date_algorithms.html#civil_from_days -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day year_month_day::__from_days(days __d) noexcept -{ - static_assert(numeric_limits::digits >= 18, ""); - static_assert(numeric_limits::digits >= 20 , ""); - const int __z = __d.count() + 719468; - const int __era = (__z >= 0 ? __z : __z - 146096) / 146097; - const unsigned __doe = static_cast(__z - __era * 146097); // [0, 146096] - const unsigned __yoe = (__doe - __doe/1460 + __doe/36524 - __doe/146096) / 365; // [0, 399] - const int __yr = static_cast(__yoe) + __era * 400; - const unsigned __doy = __doe - (365 * __yoe + __yoe/4 - __yoe/100); // [0, 365] - const unsigned __mp = (5 * __doy + 2)/153; // [0, 11] - const unsigned __dy = __doy - (153 * __mp + 2)/5 + 1; // [1, 31] - const unsigned __mth = __mp + (__mp < 10 ? 3 : -9); // [1, 12] - return year_month_day{chrono::year{__yr + (__mth <= 2)}, chrono::month{__mth}, chrono::day{__dy}}; +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day year_month_day::__from_days(days __d) noexcept { + static_assert(numeric_limits::digits >= 18, ""); + static_assert(numeric_limits::digits >= 20, ""); + const int __z = __d.count() + 719468; + const int __era = (__z >= 0 ? __z : __z - 146096) / 146097; + const unsigned __doe = static_cast(__z - __era * 146097); // [0, 146096] + const unsigned __yoe = (__doe - __doe / 1460 + __doe / 36524 - __doe / 146096) / 365; // [0, 399] + const int __yr = static_cast(__yoe) + __era * 400; + const unsigned __doy = __doe - (365 * __yoe + __yoe / 4 - __yoe / 100); // [0, 365] + const unsigned __mp = (5 * __doy + 2) / 153; // [0, 11] + const unsigned __dy = __doy - (153 * __mp + 2) / 5 + 1; // [1, 31] + const unsigned __mth = __mp + (__mp < 10 ? 3 : -9); // [1, 12] + return year_month_day{chrono::year{__yr + (__mth <= 2)}, chrono::month{__mth}, chrono::day{__dy}}; } // https://howardhinnant.github.io/date_algorithms.html#days_from_civil -_LIBCPP_HIDE_FROM_ABI inline constexpr -days year_month_day::__to_days() const noexcept -{ - static_assert(numeric_limits::digits >= 18, ""); - static_assert(numeric_limits::digits >= 20 , ""); - - const int __yr = static_cast(__y_) - (__m_ <= February); - const unsigned __mth = static_cast(__m_); - const unsigned __dy = static_cast(__d_); - - const int __era = (__yr >= 0 ? __yr : __yr - 399) / 400; - const unsigned __yoe = static_cast(__yr - __era * 400); // [0, 399] - const unsigned __doy = (153 * (__mth + (__mth > 2 ? -3 : 9)) + 2) / 5 + __dy-1; // [0, 365] - const unsigned __doe = __yoe * 365 + __yoe/4 - __yoe/100 + __doy; // [0, 146096] - return days{__era * 146097 + static_cast(__doe) - 719468}; +_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_day::__to_days() const noexcept { + static_assert(numeric_limits::digits >= 18, ""); + static_assert(numeric_limits::digits >= 20, ""); + + const int __yr = static_cast(__y_) - (__m_ <= February); + const unsigned __mth = static_cast(__m_); + const unsigned __dy = static_cast(__d_); + + const int __era = (__yr >= 0 ? __yr : __yr - 399) / 400; + const unsigned __yoe = static_cast(__yr - __era * 400); // [0, 399] + const unsigned __doy = (153 * (__mth + (__mth > 2 ? -3 : 9)) + 2) / 5 + __dy - 1; // [0, 365] + const unsigned __doe = __yoe * 365 + __yoe / 4 - __yoe / 100 + __doy; // [0, 146096] + return days{__era * 146097 + static_cast(__doe) - 719468}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year_month_day& __lhs, const year_month_day& __rhs) noexcept -{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const year_month_day& __lhs, const year_month_day& __rhs) noexcept { + return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); +} _LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const year_month_day& __lhs, const year_month_day& __rhs) noexcept { - if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0) - return __c; - if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0) - return __c; - return __lhs.day() <=> __rhs.day(); + if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0) + return __c; + if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0) + return __c; + return __lhs.day() <=> __rhs.day(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(const year_month& __lhs, const day& __rhs) noexcept -{ return year_month_day{__lhs.year(), __lhs.month(), __rhs}; } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(const year_month& __lhs, int __rhs) noexcept -{ return __lhs / day(__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year_month& __lhs, const day& __rhs) noexcept { + return year_month_day{__lhs.year(), __lhs.month(), __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(const year& __lhs, const month_day& __rhs) noexcept -{ return __lhs / __rhs.month() / __rhs.day(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year_month& __lhs, int __rhs) noexcept { + return __lhs / day(__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(int __lhs, const month_day& __rhs) noexcept -{ return year(__lhs) / __rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const year& __lhs, const month_day& __rhs) noexcept { + return __lhs / __rhs.month() / __rhs.day(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(const month_day& __lhs, const year& __rhs) noexcept -{ return __rhs / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(int __lhs, const month_day& __rhs) noexcept { + return year(__lhs) / __rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator/(const month_day& __lhs, int __rhs) noexcept -{ return year(__rhs) / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const month_day& __lhs, const year& __rhs) noexcept { + return __rhs / __lhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day operator/(const month_day& __lhs, int __rhs) noexcept { + return year(__rhs) / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator+(const year_month_day& __lhs, const months& __rhs) noexcept -{ return (__lhs.year()/__lhs.month() + __rhs)/__lhs.day(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator+(const year_month_day& __lhs, const months& __rhs) noexcept { + return (__lhs.year() / __lhs.month() + __rhs) / __lhs.day(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator+(const months& __lhs, const year_month_day& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator+(const months& __lhs, const year_month_day& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator-(const year_month_day& __lhs, const months& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator-(const year_month_day& __lhs, const months& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator+(const year_month_day& __lhs, const years& __rhs) noexcept -{ return (__lhs.year() + __rhs) / __lhs.month() / __lhs.day(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator+(const year_month_day& __lhs, const years& __rhs) noexcept { + return (__lhs.year() + __rhs) / __lhs.month() / __lhs.day(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator+(const years& __lhs, const year_month_day& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator+(const years& __lhs, const year_month_day& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day operator-(const year_month_day& __lhs, const years& __rhs) noexcept -{ return __lhs + -__rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day +operator-(const year_month_day& __lhs, const years& __rhs) noexcept { + return __lhs + -__rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const months& __dm) noexcept { + *this = *this + __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const months& __dm) noexcept { + *this = *this - __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator+=(const years& __dy) noexcept { + *this = *this + __dy; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day& year_month_day::operator-=(const years& __dy) noexcept { + *this = *this - __dy; + return *this; +} class year_month_day_last { private: - chrono::year __y_; - chrono::month_day_last __mdl_; + chrono::year __y_; + chrono::month_day_last __mdl_; + public: - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last(const year& __yval, const month_day_last& __mdlval) noexcept - : __y_{__yval}, __mdl_{__mdlval} {} - - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const months& __m) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const months& __m) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const years& __y) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const years& __y) noexcept; - - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __mdl_.month(); } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month_day_last month_day_last() const noexcept { return __mdl_; } - _LIBCPP_HIDE_FROM_ABI constexpr chrono::day day() const noexcept; - _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{year()/month()/day()}; } - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { return local_days{year()/month()/day()}; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __mdl_.ok(); } + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last(const year& __yval, const month_day_last& __mdlval) noexcept + : __y_{__yval}, __mdl_{__mdlval} {} + + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const months& __m) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const months& __m) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator+=(const years& __y) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_day_last& operator-=(const years& __y) noexcept; + + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __mdl_.month(); } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month_day_last month_day_last() const noexcept { return __mdl_; } + _LIBCPP_HIDE_FROM_ABI constexpr chrono::day day() const noexcept; + _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { + return sys_days{year() / month() / day()}; + } + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { + return local_days{year() / month() / day()}; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __mdl_.ok(); } }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -chrono::day year_month_day_last::day() const noexcept -{ - constexpr chrono::day __d[] = - { - chrono::day(31), chrono::day(28), chrono::day(31), - chrono::day(30), chrono::day(31), chrono::day(30), - chrono::day(31), chrono::day(31), chrono::day(30), - chrono::day(31), chrono::day(30), chrono::day(31) - }; - return (month() != February || !__y_.is_leap()) && month().ok() ? - __d[static_cast(month()) - 1] : chrono::day{29}; +_LIBCPP_HIDE_FROM_ABI inline constexpr chrono::day year_month_day_last::day() const noexcept { + constexpr chrono::day __d[] = { + chrono::day(31), + chrono::day(28), + chrono::day(31), + chrono::day(30), + chrono::day(31), + chrono::day(30), + chrono::day(31), + chrono::day(31), + chrono::day(30), + chrono::day(31), + chrono::day(30), + chrono::day(31)}; + return (month() != February || !__y_.is_leap()) && month().ok() + ? __d[static_cast(month()) - 1] + : chrono::day{29}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ return __lhs.year() == __rhs.year() && __lhs.month_day_last() == __rhs.month_day_last(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + return __lhs.year() == __rhs.year() && __lhs.month_day_last() == __rhs.month_day_last(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator!=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ return !(__lhs == __rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator!=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + return !(__lhs == __rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator< (const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ - if (__lhs.year() < __rhs.year()) return true; - if (__lhs.year() > __rhs.year()) return false; - return __lhs.month_day_last() < __rhs.month_day_last(); +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator<(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + if (__lhs.year() < __rhs.year()) + return true; + if (__lhs.year() > __rhs.year()) + return false; + return __lhs.month_day_last() < __rhs.month_day_last(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator> (const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ return __rhs < __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator>(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + return __rhs < __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator<=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ return !(__rhs < __lhs);} +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator<=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + return !(__rhs < __lhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator>=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept -{ return !(__lhs < __rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator>=(const year_month_day_last& __lhs, const year_month_day_last& __rhs) noexcept { + return !(__lhs < __rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator/(const year_month& __lhs, last_spec) noexcept -{ return year_month_day_last{__lhs.year(), month_day_last{__lhs.month()}}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(const year_month& __lhs, last_spec) noexcept { + return year_month_day_last{__lhs.year(), month_day_last{__lhs.month()}}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator/(const year& __lhs, const month_day_last& __rhs) noexcept -{ return year_month_day_last{__lhs, __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator/(const year& __lhs, const month_day_last& __rhs) noexcept { + return year_month_day_last{__lhs, __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator/(int __lhs, const month_day_last& __rhs) noexcept -{ return year_month_day_last{year{__lhs}, __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(int __lhs, const month_day_last& __rhs) noexcept { + return year_month_day_last{year{__lhs}, __rhs}; +} _LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last -operator/(const month_day_last& __lhs, const year& __rhs) noexcept -{ return __rhs / __lhs; } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator/(const month_day_last& __lhs, int __rhs) noexcept -{ return year{__rhs} / __lhs; } +operator/(const month_day_last& __lhs, const year& __rhs) noexcept { + return __rhs / __lhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last operator/(const month_day_last& __lhs, int __rhs) noexcept { + return year{__rhs} / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator+(const year_month_day_last& __lhs, const months& __rhs) noexcept -{ return (__lhs.year() / __lhs.month() + __rhs) / last; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator+(const year_month_day_last& __lhs, const months& __rhs) noexcept { + return (__lhs.year() / __lhs.month() + __rhs) / last; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator+(const months& __lhs, const year_month_day_last& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator+(const months& __lhs, const year_month_day_last& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator-(const year_month_day_last& __lhs, const months& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator-(const year_month_day_last& __lhs, const months& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator+(const year_month_day_last& __lhs, const years& __rhs) noexcept -{ return year_month_day_last{__lhs.year() + __rhs, __lhs.month_day_last()}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator+(const year_month_day_last& __lhs, const years& __rhs) noexcept { + return year_month_day_last{__lhs.year() + __rhs, __lhs.month_day_last()}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator+(const years& __lhs, const year_month_day_last& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator+(const years& __lhs, const year_month_day_last& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day_last operator-(const year_month_day_last& __lhs, const years& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last +operator-(const year_month_day_last& __lhs, const years& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& year_month_day_last::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& year_month_day_last::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& year_month_day_last::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& year_month_day_last::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& +year_month_day_last::operator+=(const months& __dm) noexcept { + *this = *this + __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& +year_month_day_last::operator-=(const months& __dm) noexcept { + *this = *this - __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& +year_month_day_last::operator+=(const years& __dy) noexcept { + *this = *this + __dy; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day_last& +year_month_day_last::operator-=(const years& __dy) noexcept { + *this = *this - __dy; + return *this; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_day::year_month_day(const year_month_day_last& __ymdl) noexcept +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_day::year_month_day(const year_month_day_last& __ymdl) noexcept : __y_{__ymdl.year()}, __m_{__ymdl.month()}, __d_{__ymdl.day()} {} -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool year_month_day::ok() const noexcept -{ - if (!__y_.ok() || !__m_.ok()) return false; - return chrono::day{1} <= __d_ && __d_ <= (__y_ / __m_ / last).day(); +_LIBCPP_HIDE_FROM_ABI inline constexpr bool year_month_day::ok() const noexcept { + if (!__y_.ok() || !__m_.ok()) + return false; + return chrono::day{1} <= __d_ && __d_ <= (__y_ / __m_ / last).day(); } } // namespace chrono diff --git a/libcxx/include/__chrono/year_month_weekday.h b/libcxx/include/__chrono/year_month_weekday.h index 4b5cb492a1914..0c3dd494c8787 100644 --- a/libcxx/include/__chrono/year_month_weekday.h +++ b/libcxx/include/__chrono/year_month_weekday.h @@ -31,211 +31,252 @@ _LIBCPP_BEGIN_NAMESPACE_STD -namespace chrono -{ +namespace chrono { class year_month_weekday { - chrono::year __y_; - chrono::month __m_; - chrono::weekday_indexed __wdi_; + chrono::year __y_; + chrono::month __m_; + chrono::weekday_indexed __wdi_; + public: - year_month_weekday() = default; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday(const chrono::year& __yval, const chrono::month& __mval, - const chrono::weekday_indexed& __wdival) noexcept - : __y_{__yval}, __m_{__mval}, __wdi_{__wdival} {} - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday(const sys_days& __sysd) noexcept - : year_month_weekday(__from_days(__sysd.time_since_epoch())) {} - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_weekday(const local_days& __locd) noexcept - : year_month_weekday(__from_days(__locd.time_since_epoch())) {} - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const months&) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const months&) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const years&) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const years&) noexcept; - - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdi_.weekday(); } - _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __wdi_.index(); } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; } - - _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept - { - if (!__y_.ok() || !__m_.ok() || !__wdi_.ok()) return false; - if (__wdi_.index() <= 4) return true; - auto __nth_weekday_day = - __wdi_.weekday() - - chrono::weekday{static_cast(__y_ / __m_ / 1)} + - days{(__wdi_.index() - 1) * 7 + 1}; - return static_cast(__nth_weekday_day.count()) <= - static_cast((__y_ / __m_ / last).day()); - } - - _LIBCPP_HIDE_FROM_ABI static constexpr year_month_weekday __from_days(days __d) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; + year_month_weekday() = default; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday( + const chrono::year& __yval, const chrono::month& __mval, const chrono::weekday_indexed& __wdival) noexcept + : __y_{__yval}, __m_{__mval}, __wdi_{__wdival} {} + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday(const sys_days& __sysd) noexcept + : year_month_weekday(__from_days(__sysd.time_since_epoch())) {} + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr year_month_weekday(const local_days& __locd) noexcept + : year_month_weekday(__from_days(__locd.time_since_epoch())) {} + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const months&) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const months&) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator+=(const years&) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday& operator-=(const years&) noexcept; + + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdi_.weekday(); } + _LIBCPP_HIDE_FROM_ABI inline constexpr unsigned index() const noexcept { return __wdi_.index(); } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_indexed weekday_indexed() const noexcept { return __wdi_; } + + _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { + return local_days{__to_days()}; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { + if (!__y_.ok() || !__m_.ok() || !__wdi_.ok()) + return false; + if (__wdi_.index() <= 4) + return true; + auto __nth_weekday_day = + __wdi_.weekday() - chrono::weekday{static_cast(__y_ / __m_ / 1)} + days{(__wdi_.index() - 1) * 7 + 1}; + return static_cast(__nth_weekday_day.count()) <= static_cast((__y_ / __m_ / last).day()); + } + + _LIBCPP_HIDE_FROM_ABI static constexpr year_month_weekday __from_days(days __d) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday year_month_weekday::__from_days(days __d) noexcept -{ - const sys_days __sysd{__d}; - const chrono::weekday __wd = chrono::weekday(__sysd); - const year_month_day __ymd = year_month_day(__sysd); - return year_month_weekday{__ymd.year(), __ymd.month(), - __wd[(static_cast(__ymd.day())-1)/7+1]}; +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday year_month_weekday::__from_days(days __d) noexcept { + const sys_days __sysd{__d}; + const chrono::weekday __wd = chrono::weekday(__sysd); + const year_month_day __ymd = year_month_day(__sysd); + return year_month_weekday{__ymd.year(), __ymd.month(), __wd[(static_cast(__ymd.day()) - 1) / 7 + 1]}; } -_LIBCPP_HIDE_FROM_ABI inline constexpr -days year_month_weekday::__to_days() const noexcept -{ - const sys_days __sysd = sys_days(__y_/__m_/1); - return (__sysd + (__wdi_.weekday() - chrono::weekday(__sysd) + days{(__wdi_.index()-1)*7})) - .time_since_epoch(); +_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_weekday::__to_days() const noexcept { + const sys_days __sysd = sys_days(__y_ / __m_ / 1); + return (__sysd + (__wdi_.weekday() - chrono::weekday(__sysd) + days{(__wdi_.index() - 1) * 7})).time_since_epoch(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year_month_weekday& __lhs, const year_month_weekday& __rhs) noexcept -{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_indexed() == __rhs.weekday_indexed(); } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator/(const year_month& __lhs, const weekday_indexed& __rhs) noexcept -{ return year_month_weekday{__lhs.year(), __lhs.month(), __rhs}; } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator/(const year& __lhs, const month_weekday& __rhs) noexcept -{ return year_month_weekday{__lhs, __rhs.month(), __rhs.weekday_indexed()}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const year_month_weekday& __lhs, const year_month_weekday& __rhs) noexcept { + return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && + __lhs.weekday_indexed() == __rhs.weekday_indexed(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator/(int __lhs, const month_weekday& __rhs) noexcept -{ return year(__lhs) / __rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator/(const year_month& __lhs, const weekday_indexed& __rhs) noexcept { + return year_month_weekday{__lhs.year(), __lhs.month(), __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator/(const month_weekday& __lhs, const year& __rhs) noexcept -{ return __rhs / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator/(const year& __lhs, const month_weekday& __rhs) noexcept { + return year_month_weekday{__lhs, __rhs.month(), __rhs.weekday_indexed()}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator/(const month_weekday& __lhs, int __rhs) noexcept -{ return year(__rhs) / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday operator/(int __lhs, const month_weekday& __rhs) noexcept { + return year(__lhs) / __rhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator/(const month_weekday& __lhs, const year& __rhs) noexcept { + return __rhs / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator+(const year_month_weekday& __lhs, const months& __rhs) noexcept -{ return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_indexed(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday operator/(const month_weekday& __lhs, int __rhs) noexcept { + return year(__rhs) / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator+(const months& __lhs, const year_month_weekday& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator+(const year_month_weekday& __lhs, const months& __rhs) noexcept { + return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_indexed(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator-(const year_month_weekday& __lhs, const months& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator+(const months& __lhs, const year_month_weekday& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator+(const year_month_weekday& __lhs, const years& __rhs) noexcept -{ return year_month_weekday{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_indexed()}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator-(const year_month_weekday& __lhs, const months& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator+(const years& __lhs, const year_month_weekday& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator+(const year_month_weekday& __lhs, const years& __rhs) noexcept { + return year_month_weekday{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_indexed()}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday operator-(const year_month_weekday& __lhs, const years& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator+(const years& __lhs, const year_month_weekday& __rhs) noexcept { + return __rhs + __lhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday +operator-(const year_month_weekday& __lhs, const years& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const months& __dm) noexcept { + *this = *this + __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const months& __dm) noexcept { + *this = *this - __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator+=(const years& __dy) noexcept { + *this = *this + __dy; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday& year_month_weekday::operator-=(const years& __dy) noexcept { + *this = *this - __dy; + return *this; +} class year_month_weekday_last { private: - chrono::year __y_; - chrono::month __m_; - chrono::weekday_last __wdl_; -public: - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last(const chrono::year& __yval, const chrono::month& __mval, - const chrono::weekday_last& __wdlval) noexcept - : __y_{__yval}, __m_{__mval}, __wdl_{__wdlval} {} - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const months& __dm) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const years& __dy) noexcept; - _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const years& __dy) noexcept; - - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdl_.weekday(); } - _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; } - _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } - _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { return local_days{__to_days()}; } - _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok() && __wdl_.ok(); } - - _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; + chrono::year __y_; + chrono::month __m_; + chrono::weekday_last __wdl_; +public: + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last( + const chrono::year& __yval, const chrono::month& __mval, const chrono::weekday_last& __wdlval) noexcept + : __y_{__yval}, __m_{__mval}, __wdl_{__wdlval} {} + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const months& __dm) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator+=(const years& __dy) noexcept; + _LIBCPP_HIDE_FROM_ABI constexpr year_month_weekday_last& operator-=(const years& __dy) noexcept; + + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::year year() const noexcept { return __y_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::month month() const noexcept { return __m_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday weekday() const noexcept { return __wdl_.weekday(); } + _LIBCPP_HIDE_FROM_ABI inline constexpr chrono::weekday_last weekday_last() const noexcept { return __wdl_; } + _LIBCPP_HIDE_FROM_ABI inline constexpr operator sys_days() const noexcept { return sys_days{__to_days()}; } + _LIBCPP_HIDE_FROM_ABI inline explicit constexpr operator local_days() const noexcept { + return local_days{__to_days()}; + } + _LIBCPP_HIDE_FROM_ABI inline constexpr bool ok() const noexcept { return __y_.ok() && __m_.ok() && __wdl_.ok(); } + + _LIBCPP_HIDE_FROM_ABI constexpr days __to_days() const noexcept; }; -_LIBCPP_HIDE_FROM_ABI inline constexpr -days year_month_weekday_last::__to_days() const noexcept -{ - const sys_days __last = sys_days{__y_/__m_/last}; - return (__last - (chrono::weekday{__last} - __wdl_.weekday())).time_since_epoch(); - +_LIBCPP_HIDE_FROM_ABI inline constexpr days year_month_weekday_last::__to_days() const noexcept { + const sys_days __last = sys_days{__y_ / __m_ / last}; + return (__last - (chrono::weekday{__last} - __wdl_.weekday())).time_since_epoch(); } -_LIBCPP_HIDE_FROM_ABI inline constexpr -bool operator==(const year_month_weekday_last& __lhs, const year_month_weekday_last& __rhs) noexcept -{ return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); } - -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator/(const year_month& __lhs, const weekday_last& __rhs) noexcept -{ return year_month_weekday_last{__lhs.year(), __lhs.month(), __rhs}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr bool +operator==(const year_month_weekday_last& __lhs, const year_month_weekday_last& __rhs) noexcept { + return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.weekday_last() == __rhs.weekday_last(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator/(const year& __lhs, const month_weekday_last& __rhs) noexcept -{ return year_month_weekday_last{__lhs, __rhs.month(), __rhs.weekday_last()}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator/(const year_month& __lhs, const weekday_last& __rhs) noexcept { + return year_month_weekday_last{__lhs.year(), __lhs.month(), __rhs}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator/(int __lhs, const month_weekday_last& __rhs) noexcept -{ return year(__lhs) / __rhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator/(const year& __lhs, const month_weekday_last& __rhs) noexcept { + return year_month_weekday_last{__lhs, __rhs.month(), __rhs.weekday_last()}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator/(const month_weekday_last& __lhs, const year& __rhs) noexcept -{ return __rhs / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator/(int __lhs, const month_weekday_last& __rhs) noexcept { + return year(__lhs) / __rhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator/(const month_weekday_last& __lhs, int __rhs) noexcept -{ return year(__rhs) / __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator/(const month_weekday_last& __lhs, const year& __rhs) noexcept { + return __rhs / __lhs; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator/(const month_weekday_last& __lhs, int __rhs) noexcept { + return year(__rhs) / __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator+(const year_month_weekday_last& __lhs, const months& __rhs) noexcept -{ return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_last(); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator+(const year_month_weekday_last& __lhs, const months& __rhs) noexcept { + return (__lhs.year() / __lhs.month() + __rhs) / __lhs.weekday_last(); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator+(const months& __lhs, const year_month_weekday_last& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator+(const months& __lhs, const year_month_weekday_last& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator-(const year_month_weekday_last& __lhs, const months& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator-(const year_month_weekday_last& __lhs, const months& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator+(const year_month_weekday_last& __lhs, const years& __rhs) noexcept -{ return year_month_weekday_last{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_last()}; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator+(const year_month_weekday_last& __lhs, const years& __rhs) noexcept { + return year_month_weekday_last{__lhs.year() + __rhs, __lhs.month(), __lhs.weekday_last()}; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator+(const years& __lhs, const year_month_weekday_last& __rhs) noexcept -{ return __rhs + __lhs; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator+(const years& __lhs, const year_month_weekday_last& __rhs) noexcept { + return __rhs + __lhs; +} -_LIBCPP_HIDE_FROM_ABI inline constexpr -year_month_weekday_last operator-(const year_month_weekday_last& __lhs, const years& __rhs) noexcept -{ return __lhs + (-__rhs); } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last +operator-(const year_month_weekday_last& __lhs, const years& __rhs) noexcept { + return __lhs + (-__rhs); +} -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& year_month_weekday_last::operator+=(const months& __dm) noexcept { *this = *this + __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& year_month_weekday_last::operator-=(const months& __dm) noexcept { *this = *this - __dm; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& year_month_weekday_last::operator+=(const years& __dy) noexcept { *this = *this + __dy; return *this; } -_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& year_month_weekday_last::operator-=(const years& __dy) noexcept { *this = *this - __dy; return *this; } +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& +year_month_weekday_last::operator+=(const months& __dm) noexcept { + *this = *this + __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& +year_month_weekday_last::operator-=(const months& __dm) noexcept { + *this = *this - __dm; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& +year_month_weekday_last::operator+=(const years& __dy) noexcept { + *this = *this + __dy; + return *this; +} +_LIBCPP_HIDE_FROM_ABI inline constexpr year_month_weekday_last& +year_month_weekday_last::operator-=(const years& __dy) noexcept { + *this = *this - __dy; + return *this; +} } // namespace chrono diff --git a/libcxx/include/__compare/common_comparison_category.h b/libcxx/include/__compare/common_comparison_category.h index 5fad99bf5ce0a..7aeb3da03a4f4 100644 --- a/libcxx/include/__compare/common_comparison_category.h +++ b/libcxx/include/__compare/common_comparison_category.h @@ -24,17 +24,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace __comp_detail { -enum _ClassifyCompCategory : unsigned { - _None, - _PartialOrd, - _WeakOrd, - _StrongOrd, - _CCC_Size -}; +enum _ClassifyCompCategory : unsigned { _None, _PartialOrd, _WeakOrd, _StrongOrd, _CCC_Size }; template -_LIBCPP_HIDE_FROM_ABI -constexpr _ClassifyCompCategory __type_to_enum() noexcept { +_LIBCPP_HIDE_FROM_ABI constexpr _ClassifyCompCategory __type_to_enum() noexcept { if (is_same_v<_Tp, partial_ordering>) return _PartialOrd; if (is_same_v<_Tp, weak_ordering>) @@ -45,8 +38,7 @@ constexpr _ClassifyCompCategory __type_to_enum() noexcept { } template -_LIBCPP_HIDE_FROM_ABI -constexpr _ClassifyCompCategory +_LIBCPP_HIDE_FROM_ABI constexpr _ClassifyCompCategory __compute_comp_type(const _ClassifyCompCategory (&__types)[_Size]) { int __seen[_CCC_Size] = {}; for (auto __type : __types) @@ -60,12 +52,11 @@ __compute_comp_type(const _ClassifyCompCategory (&__types)[_Size]) { return _StrongOrd; } -template -_LIBCPP_HIDE_FROM_ABI -constexpr auto __get_comp_type() { - using _CCC = _ClassifyCompCategory; +template +_LIBCPP_HIDE_FROM_ABI constexpr auto __get_comp_type() { + using _CCC = _ClassifyCompCategory; constexpr _CCC __type_kinds[] = {_StrongOrd, __type_to_enum<_Ts>()...}; - constexpr _CCC __cat = __comp_detail::__compute_comp_type(__type_kinds); + constexpr _CCC __cat = __comp_detail::__compute_comp_type(__type_kinds); if constexpr (__cat == _None) return void(); else if constexpr (__cat == _PartialOrd) @@ -80,12 +71,12 @@ constexpr auto __get_comp_type() { } // namespace __comp_detail // [cmp.common], common comparison category type -template +template struct _LIBCPP_TEMPLATE_VIS common_comparison_category { using type = decltype(__comp_detail::__get_comp_type<_Ts...>()); }; -template +template using common_comparison_category_t = typename common_comparison_category<_Ts...>::type; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/compare_partial_order_fallback.h b/libcxx/include/__compare/compare_partial_order_fallback.h index a49408e1cfda9..e0efa3ccb88db 100644 --- a/libcxx/include/__compare/compare_partial_order_fallback.h +++ b/libcxx/include/__compare/compare_partial_order_fallback.h @@ -27,44 +27,46 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __compare_partial_order_fallback { - struct __fn { - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) - noexcept(noexcept(std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - -> decltype( std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) - { return std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); } +struct __fn { + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept( + noexcept(std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) + -> decltype(std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) { + return std::partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less : - std::forward<_Up>(__u) < std::forward<_Tp>(__t) ? partial_ordering::greater : - partial_ordering::unordered)) - -> decltype( std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less : - std::forward<_Up>(__u) < std::forward<_Tp>(__t) ? partial_ordering::greater : - partial_ordering::unordered) - { - return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less : - std::forward<_Up>(__u) < std::forward<_Tp>(__t) ? partial_ordering::greater : - partial_ordering::unordered; - } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept( + std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less + : std::forward<_Up>(__u) < std::forward<_Tp>(__t) + ? partial_ordering::greater + : partial_ordering::unordered)) + -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less + : std::forward<_Up>(__u) < std::forward<_Tp>(__t) + ? partial_ordering::greater + : partial_ordering::unordered) { + return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less + : std::forward<_Up>(__u) < std::forward<_Tp>(__t) + ? partial_ordering::greater + : partial_ordering::unordered; + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); + } +}; } // namespace __compare_partial_order_fallback inline namespace __cpo { - inline constexpr auto compare_partial_order_fallback = __compare_partial_order_fallback::__fn{}; +inline constexpr auto compare_partial_order_fallback = __compare_partial_order_fallback::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/compare_strong_order_fallback.h b/libcxx/include/__compare/compare_strong_order_fallback.h index 9e2dfee6b389f..a94d517ed30fc 100644 --- a/libcxx/include/__compare/compare_strong_order_fallback.h +++ b/libcxx/include/__compare/compare_strong_order_fallback.h @@ -27,41 +27,43 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __compare_strong_order_fallback { - struct __fn { - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) - noexcept(noexcept(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - -> decltype( std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) - { return std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); } +struct __fn { + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept( + noexcept(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) + -> decltype(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) { + return std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? strong_ordering::less : - strong_ordering::greater)) - -> decltype( std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? strong_ordering::less : - strong_ordering::greater) - { - return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? strong_ordering::less : - strong_ordering::greater; - } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept( + std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? strong_ordering::less + : strong_ordering::greater)) + -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? strong_ordering::less + : strong_ordering::greater) { + return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? strong_ordering::less + : strong_ordering::greater; + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); + } +}; } // namespace __compare_strong_order_fallback inline namespace __cpo { - inline constexpr auto compare_strong_order_fallback = __compare_strong_order_fallback::__fn{}; +inline constexpr auto compare_strong_order_fallback = __compare_strong_order_fallback::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/compare_three_way.h b/libcxx/include/__compare/compare_three_way.h index b1ed2d1e7251c..01c12076c0d73 100644 --- a/libcxx/include/__compare/compare_three_way.h +++ b/libcxx/include/__compare/compare_three_way.h @@ -22,16 +22,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -struct _LIBCPP_TEMPLATE_VIS compare_three_way -{ - template - requires three_way_comparable_with<_T1, _T2> - constexpr _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) <=> std::forward<_T2>(__u))) - { return std::forward<_T1>(__t) <=> std::forward<_T2>(__u); } - - using is_transparent = void; +struct _LIBCPP_TEMPLATE_VIS compare_three_way { + template + requires three_way_comparable_with<_T1, _T2> + constexpr _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) <=> std::forward<_T2>(__u))) { + return std::forward<_T1>(__t) <=> std::forward<_T2>(__u); + } + + using is_transparent = void; }; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/compare_three_way_result.h b/libcxx/include/__compare/compare_three_way_result.h index 632ebdce1bce4..d7508073433af 100644 --- a/libcxx/include/__compare/compare_three_way_result.h +++ b/libcxx/include/__compare/compare_three_way_result.h @@ -21,20 +21,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template -struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result { }; - -template -struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result<_Tp, _Up, decltype( - std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>(), void() -)> { - using type = decltype(std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>()); +template +struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result {}; + +template +struct _LIBCPP_HIDE_FROM_ABI __compare_three_way_result< + _Tp, + _Up, + decltype(std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>(), void())> { + using type = decltype(std::declval<__make_const_lvalue_ref<_Tp>>() <=> std::declval<__make_const_lvalue_ref<_Up>>()); }; -template -struct _LIBCPP_TEMPLATE_VIS compare_three_way_result : __compare_three_way_result<_Tp, _Up, void> { }; +template +struct _LIBCPP_TEMPLATE_VIS compare_three_way_result : __compare_three_way_result<_Tp, _Up, void> {}; -template +template using compare_three_way_result_t = typename compare_three_way_result<_Tp, _Up>::type; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/compare_weak_order_fallback.h b/libcxx/include/__compare/compare_weak_order_fallback.h index b0a04abcaaf03..062b7b582cd7e 100644 --- a/libcxx/include/__compare/compare_weak_order_fallback.h +++ b/libcxx/include/__compare/compare_weak_order_fallback.h @@ -27,41 +27,43 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __compare_weak_order_fallback { - struct __fn { - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) - noexcept(noexcept(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - -> decltype( std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) - { return std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); } +struct __fn { + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept( + noexcept(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) + -> decltype(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))) { + return std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)); + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? weak_ordering::less : - weak_ordering::greater)) - -> decltype( std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? weak_ordering::less : - weak_ordering::greater) - { - return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent : - std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? weak_ordering::less : - weak_ordering::greater; - } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept( + std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? weak_ordering::less + : weak_ordering::greater)) + -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? weak_ordering::less + : weak_ordering::greater) { + return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent + : std::forward<_Tp>(__t) < std::forward<_Up>(__u) + ? weak_ordering::less + : weak_ordering::greater; + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<1>()); + } +}; } // namespace __compare_weak_order_fallback inline namespace __cpo { - inline constexpr auto compare_weak_order_fallback = __compare_weak_order_fallback::__fn{}; +inline constexpr auto compare_weak_order_fallback = __compare_weak_order_fallback::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/ordering.h b/libcxx/include/__compare/ordering.h index c9a15efb3c2fc..2995d381304f0 100644 --- a/libcxx/include/__compare/ordering.h +++ b/libcxx/include/__compare/ordering.h @@ -22,46 +22,35 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 // exposition only -enum class _OrdResult : signed char { - __less = -1, - __equiv = 0, - __greater = 1 -}; +enum class _OrdResult : signed char { __less = -1, __equiv = 0, __greater = 1 }; -enum class _NCmpResult : signed char { - __unordered = -127 -}; +enum class _NCmpResult : signed char { __unordered = -127 }; class partial_ordering; class weak_ordering; class strong_ordering; -template +template inline constexpr bool __one_of_v = (is_same_v<_Tp, _Args> || ...); struct _CmpUnspecifiedParam { - _LIBCPP_HIDE_FROM_ABI constexpr - _CmpUnspecifiedParam(int _CmpUnspecifiedParam::*) noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr _CmpUnspecifiedParam(int _CmpUnspecifiedParam::*) noexcept {} - template>> + template >> _CmpUnspecifiedParam(_Tp) = delete; }; class partial_ordering { using _ValueT = signed char; - _LIBCPP_HIDE_FROM_ABI - explicit constexpr partial_ordering(_OrdResult __v) noexcept - : __value_(_ValueT(__v)) {} + _LIBCPP_HIDE_FROM_ABI explicit constexpr partial_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {} - _LIBCPP_HIDE_FROM_ABI - explicit constexpr partial_ordering(_NCmpResult __v) noexcept - : __value_(_ValueT(__v)) {} + _LIBCPP_HIDE_FROM_ABI explicit constexpr partial_ordering(_NCmpResult __v) noexcept : __value_(_ValueT(__v)) {} - _LIBCPP_HIDE_FROM_ABI - constexpr bool __is_ordered() const noexcept { + _LIBCPP_HIDE_FROM_ABI constexpr bool __is_ordered() const noexcept { return __value_ != _ValueT(_NCmpResult::__unordered); } + public: // valid values static const partial_ordering less; @@ -70,63 +59,54 @@ class partial_ordering { static const partial_ordering unordered; // comparisons - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(partial_ordering, partial_ordering) noexcept = default; + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(partial_ordering, partial_ordering) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__is_ordered() && __v.__value_ == 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__is_ordered() && __v.__value_ < 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__is_ordered() && __v.__value_ <= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__is_ordered() && __v.__value_ > 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__is_ordered() && __v.__value_ >= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (_CmpUnspecifiedParam, partial_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, partial_ordering __v) noexcept { return __v.__is_ordered() && 0 < __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(_CmpUnspecifiedParam, partial_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, partial_ordering __v) noexcept { return __v.__is_ordered() && 0 <= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (_CmpUnspecifiedParam, partial_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, partial_ordering __v) noexcept { return __v.__is_ordered() && 0 > __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(_CmpUnspecifiedParam, partial_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, partial_ordering __v) noexcept { return __v.__is_ordered() && 0 >= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr partial_ordering operator<=>(partial_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr partial_ordering + operator<=>(partial_ordering __v, _CmpUnspecifiedParam) noexcept { return __v; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr partial_ordering operator<=>(_CmpUnspecifiedParam, partial_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr partial_ordering + operator<=>(_CmpUnspecifiedParam, partial_ordering __v) noexcept { return __v < 0 ? partial_ordering::greater : (__v > 0 ? partial_ordering::less : __v); } + private: _ValueT __value_; }; @@ -139,76 +119,62 @@ inline constexpr partial_ordering partial_ordering::unordered(_NCmpResult ::__un class weak_ordering { using _ValueT = signed char; - _LIBCPP_HIDE_FROM_ABI - explicit constexpr weak_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {} + _LIBCPP_HIDE_FROM_ABI explicit constexpr weak_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {} public: static const weak_ordering less; static const weak_ordering equivalent; static const weak_ordering greater; - _LIBCPP_HIDE_FROM_ABI - constexpr operator partial_ordering() const noexcept { + _LIBCPP_HIDE_FROM_ABI constexpr operator partial_ordering() const noexcept { return __value_ == 0 ? partial_ordering::equivalent - : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater); + : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater); } // comparisons - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(weak_ordering, weak_ordering) noexcept = default; + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(weak_ordering, weak_ordering) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ == 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ < 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ <= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ > 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ >= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (_CmpUnspecifiedParam, weak_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, weak_ordering __v) noexcept { return 0 < __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(_CmpUnspecifiedParam, weak_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, weak_ordering __v) noexcept { return 0 <= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (_CmpUnspecifiedParam, weak_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, weak_ordering __v) noexcept { return 0 > __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(_CmpUnspecifiedParam, weak_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, weak_ordering __v) noexcept { return 0 >= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr weak_ordering operator<=>(weak_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr weak_ordering operator<=>(weak_ordering __v, _CmpUnspecifiedParam) noexcept { return __v; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr weak_ordering operator<=>(_CmpUnspecifiedParam, weak_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr weak_ordering operator<=>(_CmpUnspecifiedParam, weak_ordering __v) noexcept { return __v < 0 ? weak_ordering::greater : (__v > 0 ? weak_ordering::less : __v); } @@ -223,8 +189,7 @@ inline constexpr weak_ordering weak_ordering::greater(_OrdResult::__greater); class strong_ordering { using _ValueT = signed char; - _LIBCPP_HIDE_FROM_ABI - explicit constexpr strong_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {} + _LIBCPP_HIDE_FROM_ABI explicit constexpr strong_ordering(_OrdResult __v) noexcept : __value_(_ValueT(__v)) {} public: static const strong_ordering less; @@ -233,74 +198,61 @@ class strong_ordering { static const strong_ordering greater; // conversions - _LIBCPP_HIDE_FROM_ABI - constexpr operator partial_ordering() const noexcept { + _LIBCPP_HIDE_FROM_ABI constexpr operator partial_ordering() const noexcept { return __value_ == 0 ? partial_ordering::equivalent - : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater); + : (__value_ < 0 ? partial_ordering::less : partial_ordering::greater); } - _LIBCPP_HIDE_FROM_ABI - constexpr operator weak_ordering() const noexcept { - return __value_ == 0 ? weak_ordering::equivalent - : (__value_ < 0 ? weak_ordering::less : weak_ordering::greater); + _LIBCPP_HIDE_FROM_ABI constexpr operator weak_ordering() const noexcept { + return __value_ == 0 ? weak_ordering::equivalent : (__value_ < 0 ? weak_ordering::less : weak_ordering::greater); } // comparisons - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(strong_ordering, strong_ordering) noexcept = default; + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(strong_ordering, strong_ordering) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ == 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ < 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ <= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ > 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v.__value_ >= 0; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator< (_CmpUnspecifiedParam, strong_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(_CmpUnspecifiedParam, strong_ordering __v) noexcept { return 0 < __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator<=(_CmpUnspecifiedParam, strong_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(_CmpUnspecifiedParam, strong_ordering __v) noexcept { return 0 <= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator> (_CmpUnspecifiedParam, strong_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(_CmpUnspecifiedParam, strong_ordering __v) noexcept { return 0 > __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator>=(_CmpUnspecifiedParam, strong_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(_CmpUnspecifiedParam, strong_ordering __v) noexcept { return 0 >= __v.__value_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr strong_ordering operator<=>(strong_ordering __v, _CmpUnspecifiedParam) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering + operator<=>(strong_ordering __v, _CmpUnspecifiedParam) noexcept { return __v; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr strong_ordering operator<=>(_CmpUnspecifiedParam, strong_ordering __v) noexcept { + _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering + operator<=>(_CmpUnspecifiedParam, strong_ordering __v) noexcept { return __v < 0 ? strong_ordering::greater : (__v > 0 ? strong_ordering::less : __v); } diff --git a/libcxx/include/__compare/partial_order.h b/libcxx/include/__compare/partial_order.h index 36a11dfaa2881..f3ed4900fbff2 100644 --- a/libcxx/include/__compare/partial_order.h +++ b/libcxx/include/__compare/partial_order.h @@ -28,43 +28,44 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __partial_order { - struct __fn { - // NOLINTBEGIN(libcpp-robust-against-adl) partial_order should use ADL, but only here - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) - noexcept(noexcept(partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } - // NOLINTEND(libcpp-robust-against-adl) +struct __fn { + // NOLINTBEGIN(libcpp-robust-against-adl) partial_order should use ADL, but only here + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept( + noexcept(partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return partial_ordering(partial_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } + // NOLINTEND(libcpp-robust-against-adl) - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) - noexcept(noexcept(partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept( + noexcept(partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return partial_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept( + noexcept(partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return partial_ordering(std::weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()); + } +}; } // namespace __partial_order inline namespace __cpo { - inline constexpr auto partial_order = __partial_order::__fn{}; +inline constexpr auto partial_order = __partial_order::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/strong_order.h b/libcxx/include/__compare/strong_order.h index cbfcf7316de9e..5f6ade5aef8e4 100644 --- a/libcxx/include/__compare/strong_order.h +++ b/libcxx/include/__compare/strong_order.h @@ -34,100 +34,99 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __strong_order { - struct __fn { - // NOLINTBEGIN(libcpp-robust-against-adl) strong_order should use ADL, but only here - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) - noexcept(noexcept(strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } - // NOLINTEND(libcpp-robust-against-adl) +struct __fn { + // NOLINTBEGIN(libcpp-robust-against-adl) strong_order should use ADL, but only here + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept( + noexcept(strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return strong_ordering(strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } + // NOLINTEND(libcpp-robust-against-adl) - template> - requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp> - _LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept - { - if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int32_t)) { - int32_t __rx = std::bit_cast(__t); - int32_t __ry = std::bit_cast(__u); - __rx = (__rx < 0) ? (numeric_limits::min() - __rx - 1) : __rx; - __ry = (__ry < 0) ? (numeric_limits::min() - __ry - 1) : __ry; - return (__rx <=> __ry); - } else if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int64_t)) { - int64_t __rx = std::bit_cast(__t); - int64_t __ry = std::bit_cast(__u); - __rx = (__rx < 0) ? (numeric_limits::min() - __rx - 1) : __rx; - __ry = (__ry < 0) ? (numeric_limits::min() - __ry - 1) : __ry; - return (__rx <=> __ry); - } else if (__t < __u) { - return strong_ordering::less; - } else if (__t > __u) { - return strong_ordering::greater; - } else if (__t == __u) { - if constexpr (numeric_limits<_Dp>::radix == 2) { - return std::signbit(__u) <=> std::signbit(__t); - } else { - // This is bullet 3 of the IEEE754 algorithm, relevant - // only for decimal floating-point; - // see https://stackoverflow.com/questions/69068075/ - if (__t == 0 || std::isinf(__t)) { - return std::signbit(__u) <=> std::signbit(__t); - } else { - int __texp, __uexp; - (void)std::frexp(__t, &__texp); - (void)std::frexp(__u, &__uexp); - return (__t < 0) ? (__texp <=> __uexp) : (__uexp <=> __texp); - } - } - } else { - // They're unordered, so one of them must be a NAN. - // The order is -QNAN, -SNAN, numbers, +SNAN, +QNAN. - bool __t_is_nan = std::isnan(__t); - bool __u_is_nan = std::isnan(__u); - bool __t_is_negative = std::signbit(__t); - bool __u_is_negative = std::signbit(__u); - using _IntType = conditional_t< - sizeof(__t) == sizeof(int32_t), int32_t, conditional_t< - sizeof(__t) == sizeof(int64_t), int64_t, void> - >; - if constexpr (is_same_v<_IntType, void>) { - static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type"); - } else if (__t_is_nan && __u_is_nan) { - // Order by sign bit, then by "payload bits" (we'll just use bit_cast). - if (__t_is_negative != __u_is_negative) { - return (__u_is_negative <=> __t_is_negative); - } else { - return std::bit_cast<_IntType>(__t) <=> std::bit_cast<_IntType>(__u); - } - } else if (__t_is_nan) { - return __t_is_negative ? strong_ordering::less : strong_ordering::greater; - } else { - return __u_is_negative ? strong_ordering::greater : strong_ordering::less; - } - } + template > + requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp> + _LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept { + if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int32_t)) { + int32_t __rx = std::bit_cast(__t); + int32_t __ry = std::bit_cast(__u); + __rx = (__rx < 0) ? (numeric_limits::min() - __rx - 1) : __rx; + __ry = (__ry < 0) ? (numeric_limits::min() - __ry - 1) : __ry; + return (__rx <=> __ry); + } else if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int64_t)) { + int64_t __rx = std::bit_cast(__t); + int64_t __ry = std::bit_cast(__u); + __rx = (__rx < 0) ? (numeric_limits::min() - __rx - 1) : __rx; + __ry = (__ry < 0) ? (numeric_limits::min() - __ry - 1) : __ry; + return (__rx <=> __ry); + } else if (__t < __u) { + return strong_ordering::less; + } else if (__t > __u) { + return strong_ordering::greater; + } else if (__t == __u) { + if constexpr (numeric_limits<_Dp>::radix == 2) { + return std::signbit(__u) <=> std::signbit(__t); + } else { + // This is bullet 3 of the IEEE754 algorithm, relevant + // only for decimal floating-point; + // see https://stackoverflow.com/questions/69068075/ + if (__t == 0 || std::isinf(__t)) { + return std::signbit(__u) <=> std::signbit(__t); + } else { + int __texp, __uexp; + (void)std::frexp(__t, &__texp); + (void)std::frexp(__u, &__uexp); + return (__t < 0) ? (__texp <=> __uexp) : (__uexp <=> __texp); } + } + } else { + // They're unordered, so one of them must be a NAN. + // The order is -QNAN, -SNAN, numbers, +SNAN, +QNAN. + bool __t_is_nan = std::isnan(__t); + bool __u_is_nan = std::isnan(__u); + bool __t_is_negative = std::signbit(__t); + bool __u_is_negative = std::signbit(__u); + using _IntType = + conditional_t< sizeof(__t) == sizeof(int32_t), + int32_t, + conditional_t< sizeof(__t) == sizeof(int64_t), int64_t, void> >; + if constexpr (is_same_v<_IntType, void>) { + static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type"); + } else if (__t_is_nan && __u_is_nan) { + // Order by sign bit, then by "payload bits" (we'll just use bit_cast). + if (__t_is_negative != __u_is_negative) { + return (__u_is_negative <=> __t_is_negative); + } else { + return std::bit_cast<_IntType>(__t) <=> std::bit_cast<_IntType>(__u); + } + } else if (__t_is_nan) { + return __t_is_negative ? strong_ordering::less : strong_ordering::greater; + } else { + return __u_is_negative ? strong_ordering::greater : strong_ordering::less; + } + } + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept( + noexcept(strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return strong_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<2>()); + } +}; } // namespace __strong_order inline namespace __cpo { - inline constexpr auto strong_order = __strong_order::__fn{}; +inline constexpr auto strong_order = __strong_order::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/three_way_comparable.h b/libcxx/include/__compare/three_way_comparable.h index 2b77bc3f54e6f..7a44ea9158a6f 100644 --- a/libcxx/include/__compare/three_way_comparable.h +++ b/libcxx/include/__compare/three_way_comparable.h @@ -27,30 +27,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template -concept __compares_as = - same_as, _Cat>; +template +concept __compares_as = same_as, _Cat>; -template +template concept three_way_comparable = - __weakly_equality_comparable_with<_Tp, _Tp> && - __partially_ordered_with<_Tp, _Tp> && - requires(__make_const_lvalue_ref<_Tp> __a, __make_const_lvalue_ref<_Tp> __b) { - { __a <=> __b } -> __compares_as<_Cat>; - }; + __weakly_equality_comparable_with<_Tp, _Tp> && __partially_ordered_with<_Tp, _Tp> && + requires(__make_const_lvalue_ref<_Tp> __a, __make_const_lvalue_ref<_Tp> __b) { + { __a <=> __b } -> __compares_as<_Cat>; + }; -template +template concept three_way_comparable_with = - three_way_comparable<_Tp, _Cat> && - three_way_comparable<_Up, _Cat> && - common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> && - three_way_comparable, __make_const_lvalue_ref<_Up>>, _Cat> && - __weakly_equality_comparable_with<_Tp, _Up> && - __partially_ordered_with<_Tp, _Up> && - requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) { - { __t <=> __u } -> __compares_as<_Cat>; - { __u <=> __t } -> __compares_as<_Cat>; - }; + three_way_comparable<_Tp, _Cat> && three_way_comparable<_Up, _Cat> && + common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> && + three_way_comparable, __make_const_lvalue_ref<_Up>>, _Cat> && + __weakly_equality_comparable_with<_Tp, _Up> && __partially_ordered_with<_Tp, _Up> && + requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) { + { __t <=> __u } -> __compares_as<_Cat>; + { __u <=> __t } -> __compares_as<_Cat>; + }; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__compare/weak_order.h b/libcxx/include/__compare/weak_order.h index e6a42ac4c9235..9f719eb64bbca 100644 --- a/libcxx/include/__compare/weak_order.h +++ b/libcxx/include/__compare/weak_order.h @@ -28,71 +28,70 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [cmp.alg] namespace __weak_order { - struct __fn { - // NOLINTBEGIN(libcpp-robust-against-adl) weak_order should use ADL, but only here - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<3>) - noexcept(noexcept(weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } - // NOLINTEND(libcpp-robust-against-adl) +struct __fn { + // NOLINTBEGIN(libcpp-robust-against-adl) weak_order should use ADL, but only here + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<3>) noexcept( + noexcept(weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return weak_ordering(weak_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } + // NOLINTEND(libcpp-robust-against-adl) - template> - requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp> - _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering - __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept - { - partial_ordering __po = (__t <=> __u); - if (__po == partial_ordering::less) { - return weak_ordering::less; - } else if (__po == partial_ordering::equivalent) { - return weak_ordering::equivalent; - } else if (__po == partial_ordering::greater) { - return weak_ordering::greater; - } else { - // Otherwise, at least one of them is a NaN. - bool __t_is_nan = std::isnan(__t); - bool __u_is_nan = std::isnan(__u); - bool __t_is_negative = std::signbit(__t); - bool __u_is_negative = std::signbit(__u); - if (__t_is_nan && __u_is_nan) { - return (__u_is_negative <=> __t_is_negative); - } else if (__t_is_nan) { - return __t_is_negative ? weak_ordering::less : weak_ordering::greater; - } else { - return __u_is_negative ? weak_ordering::greater : weak_ordering::less; - } - } - } + template > + requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp> + _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept { + partial_ordering __po = (__t <=> __u); + if (__po == partial_ordering::less) { + return weak_ordering::less; + } else if (__po == partial_ordering::equivalent) { + return weak_ordering::equivalent; + } else if (__po == partial_ordering::greater) { + return weak_ordering::greater; + } else { + // Otherwise, at least one of them is a NaN. + bool __t_is_nan = std::isnan(__t); + bool __u_is_nan = std::isnan(__u); + bool __t_is_negative = std::signbit(__t); + bool __u_is_negative = std::signbit(__u); + if (__t_is_nan && __u_is_nan) { + return (__u_is_negative <=> __t_is_negative); + } else if (__t_is_nan) { + return __t_is_negative ? weak_ordering::less : weak_ordering::greater; + } else { + return __u_is_negative ? weak_ordering::greater : weak_ordering::less; + } + } + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) - noexcept(noexcept(weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept( + noexcept(weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return weak_ordering(compare_three_way()(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } - template - requires is_same_v, decay_t<_Up>> - _LIBCPP_HIDE_FROM_ABI static constexpr auto - __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) - noexcept(noexcept(weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) - -> decltype( weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) - { return weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); } + template + requires is_same_v, decay_t<_Up>> + _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept( + noexcept(weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))))) + -> decltype(weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u)))) { + return weak_ordering(std::strong_order(std::forward<_Tp>(__t), std::forward<_Up>(__u))); + } - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const - noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>()))) - -> decltype( __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>())) - { return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>()); } - }; + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>()))) + -> decltype(__go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>())) { + return __go(std::forward<_Tp>(__t), std::forward<_Up>(__u), __priority_tag<3>()); + } +}; } // namespace __weak_order inline namespace __cpo { - inline constexpr auto weak_order = __weak_order::__fn{}; +inline constexpr auto weak_order = __weak_order::__fn{}; } // namespace __cpo #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__config b/libcxx/include/__config index 7f66042f90256..adff13e714cb6 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1010,8 +1010,8 @@ typedef __char32_t char32_t; # endif # ifndef _LIBCPP_HAS_NO_ASAN - extern "C" _LIBCPP_EXPORTED_FROM_ABI void - __sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*); +extern "C" _LIBCPP_EXPORTED_FROM_ABI void +__sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*); extern "C" _LIBCPP_EXPORTED_FROM_ABI void __sanitizer_annotate_double_ended_contiguous_container( const void*, const void*, const void*, const void*, const void*, const void*); extern "C" _LIBCPP_EXPORTED_FROM_ABI int diff --git a/libcxx/include/__coroutine/coroutine_handle.h b/libcxx/include/__coroutine/coroutine_handle.h index 4e4e3eb5ba309..54bfe5b44f4c6 100644 --- a/libcxx/include/__coroutine/coroutine_handle.h +++ b/libcxx/include/__coroutine/coroutine_handle.h @@ -32,166 +32,141 @@ struct _LIBCPP_TEMPLATE_VIS coroutine_handle; template <> struct _LIBCPP_TEMPLATE_VIS coroutine_handle { public: - // [coroutine.handle.con], construct/reset - constexpr coroutine_handle() noexcept = default; - - _LIBCPP_HIDE_FROM_ABI - constexpr coroutine_handle(nullptr_t) noexcept {} - - _LIBCPP_HIDE_FROM_ABI - coroutine_handle& operator=(nullptr_t) noexcept { - __handle_ = nullptr; - return *this; - } - - // [coroutine.handle.export.import], export/import - _LIBCPP_HIDE_FROM_ABI - constexpr void* address() const noexcept { return __handle_; } - - _LIBCPP_HIDE_FROM_ABI - static constexpr coroutine_handle from_address(void* __addr) noexcept { - coroutine_handle __tmp; - __tmp.__handle_ = __addr; - return __tmp; - } - - // [coroutine.handle.observers], observers - _LIBCPP_HIDE_FROM_ABI - constexpr explicit operator bool() const noexcept { - return __handle_ != nullptr; - } - - _LIBCPP_HIDE_FROM_ABI - bool done() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "done() can be called only on suspended coroutines"); - return __builtin_coro_done(__handle_); - } - - // [coroutine.handle.resumption], resumption - _LIBCPP_HIDE_FROM_ABI - void operator()() const { resume(); } - - _LIBCPP_HIDE_FROM_ABI - void resume() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "resume() can be called only on suspended coroutines"); - _LIBCPP_ASSERT_UNCATEGORIZED(!done(), "resume() has undefined behavior when the coroutine is done"); - __builtin_coro_resume(__handle_); - } - - _LIBCPP_HIDE_FROM_ABI - void destroy() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "destroy() can be called only on suspended coroutines"); - __builtin_coro_destroy(__handle_); - } + // [coroutine.handle.con], construct/reset + constexpr coroutine_handle() noexcept = default; + + _LIBCPP_HIDE_FROM_ABI constexpr coroutine_handle(nullptr_t) noexcept {} + + _LIBCPP_HIDE_FROM_ABI coroutine_handle& operator=(nullptr_t) noexcept { + __handle_ = nullptr; + return *this; + } + + // [coroutine.handle.export.import], export/import + _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; } + + _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept { + coroutine_handle __tmp; + __tmp.__handle_ = __addr; + return __tmp; + } + + // [coroutine.handle.observers], observers + _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; } + + _LIBCPP_HIDE_FROM_ABI bool done() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "done() can be called only on suspended coroutines"); + return __builtin_coro_done(__handle_); + } + + // [coroutine.handle.resumption], resumption + _LIBCPP_HIDE_FROM_ABI void operator()() const { resume(); } + + _LIBCPP_HIDE_FROM_ABI void resume() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "resume() can be called only on suspended coroutines"); + _LIBCPP_ASSERT_UNCATEGORIZED(!done(), "resume() has undefined behavior when the coroutine is done"); + __builtin_coro_resume(__handle_); + } + + _LIBCPP_HIDE_FROM_ABI void destroy() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "destroy() can be called only on suspended coroutines"); + __builtin_coro_destroy(__handle_); + } private: - _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const { - // FIXME actually implement a check for if the coro is suspended. - return __handle_ != nullptr; - } + _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const { + // FIXME actually implement a check for if the coro is suspended. + return __handle_ != nullptr; + } - void* __handle_ = nullptr; + void* __handle_ = nullptr; }; // [coroutine.handle.compare] -inline _LIBCPP_HIDE_FROM_ABI -constexpr bool operator==(coroutine_handle<> __x, coroutine_handle<> __y) noexcept { - return __x.address() == __y.address(); +inline _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(coroutine_handle<> __x, coroutine_handle<> __y) noexcept { + return __x.address() == __y.address(); } -inline _LIBCPP_HIDE_FROM_ABI -constexpr strong_ordering operator<=>(coroutine_handle<> __x, coroutine_handle<> __y) noexcept { - return compare_three_way()(__x.address(), __y.address()); +inline _LIBCPP_HIDE_FROM_ABI constexpr strong_ordering +operator<=>(coroutine_handle<> __x, coroutine_handle<> __y) noexcept { + return compare_three_way()(__x.address(), __y.address()); } template struct _LIBCPP_TEMPLATE_VIS coroutine_handle { public: - // [coroutine.handle.con], construct/reset - constexpr coroutine_handle() noexcept = default; - - _LIBCPP_HIDE_FROM_ABI - constexpr coroutine_handle(nullptr_t) noexcept {} - - _LIBCPP_HIDE_FROM_ABI - static coroutine_handle from_promise(_Promise& __promise) { - using _RawPromise = __remove_cv_t<_Promise>; - coroutine_handle __tmp; - __tmp.__handle_ = - __builtin_coro_promise(std::addressof(const_cast<_RawPromise&>(__promise)), alignof(_Promise), true); - return __tmp; - } - - _LIBCPP_HIDE_FROM_ABI - coroutine_handle& operator=(nullptr_t) noexcept { - __handle_ = nullptr; - return *this; - } - - // [coroutine.handle.export.import], export/import - _LIBCPP_HIDE_FROM_ABI - constexpr void* address() const noexcept { return __handle_; } - - _LIBCPP_HIDE_FROM_ABI - static constexpr coroutine_handle from_address(void* __addr) noexcept { - coroutine_handle __tmp; - __tmp.__handle_ = __addr; - return __tmp; - } - - // [coroutine.handle.conv], conversion - _LIBCPP_HIDE_FROM_ABI - constexpr operator coroutine_handle<>() const noexcept { - return coroutine_handle<>::from_address(address()); - } - - // [coroutine.handle.observers], observers - _LIBCPP_HIDE_FROM_ABI - constexpr explicit operator bool() const noexcept { - return __handle_ != nullptr; - } - - _LIBCPP_HIDE_FROM_ABI - bool done() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "done() can be called only on suspended coroutines"); - return __builtin_coro_done(__handle_); - } - - // [coroutine.handle.resumption], resumption - _LIBCPP_HIDE_FROM_ABI - void operator()() const { resume(); } - - _LIBCPP_HIDE_FROM_ABI - void resume() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "resume() can be called only on suspended coroutines"); - _LIBCPP_ASSERT_UNCATEGORIZED(!done(), "resume() has undefined behavior when the coroutine is done"); - __builtin_coro_resume(__handle_); - } - - _LIBCPP_HIDE_FROM_ABI - void destroy() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "destroy() can be called only on suspended coroutines"); - __builtin_coro_destroy(__handle_); - } - - // [coroutine.handle.promise], promise access - _LIBCPP_HIDE_FROM_ABI - _Promise& promise() const { - return *static_cast<_Promise*>(__builtin_coro_promise(this->__handle_, alignof(_Promise), false)); - } + // [coroutine.handle.con], construct/reset + constexpr coroutine_handle() noexcept = default; + + _LIBCPP_HIDE_FROM_ABI constexpr coroutine_handle(nullptr_t) noexcept {} + + _LIBCPP_HIDE_FROM_ABI static coroutine_handle from_promise(_Promise& __promise) { + using _RawPromise = __remove_cv_t<_Promise>; + coroutine_handle __tmp; + __tmp.__handle_ = + __builtin_coro_promise(std::addressof(const_cast<_RawPromise&>(__promise)), alignof(_Promise), true); + return __tmp; + } + + _LIBCPP_HIDE_FROM_ABI coroutine_handle& operator=(nullptr_t) noexcept { + __handle_ = nullptr; + return *this; + } + + // [coroutine.handle.export.import], export/import + _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; } + + _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept { + coroutine_handle __tmp; + __tmp.__handle_ = __addr; + return __tmp; + } + + // [coroutine.handle.conv], conversion + _LIBCPP_HIDE_FROM_ABI constexpr operator coroutine_handle<>() const noexcept { + return coroutine_handle<>::from_address(address()); + } + + // [coroutine.handle.observers], observers + _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; } + + _LIBCPP_HIDE_FROM_ABI bool done() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "done() can be called only on suspended coroutines"); + return __builtin_coro_done(__handle_); + } + + // [coroutine.handle.resumption], resumption + _LIBCPP_HIDE_FROM_ABI void operator()() const { resume(); } + + _LIBCPP_HIDE_FROM_ABI void resume() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "resume() can be called only on suspended coroutines"); + _LIBCPP_ASSERT_UNCATEGORIZED(!done(), "resume() has undefined behavior when the coroutine is done"); + __builtin_coro_resume(__handle_); + } + + _LIBCPP_HIDE_FROM_ABI void destroy() const { + _LIBCPP_ASSERT_UNCATEGORIZED(__is_suspended(), "destroy() can be called only on suspended coroutines"); + __builtin_coro_destroy(__handle_); + } + + // [coroutine.handle.promise], promise access + _LIBCPP_HIDE_FROM_ABI _Promise& promise() const { + return *static_cast<_Promise*>(__builtin_coro_promise(this->__handle_, alignof(_Promise), false)); + } private: - _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const { - // FIXME actually implement a check for if the coro is suspended. - return __handle_ != nullptr; - } - void* __handle_ = nullptr; + _LIBCPP_HIDE_FROM_ABI bool __is_suspended() const { + // FIXME actually implement a check for if the coro is suspended. + return __handle_ != nullptr; + } + void* __handle_ = nullptr; }; // [coroutine.handle.hash] template struct hash> { - _LIBCPP_HIDE_FROM_ABI - size_t operator()(const coroutine_handle<_Tp>& __v) const noexcept { return hash()(__v.address()); } + _LIBCPP_HIDE_FROM_ABI size_t operator()(const coroutine_handle<_Tp>& __v) const noexcept { + return hash()(__v.address()); + } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__coroutine/coroutine_traits.h b/libcxx/include/__coroutine/coroutine_traits.h index 7122cc8ea8fb2..78f05341f7486 100644 --- a/libcxx/include/__coroutine/coroutine_traits.h +++ b/libcxx/include/__coroutine/coroutine_traits.h @@ -34,17 +34,12 @@ template struct __coroutine_traits_sfinae {}; template -struct __coroutine_traits_sfinae< - _Tp, __void_t > -{ +struct __coroutine_traits_sfinae< _Tp, __void_t > { using promise_type = typename _Tp::promise_type; }; template -struct coroutine_traits - : public __coroutine_traits_sfinae<_Ret> -{ -}; +struct coroutine_traits : public __coroutine_traits_sfinae<_Ret> {}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__coroutine/noop_coroutine_handle.h b/libcxx/include/__coroutine/noop_coroutine_handle.h index 9b7802d1e2827..da13d579604b5 100644 --- a/libcxx/include/__coroutine/noop_coroutine_handle.h +++ b/libcxx/include/__coroutine/noop_coroutine_handle.h @@ -20,7 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC) +# if __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC) // [coroutine.noop] // [coroutine.promise.noop] @@ -30,80 +30,67 @@ struct noop_coroutine_promise {}; template <> struct _LIBCPP_TEMPLATE_VIS coroutine_handle { public: - // [coroutine.handle.noop.conv], conversion - _LIBCPP_HIDE_FROM_ABI - constexpr operator coroutine_handle<>() const noexcept { - return coroutine_handle<>::from_address(address()); - } - - // [coroutine.handle.noop.observers], observers - _LIBCPP_HIDE_FROM_ABI - constexpr explicit operator bool() const noexcept { return true; } - _LIBCPP_HIDE_FROM_ABI - constexpr bool done() const noexcept { return false; } - - // [coroutine.handle.noop.resumption], resumption - _LIBCPP_HIDE_FROM_ABI - constexpr void operator()() const noexcept {} - _LIBCPP_HIDE_FROM_ABI - constexpr void resume() const noexcept {} - _LIBCPP_HIDE_FROM_ABI - constexpr void destroy() const noexcept {} - - // [coroutine.handle.noop.promise], promise access - _LIBCPP_HIDE_FROM_ABI - noop_coroutine_promise& promise() const noexcept { - return *static_cast( - __builtin_coro_promise(this->__handle_, alignof(noop_coroutine_promise), false)); - } - - // [coroutine.handle.noop.address], address - _LIBCPP_HIDE_FROM_ABI - constexpr void* address() const noexcept { return __handle_; } + // [coroutine.handle.noop.conv], conversion + _LIBCPP_HIDE_FROM_ABI constexpr operator coroutine_handle<>() const noexcept { + return coroutine_handle<>::from_address(address()); + } + + // [coroutine.handle.noop.observers], observers + _LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return true; } + _LIBCPP_HIDE_FROM_ABI constexpr bool done() const noexcept { return false; } + + // [coroutine.handle.noop.resumption], resumption + _LIBCPP_HIDE_FROM_ABI constexpr void operator()() const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr void resume() const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr void destroy() const noexcept {} + + // [coroutine.handle.noop.promise], promise access + _LIBCPP_HIDE_FROM_ABI noop_coroutine_promise& promise() const noexcept { + return *static_cast( + __builtin_coro_promise(this->__handle_, alignof(noop_coroutine_promise), false)); + } + + // [coroutine.handle.noop.address], address + _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; } private: - _LIBCPP_HIDE_FROM_ABI - friend coroutine_handle noop_coroutine() noexcept; + _LIBCPP_HIDE_FROM_ABI friend coroutine_handle noop_coroutine() noexcept; -#if __has_builtin(__builtin_coro_noop) - _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept { - this->__handle_ = __builtin_coro_noop(); - } +# if __has_builtin(__builtin_coro_noop) + _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept { this->__handle_ = __builtin_coro_noop(); } - void* __handle_ = nullptr; + void* __handle_ = nullptr; -#elif defined(_LIBCPP_COMPILER_GCC) - // GCC doesn't implement __builtin_coro_noop(). - // Construct the coroutine frame manually instead. - struct __noop_coroutine_frame_ty_ { - static void __dummy_resume_destroy_func() { } +# elif defined(_LIBCPP_COMPILER_GCC) + // GCC doesn't implement __builtin_coro_noop(). + // Construct the coroutine frame manually instead. + struct __noop_coroutine_frame_ty_ { + static void __dummy_resume_destroy_func() {} - void (*__resume_)() = __dummy_resume_destroy_func; - void (*__destroy_)() = __dummy_resume_destroy_func; - struct noop_coroutine_promise __promise_; - }; + void (*__resume_)() = __dummy_resume_destroy_func; + void (*__destroy_)() = __dummy_resume_destroy_func; + struct noop_coroutine_promise __promise_; + }; - static __noop_coroutine_frame_ty_ __noop_coroutine_frame_; + static __noop_coroutine_frame_ty_ __noop_coroutine_frame_; - void* __handle_ = &__noop_coroutine_frame_; + void* __handle_ = &__noop_coroutine_frame_; - _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept = default; + _LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept = default; -#endif // __has_builtin(__builtin_coro_noop) +# endif // __has_builtin(__builtin_coro_noop) }; using noop_coroutine_handle = coroutine_handle; -#if defined(_LIBCPP_COMPILER_GCC) -inline noop_coroutine_handle::__noop_coroutine_frame_ty_ - noop_coroutine_handle::__noop_coroutine_frame_{}; -#endif +# if defined(_LIBCPP_COMPILER_GCC) +inline noop_coroutine_handle::__noop_coroutine_frame_ty_ noop_coroutine_handle::__noop_coroutine_frame_{}; +# endif // [coroutine.noop.coroutine] -inline _LIBCPP_HIDE_FROM_ABI -noop_coroutine_handle noop_coroutine() noexcept { return noop_coroutine_handle(); } +inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept { return noop_coroutine_handle(); } -#endif // __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC) +# endif // __has_builtin(__builtin_coro_noop) || defined(_LIBCPP_COMPILER_GCC) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__coroutine/trivial_awaitables.h b/libcxx/include/__coroutine/trivial_awaitables.h index 0e4b08e377c0e..b604bd3c2d8ad 100644 --- a/libcxx/include/__coroutine/trivial_awaitables.h +++ b/libcxx/include/__coroutine/trivial_awaitables.h @@ -22,21 +22,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD // [coroutine.trivial.awaitables] struct suspend_never { - _LIBCPP_HIDE_FROM_ABI - constexpr bool await_ready() const noexcept { return true; } - _LIBCPP_HIDE_FROM_ABI - constexpr void await_suspend(coroutine_handle<>) const noexcept {} - _LIBCPP_HIDE_FROM_ABI - constexpr void await_resume() const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr bool await_ready() const noexcept { return true; } + _LIBCPP_HIDE_FROM_ABI constexpr void await_suspend(coroutine_handle<>) const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr void await_resume() const noexcept {} }; struct suspend_always { - _LIBCPP_HIDE_FROM_ABI - constexpr bool await_ready() const noexcept { return false; } - _LIBCPP_HIDE_FROM_ABI - constexpr void await_suspend(coroutine_handle<>) const noexcept {} - _LIBCPP_HIDE_FROM_ABI - constexpr void await_resume() const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr bool await_ready() const noexcept { return false; } + _LIBCPP_HIDE_FROM_ABI constexpr void await_suspend(coroutine_handle<>) const noexcept {} + _LIBCPP_HIDE_FROM_ABI constexpr void await_resume() const noexcept {} }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__debug_utils/randomize_range.h b/libcxx/include/__debug_utils/randomize_range.h index dce61923bc9c8..7eb77d81ab2a3 100644 --- a/libcxx/include/__debug_utils/randomize_range.h +++ b/libcxx/include/__debug_utils/randomize_range.h @@ -23,8 +23,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -void __debug_randomize_range(_Iterator __first, _Sentinel __last) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __debug_randomize_range(_Iterator __first, _Sentinel __last) { #ifdef _LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY # ifdef _LIBCPP_CXX03_LANG # error Support for unspecified stability is only for C++11 and higher diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h index bf16c8f720d26..97f92bdfe44c4 100644 --- a/libcxx/include/__expected/expected.h +++ b/libcxx/include/__expected/expected.h @@ -90,19 +90,14 @@ _LIBCPP_HIDE_FROM_ABI void __throw_bad_expected_access(_Arg&& __arg) { template class expected { - static_assert( - !is_reference_v<_Tp> && - !is_function_v<_Tp> && - !is_same_v, in_place_t> && - !is_same_v, unexpect_t> && - !__is_std_unexpected>::value && - __valid_std_unexpected<_Err>::value - , - "[expected.object.general] A program that instantiates the definition of template expected for a " - "reference type, a function type, or for possibly cv-qualified types in_place_t, unexpect_t, or a " - "specialization of unexpected for the T parameter is ill-formed. A program that instantiates the " - "definition of the template expected with a type for the E parameter that is not a valid " - "template argument for unexpected is ill-formed."); + static_assert(!is_reference_v<_Tp> && !is_function_v<_Tp> && !is_same_v, in_place_t> && + !is_same_v, unexpect_t> && !__is_std_unexpected>::value && + __valid_std_unexpected<_Err>::value, + "[expected.object.general] A program that instantiates the definition of template expected for a " + "reference type, a function type, or for possibly cv-qualified types in_place_t, unexpect_t, or a " + "specialization of unexpected for the T parameter is ill-formed. A program that instantiates the " + "definition of the template expected with a type for the E parameter that is not a valid " + "template argument for unexpected is ill-formed."); template friend class expected; @@ -116,36 +111,33 @@ class expected { using rebind = expected<_Up, error_type>; // [expected.object.ctor], constructors - _LIBCPP_HIDE_FROM_ABI constexpr expected() - noexcept(is_nothrow_default_constructible_v<_Tp>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr expected() noexcept(is_nothrow_default_constructible_v<_Tp>) // strengthened requires is_default_constructible_v<_Tp> : __union_(std::in_place), __has_val_(true) {} _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&) = delete; _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&) - requires(is_copy_constructible_v<_Tp> && - is_copy_constructible_v<_Err> && - is_trivially_copy_constructible_v<_Tp> && + requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>) = default; - _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __other) - noexcept(is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_constructible_v<_Err>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __other) noexcept( + is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_constructible_v<_Err>) // strengthened requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> && !(is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>)) - : __union_(__other.__has_val_, __other.__union_), __has_val_(__other.__has_val_) { } + : __union_(__other.__has_val_, __other.__union_), __has_val_(__other.__has_val_) {} _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&&) - requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> - && is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>) + requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Tp> && + is_trivially_move_constructible_v<_Err>) = default; - _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __other) - noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_constructible_v<_Err>) + _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __other) noexcept( + is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_constructible_v<_Err>) requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> && !(is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>)) - : __union_(__other.__has_val_, std::move(__other.__union_)), __has_val_(__other.__has_val_) { } + : __union_(__other.__has_val_, std::move(__other.__union_)), __has_val_(__other.__has_val_) {} private: template @@ -182,16 +174,16 @@ class expected { requires __can_convert<_Up, _OtherErr, const _Up&, const _OtherErr&>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v || !is_convertible_v) - expected(const expected<_Up, _OtherErr>& __other) - noexcept(is_nothrow_constructible_v<_Tp, const _Up&> && - is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened + expected(const expected<_Up, _OtherErr>& __other) noexcept( + is_nothrow_constructible_v<_Tp, const _Up&> && + is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened : __union_(__other.__has_val_, __other.__union_), __has_val_(__other.__has_val_) {} template requires __can_convert<_Up, _OtherErr, _Up, _OtherErr>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp> || !is_convertible_v<_OtherErr, _Err>) - expected(expected<_Up, _OtherErr>&& __other) - noexcept(is_nothrow_constructible_v<_Tp, _Up> && is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened + expected(expected<_Up, _OtherErr>&& __other) noexcept( + is_nothrow_constructible_v<_Tp, _Up> && is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened : __union_(__other.__has_val_, std::move(__other.__union_)), __has_val_(__other.__has_val_) {} template @@ -204,42 +196,38 @@ class expected { template requires is_constructible_v<_Err, const _OtherErr&> - _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) - expected(const unexpected<_OtherErr>& __unex) - noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) expected( + const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened : __union_(std::unexpect, __unex.error()), __has_val_(false) {} template requires is_constructible_v<_Err, _OtherErr> _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) - expected(unexpected<_OtherErr>&& __unex) - noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened + expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened : __union_(std::unexpect, std::move(__unex.error())), __has_val_(false) {} template requires is_constructible_v<_Tp, _Args...> - _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Tp, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Tp, _Args...>) // strengthened : __union_(std::in_place, std::forward<_Args>(__args)...), __has_val_(true) {} template requires is_constructible_v< _Tp, initializer_list<_Up>&, _Args... > - _LIBCPP_HIDE_FROM_ABI constexpr explicit - expected(in_place_t, initializer_list<_Up> __il, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, initializer_list<_Up> __il, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>) // strengthened : __union_(std::in_place, __il, std::forward<_Args>(__args)...), __has_val_(true) {} template requires is_constructible_v<_Err, _Args...> - _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Err, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Err, _Args...>) // strengthened : __union_(std::unexpect, std::forward<_Args>(__args)...), __has_val_(false) {} template requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... > - _LIBCPP_HIDE_FROM_ABI constexpr explicit - expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened : __union_(std::unexpect, __il, std::forward<_Args>(__args)...), __has_val_(false) {} // [expected.object.dtor], destructor @@ -286,17 +274,12 @@ class expected { // [expected.object.assign], assignment _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected&) = delete; - _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) - noexcept(is_nothrow_copy_assignable_v<_Tp> && - is_nothrow_copy_constructible_v<_Tp> && - is_nothrow_copy_assignable_v<_Err> && - is_nothrow_copy_constructible_v<_Err>) // strengthened - requires(is_copy_assignable_v<_Tp> && - is_copy_constructible_v<_Tp> && - is_copy_assignable_v<_Err> && + _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) noexcept( + is_nothrow_copy_assignable_v<_Tp> && is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_assignable_v<_Err> && + is_nothrow_copy_constructible_v<_Err>) // strengthened + requires(is_copy_assignable_v<_Tp> && is_copy_constructible_v<_Tp> && is_copy_assignable_v<_Err> && is_copy_constructible_v<_Err> && - (is_nothrow_move_constructible_v<_Tp> || - is_nothrow_move_constructible_v<_Err>)) + (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>)) { if (__has_val_ && __rhs.__has_val_) { __union_.__val_ = __rhs.__union_.__val_; @@ -312,17 +295,12 @@ class expected { return *this; } - _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(expected&& __rhs) - noexcept(is_nothrow_move_assignable_v<_Tp> && - is_nothrow_move_constructible_v<_Tp> && - is_nothrow_move_assignable_v<_Err> && - is_nothrow_move_constructible_v<_Err>) - requires(is_move_constructible_v<_Tp> && - is_move_assignable_v<_Tp> && - is_move_constructible_v<_Err> && + _LIBCPP_HIDE_FROM_ABI constexpr expected& + operator=(expected&& __rhs) noexcept(is_nothrow_move_assignable_v<_Tp> && is_nothrow_move_constructible_v<_Tp> && + is_nothrow_move_assignable_v<_Err> && is_nothrow_move_constructible_v<_Err>) + requires(is_move_constructible_v<_Tp> && is_move_assignable_v<_Tp> && is_move_constructible_v<_Err> && is_move_assignable_v<_Err> && - (is_nothrow_move_constructible_v<_Tp> || - is_nothrow_move_constructible_v<_Err>)) + (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>)) { if (__has_val_ && __rhs.__has_val_) { __union_.__val_ = std::move(__rhs.__union_.__val_); @@ -340,12 +318,9 @@ class expected { template _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(_Up&& __v) - requires(!is_same_v> && - !__is_std_unexpected>::value && - is_constructible_v<_Tp, _Up> && - is_assignable_v<_Tp&, _Up> && - (is_nothrow_constructible_v<_Tp, _Up> || - is_nothrow_move_constructible_v<_Tp> || + requires(!is_same_v> && !__is_std_unexpected>::value && + is_constructible_v<_Tp, _Up> && is_assignable_v<_Tp&, _Up> && + (is_nothrow_constructible_v<_Tp, _Up> || is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>)) { if (__has_val_) { @@ -418,20 +393,14 @@ class expected { return __union_.__val_; } - public: // [expected.object.swap], swap - _LIBCPP_HIDE_FROM_ABI constexpr void swap(expected& __rhs) - noexcept(is_nothrow_move_constructible_v<_Tp> && - is_nothrow_swappable_v<_Tp> && - is_nothrow_move_constructible_v<_Err> && - is_nothrow_swappable_v<_Err>) - requires(is_swappable_v<_Tp> && - is_swappable_v<_Err> && - is_move_constructible_v<_Tp> && + _LIBCPP_HIDE_FROM_ABI constexpr void + swap(expected& __rhs) noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_swappable_v<_Tp> && + is_nothrow_move_constructible_v<_Err> && is_nothrow_swappable_v<_Err>) + requires(is_swappable_v<_Tp> && is_swappable_v<_Err> && is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> && - (is_nothrow_move_constructible_v<_Tp> || - is_nothrow_move_constructible_v<_Err>)) + (is_nothrow_move_constructible_v<_Tp> || is_nothrow_move_constructible_v<_Err>)) { auto __swap_val_unex_impl = [&](expected& __with_val, expected& __with_err) { if constexpr (is_nothrow_move_constructible_v<_Err>) { @@ -479,8 +448,7 @@ class expected { } } - _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) - noexcept(noexcept(__x.swap(__y))) + _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) noexcept(noexcept(__x.swap(__y))) requires requires { __x.swap(__y); } { __x.swap(__y); @@ -724,7 +692,8 @@ class expected { return expected<_Up, _Err>(unexpect, error()); } if constexpr (!is_void_v<_Up>) { - return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), __union_.__val_); + return expected<_Up, _Err>( + __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), __union_.__val_); } else { std::invoke(std::forward<_Func>(__f), __union_.__val_); return expected<_Up, _Err>(); @@ -739,7 +708,8 @@ class expected { return expected<_Up, _Err>(unexpect, error()); } if constexpr (!is_void_v<_Up>) { - return expected<_Up, _Err>(__expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), __union_.__val_); + return expected<_Up, _Err>( + __expected_construct_in_place_from_invoke_tag{}, std::forward<_Func>(__f), __union_.__val_); } else { std::invoke(std::forward<_Func>(__f), __union_.__val_); return expected<_Up, _Err>(); @@ -900,7 +870,7 @@ class expected { template requires(is_trivially_move_constructible_v<_ValueType> && is_trivially_move_constructible_v<_ErrorType>) union __union_t<_ValueType, _ErrorType> { - _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = default; + _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = default; _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(const __union_t&) = default; template @@ -982,8 +952,8 @@ class expected<_Tp, _Err> { requires(is_copy_constructible_v<_Err> && is_trivially_copy_constructible_v<_Err>) = default; - _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __rhs) - noexcept(is_nothrow_copy_constructible_v<_Err>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __rhs) noexcept( + is_nothrow_copy_constructible_v<_Err>) // strengthened requires(is_copy_constructible_v<_Err> && !is_trivially_copy_constructible_v<_Err>) : __union_(__rhs.__has_val_, __rhs.__union_), __has_val_(__rhs.__has_val_) {} @@ -991,51 +961,47 @@ class expected<_Tp, _Err> { requires(is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Err>) = default; - _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __rhs) - noexcept(is_nothrow_move_constructible_v<_Err>) + _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __rhs) noexcept(is_nothrow_move_constructible_v<_Err>) requires(is_move_constructible_v<_Err> && !is_trivially_move_constructible_v<_Err>) : __union_(__rhs.__has_val_, std::move(__rhs.__union_)), __has_val_(__rhs.__has_val_) {} template requires __can_convert<_Up, _OtherErr, const _OtherErr&>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) - expected(const expected<_Up, _OtherErr>& __rhs) - noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened + expected(const expected<_Up, _OtherErr>& __rhs) noexcept( + is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened : __union_(__rhs.__has_val_, __rhs.__union_), __has_val_(__rhs.__has_val_) {} template requires __can_convert<_Up, _OtherErr, _OtherErr>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) - expected(expected<_Up, _OtherErr>&& __rhs) - noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened + expected(expected<_Up, _OtherErr>&& __rhs) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened : __union_(__rhs.__has_val_, std::move(__rhs.__union_)), __has_val_(__rhs.__has_val_) {} template requires is_constructible_v<_Err, const _OtherErr&> - _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) - expected(const unexpected<_OtherErr>& __unex) - noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) expected( + const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened : __union_(std::unexpect, __unex.error()), __has_val_(false) {} template requires is_constructible_v<_Err, _OtherErr> _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) - expected(unexpected<_OtherErr>&& __unex) - noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened + expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened : __union_(std::unexpect, std::move(__unex.error())), __has_val_(false) {} _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t) noexcept : __has_val_(true) {} template requires is_constructible_v<_Err, _Args...> - _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Err, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Err, _Args...>) // strengthened : __union_(std::unexpect, std::forward<_Args>(__args)...), __has_val_(false) {} template requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... > - _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept( + is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened : __union_(std::unexpect, __il, std::forward<_Args>(__args)...), __has_val_(false) {} private: @@ -1069,8 +1035,8 @@ class expected<_Tp, _Err> { _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected&) = delete; - _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) - noexcept(is_nothrow_copy_assignable_v<_Err> && is_nothrow_copy_constructible_v<_Err>) // strengthened + _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(const expected& __rhs) noexcept( + is_nothrow_copy_assignable_v<_Err> && is_nothrow_copy_constructible_v<_Err>) // strengthened requires(is_copy_assignable_v<_Err> && is_copy_constructible_v<_Err>) { if (__has_val_) { @@ -1091,11 +1057,9 @@ class expected<_Tp, _Err> { _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(expected&&) = delete; - _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(expected&& __rhs) - noexcept(is_nothrow_move_assignable_v<_Err> && - is_nothrow_move_constructible_v<_Err>) - requires(is_move_assignable_v<_Err> && - is_move_constructible_v<_Err>) + _LIBCPP_HIDE_FROM_ABI constexpr expected& + operator=(expected&& __rhs) noexcept(is_nothrow_move_assignable_v<_Err> && is_nothrow_move_constructible_v<_Err>) + requires(is_move_assignable_v<_Err> && is_move_constructible_v<_Err>) { if (__has_val_) { if (!__rhs.__has_val_) { @@ -1145,8 +1109,8 @@ class expected<_Tp, _Err> { } // [expected.void.swap], swap - _LIBCPP_HIDE_FROM_ABI constexpr void swap(expected& __rhs) - noexcept(is_nothrow_move_constructible_v<_Err> && is_nothrow_swappable_v<_Err>) + _LIBCPP_HIDE_FROM_ABI constexpr void + swap(expected& __rhs) noexcept(is_nothrow_move_constructible_v<_Err> && is_nothrow_swappable_v<_Err>) requires(is_swappable_v<_Err> && is_move_constructible_v<_Err>) { auto __swap_val_unex_impl = [&](expected& __with_val, expected& __with_err) { @@ -1170,8 +1134,7 @@ class expected<_Tp, _Err> { } } - _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) - noexcept(noexcept(__x.swap(__y))) + _LIBCPP_HIDE_FROM_ABI friend constexpr void swap(expected& __x, expected& __y) noexcept(noexcept(__x.swap(__y))) requires requires { __x.swap(__y); } { __x.swap(__y); @@ -1318,8 +1281,8 @@ class expected<_Tp, _Err> { template _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && { using _Gp = remove_cvref_t>; - static_assert(__is_std_expected<_Gp>::value, - "The result of f(std::move(error())) must be a specialization of std::expected"); + static_assert( + __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected"); static_assert(is_same_v, "The result of f(std::move(error())) must have the same value_type as this expected"); if (has_value()) { @@ -1331,8 +1294,8 @@ class expected<_Tp, _Err> { template _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& { using _Gp = remove_cvref_t>; - static_assert(__is_std_expected<_Gp>::value, - "The result of f(std::move(error())) must be a specialization of std::expected"); + static_assert( + __is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected"); static_assert(is_same_v, "The result of f(std::move(error())) must have the same value_type as this expected"); if (has_value()) { @@ -1506,7 +1469,7 @@ class expected<_Tp, _Err> { requires is_trivially_move_constructible_v<_ErrorType> union __union_t<_ErrorType> { _LIBCPP_HIDE_FROM_ABI constexpr __union_t() : __empty_() {} - _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = default; + _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = default; _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(const __union_t&) = default; template diff --git a/libcxx/include/__filesystem/copy_options.h b/libcxx/include/__filesystem/copy_options.h index 60cab4ee69737..1bf71292c8a66 100644 --- a/libcxx/include/__filesystem/copy_options.h +++ b/libcxx/include/__filesystem/copy_options.h @@ -22,54 +22,44 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM enum class copy_options : unsigned short { - none = 0, - skip_existing = 1, - overwrite_existing = 2, - update_existing = 4, - recursive = 8, - copy_symlinks = 16, - skip_symlinks = 32, - directories_only = 64, - create_symlinks = 128, - create_hard_links = 256, + none = 0, + skip_existing = 1, + overwrite_existing = 2, + update_existing = 4, + recursive = 8, + copy_symlinks = 16, + skip_symlinks = 32, + directories_only = 64, + create_symlinks = 128, + create_hard_links = 256, __in_recursive_copy = 512, }; -_LIBCPP_HIDE_FROM_ABI -inline constexpr copy_options operator&(copy_options __lhs, copy_options __rhs) { - return static_cast(static_cast(__lhs) & - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator&(copy_options __lhs, copy_options __rhs) { + return static_cast(static_cast(__lhs) & static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr copy_options operator|(copy_options __lhs, copy_options __rhs) { - return static_cast(static_cast(__lhs) | - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator|(copy_options __lhs, copy_options __rhs) { + return static_cast(static_cast(__lhs) | static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr copy_options operator^(copy_options __lhs, copy_options __rhs) { - return static_cast(static_cast(__lhs) ^ - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator^(copy_options __lhs, copy_options __rhs) { + return static_cast(static_cast(__lhs) ^ static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr copy_options operator~(copy_options __lhs) { +_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator~(copy_options __lhs) { return static_cast(~static_cast(__lhs)); } -_LIBCPP_HIDE_FROM_ABI -inline copy_options& operator&=(copy_options& __lhs, copy_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline copy_options& operator&=(copy_options& __lhs, copy_options __rhs) { return __lhs = __lhs & __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline copy_options& operator|=(copy_options& __lhs, copy_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline copy_options& operator|=(copy_options& __lhs, copy_options __rhs) { return __lhs = __lhs | __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline copy_options& operator^=(copy_options& __lhs, copy_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline copy_options& operator^=(copy_options& __lhs, copy_options __rhs) { return __lhs = __lhs ^ __rhs; } diff --git a/libcxx/include/__filesystem/directory_entry.h b/libcxx/include/__filesystem/directory_entry.h index fe3cace853d65..016ad94a853dc 100644 --- a/libcxx/include/__filesystem/directory_entry.h +++ b/libcxx/include/__filesystem/directory_entry.h @@ -45,219 +45,149 @@ class directory_entry { public: // constructors and destructors - _LIBCPP_HIDE_FROM_ABI directory_entry() noexcept = default; - _LIBCPP_HIDE_FROM_ABI directory_entry(directory_entry const&) = default; + _LIBCPP_HIDE_FROM_ABI directory_entry() noexcept = default; + _LIBCPP_HIDE_FROM_ABI directory_entry(directory_entry const&) = default; _LIBCPP_HIDE_FROM_ABI directory_entry(directory_entry&&) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - explicit directory_entry(_Path const& __p) : __p_(__p) { + _LIBCPP_HIDE_FROM_ABI explicit directory_entry(_Path const& __p) : __p_(__p) { error_code __ec; __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) { - __refresh(&__ec); - } + _LIBCPP_HIDE_FROM_ABI directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) { __refresh(&__ec); } _LIBCPP_HIDE_FROM_ABI ~directory_entry() {} - _LIBCPP_HIDE_FROM_ABI directory_entry& operator=(directory_entry const&) = default; + _LIBCPP_HIDE_FROM_ABI directory_entry& operator=(directory_entry const&) = default; _LIBCPP_HIDE_FROM_ABI directory_entry& operator=(directory_entry&&) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - void assign(_Path const& __p) { + _LIBCPP_HIDE_FROM_ABI void assign(_Path const& __p) { __p_ = __p; error_code __ec; __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - void assign(_Path const& __p, error_code& __ec) { + _LIBCPP_HIDE_FROM_ABI void assign(_Path const& __p, error_code& __ec) { __p_ = __p; __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - void replace_filename(_Path const& __p) { + _LIBCPP_HIDE_FROM_ABI void replace_filename(_Path const& __p) { __p_.replace_filename(__p); error_code __ec; __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - void replace_filename(_Path const& __p, error_code& __ec) { + _LIBCPP_HIDE_FROM_ABI void replace_filename(_Path const& __p, error_code& __ec) { __p_ = __p_.parent_path() / __p; __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - void refresh() { __refresh(); } + _LIBCPP_HIDE_FROM_ABI void refresh() { __refresh(); } - _LIBCPP_HIDE_FROM_ABI - void refresh(error_code& __ec) noexcept { __refresh(&__ec); } + _LIBCPP_HIDE_FROM_ABI void refresh(error_code& __ec) noexcept { __refresh(&__ec); } - _LIBCPP_HIDE_FROM_ABI - _Path const& path() const noexcept { return __p_; } + _LIBCPP_HIDE_FROM_ABI _Path const& path() const noexcept { return __p_; } - _LIBCPP_HIDE_FROM_ABI - operator const _Path&() const noexcept { return __p_; } + _LIBCPP_HIDE_FROM_ABI operator const _Path&() const noexcept { return __p_; } - _LIBCPP_HIDE_FROM_ABI - bool exists() const { return filesystem::exists(file_status{__get_ft()}); } + _LIBCPP_HIDE_FROM_ABI bool exists() const { return filesystem::exists(file_status{__get_ft()}); } - _LIBCPP_HIDE_FROM_ABI - bool exists(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool exists(error_code& __ec) const noexcept { return filesystem::exists(file_status{__get_ft(&__ec)}); } - _LIBCPP_HIDE_FROM_ABI - bool is_block_file() const { return __get_ft() == file_type::block; } + _LIBCPP_HIDE_FROM_ABI bool is_block_file() const { return __get_ft() == file_type::block; } - _LIBCPP_HIDE_FROM_ABI - bool is_block_file(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_block_file(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::block; } - _LIBCPP_HIDE_FROM_ABI - bool is_character_file() const { return __get_ft() == file_type::character; } + _LIBCPP_HIDE_FROM_ABI bool is_character_file() const { return __get_ft() == file_type::character; } - _LIBCPP_HIDE_FROM_ABI - bool is_character_file(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_character_file(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::character; } - _LIBCPP_HIDE_FROM_ABI - bool is_directory() const { return __get_ft() == file_type::directory; } + _LIBCPP_HIDE_FROM_ABI bool is_directory() const { return __get_ft() == file_type::directory; } - _LIBCPP_HIDE_FROM_ABI - bool is_directory(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_directory(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::directory; } - _LIBCPP_HIDE_FROM_ABI - bool is_fifo() const { return __get_ft() == file_type::fifo; } + _LIBCPP_HIDE_FROM_ABI bool is_fifo() const { return __get_ft() == file_type::fifo; } - _LIBCPP_HIDE_FROM_ABI - bool is_fifo(error_code& __ec) const noexcept { - return __get_ft(&__ec) == file_type::fifo; - } + _LIBCPP_HIDE_FROM_ABI bool is_fifo(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::fifo; } - _LIBCPP_HIDE_FROM_ABI - bool is_other() const { return filesystem::is_other(file_status{__get_ft()}); } + _LIBCPP_HIDE_FROM_ABI bool is_other() const { return filesystem::is_other(file_status{__get_ft()}); } - _LIBCPP_HIDE_FROM_ABI - bool is_other(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_other(error_code& __ec) const noexcept { return filesystem::is_other(file_status{__get_ft(&__ec)}); } - _LIBCPP_HIDE_FROM_ABI - bool is_regular_file() const { return __get_ft() == file_type::regular; } + _LIBCPP_HIDE_FROM_ABI bool is_regular_file() const { return __get_ft() == file_type::regular; } - _LIBCPP_HIDE_FROM_ABI - bool is_regular_file(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_regular_file(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::regular; } - _LIBCPP_HIDE_FROM_ABI - bool is_socket() const { return __get_ft() == file_type::socket; } + _LIBCPP_HIDE_FROM_ABI bool is_socket() const { return __get_ft() == file_type::socket; } - _LIBCPP_HIDE_FROM_ABI - bool is_socket(error_code& __ec) const noexcept { - return __get_ft(&__ec) == file_type::socket; - } + _LIBCPP_HIDE_FROM_ABI bool is_socket(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::socket; } - _LIBCPP_HIDE_FROM_ABI - bool is_symlink() const { return __get_sym_ft() == file_type::symlink; } + _LIBCPP_HIDE_FROM_ABI bool is_symlink() const { return __get_sym_ft() == file_type::symlink; } - _LIBCPP_HIDE_FROM_ABI - bool is_symlink(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI bool is_symlink(error_code& __ec) const noexcept { return __get_sym_ft(&__ec) == file_type::symlink; } - _LIBCPP_HIDE_FROM_ABI - uintmax_t file_size() const { return __get_size(); } + _LIBCPP_HIDE_FROM_ABI uintmax_t file_size() const { return __get_size(); } - _LIBCPP_HIDE_FROM_ABI - uintmax_t file_size(error_code& __ec) const noexcept { - return __get_size(&__ec); - } + _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(error_code& __ec) const noexcept { return __get_size(&__ec); } - _LIBCPP_HIDE_FROM_ABI - uintmax_t hard_link_count() const { return __get_nlink(); } + _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count() const { return __get_nlink(); } - _LIBCPP_HIDE_FROM_ABI - uintmax_t hard_link_count(error_code& __ec) const noexcept { - return __get_nlink(&__ec); - } + _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(error_code& __ec) const noexcept { return __get_nlink(&__ec); } - _LIBCPP_HIDE_FROM_ABI - file_time_type last_write_time() const { return __get_write_time(); } + _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time() const { return __get_write_time(); } - _LIBCPP_HIDE_FROM_ABI - file_time_type last_write_time(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(error_code& __ec) const noexcept { return __get_write_time(&__ec); } - _LIBCPP_HIDE_FROM_ABI - file_status status() const { return __get_status(); } + _LIBCPP_HIDE_FROM_ABI file_status status() const { return __get_status(); } - _LIBCPP_HIDE_FROM_ABI - file_status status(error_code& __ec) const noexcept { - return __get_status(&__ec); - } + _LIBCPP_HIDE_FROM_ABI file_status status(error_code& __ec) const noexcept { return __get_status(&__ec); } - _LIBCPP_HIDE_FROM_ABI - file_status symlink_status() const { return __get_symlink_status(); } + _LIBCPP_HIDE_FROM_ABI file_status symlink_status() const { return __get_symlink_status(); } - _LIBCPP_HIDE_FROM_ABI - file_status symlink_status(error_code& __ec) const noexcept { + _LIBCPP_HIDE_FROM_ABI file_status symlink_status(error_code& __ec) const noexcept { return __get_symlink_status(&__ec); } + _LIBCPP_HIDE_FROM_ABI bool operator==(directory_entry const& __rhs) const noexcept { return __p_ == __rhs.__p_; } - _LIBCPP_HIDE_FROM_ABI - bool operator==(directory_entry const& __rhs) const noexcept { - return __p_ == __rhs.__p_; - } +# if _LIBCPP_STD_VER <= 17 + _LIBCPP_HIDE_FROM_ABI bool operator!=(directory_entry const& __rhs) const noexcept { return __p_ != __rhs.__p_; } -#if _LIBCPP_STD_VER <= 17 - _LIBCPP_HIDE_FROM_ABI - bool operator!=(directory_entry const& __rhs) const noexcept { - return __p_ != __rhs.__p_; - } + _LIBCPP_HIDE_FROM_ABI bool operator<(directory_entry const& __rhs) const noexcept { return __p_ < __rhs.__p_; } - _LIBCPP_HIDE_FROM_ABI - bool operator<(directory_entry const& __rhs) const noexcept { - return __p_ < __rhs.__p_; - } + _LIBCPP_HIDE_FROM_ABI bool operator<=(directory_entry const& __rhs) const noexcept { return __p_ <= __rhs.__p_; } - _LIBCPP_HIDE_FROM_ABI - bool operator<=(directory_entry const& __rhs) const noexcept { - return __p_ <= __rhs.__p_; - } + _LIBCPP_HIDE_FROM_ABI bool operator>(directory_entry const& __rhs) const noexcept { return __p_ > __rhs.__p_; } - _LIBCPP_HIDE_FROM_ABI - bool operator>(directory_entry const& __rhs) const noexcept { - return __p_ > __rhs.__p_; - } + _LIBCPP_HIDE_FROM_ABI bool operator>=(directory_entry const& __rhs) const noexcept { return __p_ >= __rhs.__p_; } - _LIBCPP_HIDE_FROM_ABI - bool operator>=(directory_entry const& __rhs) const noexcept { - return __p_ >= __rhs.__p_; - } - -#else // _LIBCPP_STD_VER <= 17 +# else // _LIBCPP_STD_VER <= 17 - _LIBCPP_HIDE_FROM_ABI - strong_ordering operator<=>(const directory_entry& __rhs) const noexcept { + _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const directory_entry& __rhs) const noexcept { return __p_ <=> __rhs.__p_; } -#endif // _LIBCPP_STD_VER <= 17 +# endif // _LIBCPP_STD_VER <= 17 template - _LIBCPP_HIDE_FROM_ABI - friend basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, const directory_entry& __d) { + _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>& + operator<<(basic_ostream<_CharT, _Traits>& __os, const directory_entry& __d) { return __os << __d.path(); } @@ -284,23 +214,20 @@ class directory_entry { file_type __type_; _CacheType __cache_type_; - _LIBCPP_HIDE_FROM_ABI - __cached_data() noexcept { __reset(); } + _LIBCPP_HIDE_FROM_ABI __cached_data() noexcept { __reset(); } - _LIBCPP_HIDE_FROM_ABI - void __reset() { + _LIBCPP_HIDE_FROM_ABI void __reset() { __cache_type_ = _Empty; - __type_ = file_type::none; + __type_ = file_type::none; __sym_perms_ = __non_sym_perms_ = perms::unknown; __size_ = __nlink_ = uintmax_t(-1); - __write_time_ = file_time_type::min(); + __write_time_ = file_time_type::min(); } }; - _LIBCPP_HIDE_FROM_ABI - static __cached_data __create_iter_result(file_type __ft) { + _LIBCPP_HIDE_FROM_ABI static __cached_data __create_iter_result(file_type __ft) { __cached_data __data; - __data.__type_ = __ft; + __data.__type_ = __ft; __data.__cache_type_ = [&]() { switch (__ft) { case file_type::none: @@ -314,16 +241,14 @@ class directory_entry { return __data; } - _LIBCPP_HIDE_FROM_ABI - void __assign_iter_entry(_Path&& __p, __cached_data __dt) { - __p_ = std::move(__p); + _LIBCPP_HIDE_FROM_ABI void __assign_iter_entry(_Path&& __p, __cached_data __dt) { + __p_ = std::move(__p); __data_ = __dt; } _LIBCPP_EXPORTED_FROM_ABI error_code __do_refresh() noexcept; - _LIBCPP_HIDE_FROM_ABI - static bool __is_dne_error(error_code const& __ec) { + _LIBCPP_HIDE_FROM_ABI static bool __is_dne_error(error_code const& __ec) { if (!__ec) return true; switch (static_cast(__ec.value())) { @@ -335,9 +260,8 @@ class directory_entry { } } - _LIBCPP_HIDE_FROM_ABI - void __handle_error(const char* __msg, error_code* __dest_ec, - error_code const& __ec, bool __allow_dne = false) const { + _LIBCPP_HIDE_FROM_ABI void + __handle_error(const char* __msg, error_code* __dest_ec, error_code const& __ec, bool __allow_dne = false) const { if (__dest_ec) { *__dest_ec = __ec; return; @@ -346,14 +270,14 @@ class directory_entry { __throw_filesystem_error(__msg, __p_, __ec); } - _LIBCPP_HIDE_FROM_ABI - void __refresh(error_code* __ec = nullptr) { - __handle_error("in directory_entry::refresh", __ec, __do_refresh(), + _LIBCPP_HIDE_FROM_ABI void __refresh(error_code* __ec = nullptr) { + __handle_error("in directory_entry::refresh", + __ec, + __do_refresh(), /*allow_dne*/ true); } - _LIBCPP_HIDE_FROM_ABI - file_type __get_sym_ft(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI file_type __get_sym_ft(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: return __symlink_status(__p_, __ec).type(); @@ -375,8 +299,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - file_type __get_ft(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI file_type __get_ft(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterSymlink: @@ -396,8 +319,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - file_status __get_status(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI file_status __get_status(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterNonSymlink: @@ -411,8 +333,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - file_status __get_symlink_status(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI file_status __get_symlink_status(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterNonSymlink: @@ -427,8 +348,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - uintmax_t __get_size(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI uintmax_t __get_size(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterNonSymlink: @@ -441,10 +361,8 @@ class directory_entry { file_status __st(__get_ft(&__m_ec)); __handle_error("in directory_entry::file_size", __ec, __m_ec); if (filesystem::exists(__st) && !filesystem::is_regular_file(__st)) { - errc __err_kind = filesystem::is_directory(__st) ? errc::is_a_directory - : errc::not_supported; - __handle_error("in directory_entry::file_size", __ec, - make_error_code(__err_kind)); + errc __err_kind = filesystem::is_directory(__st) ? errc::is_a_directory : errc::not_supported; + __handle_error("in directory_entry::file_size", __ec, make_error_code(__err_kind)); } return __data_.__size_; } @@ -452,8 +370,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - uintmax_t __get_nlink(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI uintmax_t __get_nlink(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterNonSymlink: @@ -471,8 +388,7 @@ class directory_entry { __libcpp_unreachable(); } - _LIBCPP_HIDE_FROM_ABI - file_time_type __get_write_time(error_code* __ec = nullptr) const { + _LIBCPP_HIDE_FROM_ABI file_time_type __get_write_time(error_code* __ec = nullptr) const { switch (__data_.__cache_type_) { case _Empty: case _IterNonSymlink: @@ -484,10 +400,8 @@ class directory_entry { error_code __m_ec; file_status __st(__get_ft(&__m_ec)); __handle_error("in directory_entry::last_write_time", __ec, __m_ec); - if (filesystem::exists(__st) && - __data_.__write_time_ == file_time_type::min()) - __handle_error("in directory_entry::last_write_time", __ec, - make_error_code(errc::value_too_large)); + if (filesystem::exists(__st) && __data_.__write_time_ == file_time_type::min()) + __handle_error("in directory_entry::last_write_time", __ec, make_error_code(errc::value_too_large)); return __data_.__write_time_; } } @@ -501,16 +415,13 @@ class directory_entry { class __dir_element_proxy { public: - inline _LIBCPP_HIDE_FROM_ABI directory_entry operator*() { - return std::move(__elem_); - } + inline _LIBCPP_HIDE_FROM_ABI directory_entry operator*() { return std::move(__elem_); } private: friend class directory_iterator; friend class recursive_directory_iterator; _LIBCPP_HIDE_FROM_ABI explicit __dir_element_proxy(directory_entry const& __e) : __elem_(__e) {} - _LIBCPP_HIDE_FROM_ABI __dir_element_proxy(__dir_element_proxy&& __o) - : __elem_(std::move(__o.__elem_)) {} + _LIBCPP_HIDE_FROM_ABI __dir_element_proxy(__dir_element_proxy&& __o) : __elem_(std::move(__o.__elem_)) {} directory_entry __elem_; }; diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h index a2a48e5933076..29bd8da6caa46 100644 --- a/libcxx/include/__filesystem/directory_iterator.h +++ b/libcxx/include/__filesystem/directory_iterator.h @@ -45,33 +45,24 @@ class directory_iterator { typedef input_iterator_tag iterator_category; public: - //ctor & dtor - _LIBCPP_HIDE_FROM_ABI - directory_iterator() noexcept {} + // ctor & dtor + _LIBCPP_HIDE_FROM_ABI directory_iterator() noexcept {} - _LIBCPP_HIDE_FROM_ABI - explicit directory_iterator(const path& __p) - : directory_iterator(__p, nullptr) {} + _LIBCPP_HIDE_FROM_ABI explicit directory_iterator(const path& __p) : directory_iterator(__p, nullptr) {} - _LIBCPP_HIDE_FROM_ABI - directory_iterator(const path& __p, directory_options __opts) + _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, directory_options __opts) : directory_iterator(__p, nullptr, __opts) {} - _LIBCPP_HIDE_FROM_ABI - directory_iterator(const path& __p, error_code& __ec) - : directory_iterator(__p, &__ec) {} + _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, error_code& __ec) : directory_iterator(__p, &__ec) {} - _LIBCPP_HIDE_FROM_ABI - directory_iterator(const path& __p, directory_options __opts, - error_code& __ec) + _LIBCPP_HIDE_FROM_ABI directory_iterator(const path& __p, directory_options __opts, error_code& __ec) : directory_iterator(__p, &__ec, __opts) {} - _LIBCPP_HIDE_FROM_ABI directory_iterator(const directory_iterator&) = default; - _LIBCPP_HIDE_FROM_ABI directory_iterator(directory_iterator&&) = default; + _LIBCPP_HIDE_FROM_ABI directory_iterator(const directory_iterator&) = default; + _LIBCPP_HIDE_FROM_ABI directory_iterator(directory_iterator&&) = default; _LIBCPP_HIDE_FROM_ABI directory_iterator& operator=(const directory_iterator&) = default; - _LIBCPP_HIDE_FROM_ABI - directory_iterator& operator=(directory_iterator&& __o) noexcept { + _LIBCPP_HIDE_FROM_ABI directory_iterator& operator=(directory_iterator&& __o) noexcept { // non-default implementation provided to support self-move assign. if (this != &__o) { __imp_ = std::move(__o.__imp_); @@ -81,27 +72,22 @@ class directory_iterator { _LIBCPP_HIDE_FROM_ABI ~directory_iterator() = default; - _LIBCPP_HIDE_FROM_ABI - const directory_entry& operator*() const { + _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { _LIBCPP_ASSERT_UNCATEGORIZED(__imp_, "The end iterator cannot be dereferenced"); return __dereference(); } - _LIBCPP_HIDE_FROM_ABI - const directory_entry* operator->() const { return &**this; } + _LIBCPP_HIDE_FROM_ABI const directory_entry* operator->() const { return &**this; } - _LIBCPP_HIDE_FROM_ABI - directory_iterator& operator++() { return __increment(); } + _LIBCPP_HIDE_FROM_ABI directory_iterator& operator++() { return __increment(); } - _LIBCPP_HIDE_FROM_ABI - __dir_element_proxy operator++(int) { + _LIBCPP_HIDE_FROM_ABI __dir_element_proxy operator++(int) { __dir_element_proxy __p(**this); __increment(); return __p; } - _LIBCPP_HIDE_FROM_ABI - directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } + _LIBCPP_HIDE_FROM_ABI directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } # if _LIBCPP_STD_VER >= 20 @@ -111,8 +97,7 @@ class directory_iterator { private: inline _LIBCPP_HIDE_FROM_ABI friend bool - operator==(const directory_iterator& __lhs, - const directory_iterator& __rhs) noexcept; + operator==(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept; // construct the dir_stream _LIBCPP_EXPORTED_FROM_ABI directory_iterator(const path&, error_code*, directory_options = directory_options::none); @@ -126,43 +111,35 @@ class directory_iterator { }; inline _LIBCPP_HIDE_FROM_ABI bool -operator==(const directory_iterator& __lhs, - const directory_iterator& __rhs) noexcept { +operator==(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept { return __lhs.__imp_ == __rhs.__imp_; } inline _LIBCPP_HIDE_FROM_ABI bool -operator!=(const directory_iterator& __lhs, - const directory_iterator& __rhs) noexcept { +operator!=(const directory_iterator& __lhs, const directory_iterator& __rhs) noexcept { return !(__lhs == __rhs); } // enable directory_iterator range-based for statements -inline _LIBCPP_HIDE_FROM_ABI directory_iterator -begin(directory_iterator __iter) noexcept { - return __iter; -} +inline _LIBCPP_HIDE_FROM_ABI directory_iterator begin(directory_iterator __iter) noexcept { return __iter; } -inline _LIBCPP_HIDE_FROM_ABI directory_iterator -end(directory_iterator) noexcept { - return directory_iterator(); -} +inline _LIBCPP_HIDE_FROM_ABI directory_iterator end(directory_iterator) noexcept { return directory_iterator(); } _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_END_NAMESPACE_FILESYSTEM -#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_STD_VER >= 20 template <> -_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline constexpr bool std::ranges::enable_borrowed_range = true; +_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool + std::ranges::enable_borrowed_range = true; template <> -_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline constexpr bool std::ranges::enable_view = true; +_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool + std::ranges::enable_view = true; -#endif // _LIBCPP_STD_VER >= 20 +# endif // _LIBCPP_STD_VER >= 20 #endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM) diff --git a/libcxx/include/__filesystem/directory_options.h b/libcxx/include/__filesystem/directory_options.h index 05885bc488b0a..683c4678e083b 100644 --- a/libcxx/include/__filesystem/directory_options.h +++ b/libcxx/include/__filesystem/directory_options.h @@ -21,53 +21,33 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM -enum class directory_options : unsigned char { - none = 0, - follow_directory_symlink = 1, - skip_permission_denied = 2 -}; - -_LIBCPP_HIDE_FROM_ABI -inline constexpr directory_options operator&(directory_options __lhs, - directory_options __rhs) { - return static_cast(static_cast(__lhs) & - static_cast(__rhs)); +enum class directory_options : unsigned char { none = 0, follow_directory_symlink = 1, skip_permission_denied = 2 }; + +_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator&(directory_options __lhs, directory_options __rhs) { + return static_cast(static_cast(__lhs) & static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr directory_options operator|(directory_options __lhs, - directory_options __rhs) { - return static_cast(static_cast(__lhs) | - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator|(directory_options __lhs, directory_options __rhs) { + return static_cast(static_cast(__lhs) | static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr directory_options operator^(directory_options __lhs, - directory_options __rhs) { - return static_cast(static_cast(__lhs) ^ - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator^(directory_options __lhs, directory_options __rhs) { + return static_cast(static_cast(__lhs) ^ static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr directory_options operator~(directory_options __lhs) { +_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator~(directory_options __lhs) { return static_cast(~static_cast(__lhs)); } -_LIBCPP_HIDE_FROM_ABI -inline directory_options& operator&=(directory_options& __lhs, - directory_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline directory_options& operator&=(directory_options& __lhs, directory_options __rhs) { return __lhs = __lhs & __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline directory_options& operator|=(directory_options& __lhs, - directory_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline directory_options& operator|=(directory_options& __lhs, directory_options __rhs) { return __lhs = __lhs | __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline directory_options& operator^=(directory_options& __lhs, - directory_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline directory_options& operator^=(directory_options& __lhs, directory_options __rhs) { return __lhs = __lhs ^ __rhs; } diff --git a/libcxx/include/__filesystem/file_status.h b/libcxx/include/__filesystem/file_status.h index 2a1ce85a37543..3e2b32eef82e0 100644 --- a/libcxx/include/__filesystem/file_status.h +++ b/libcxx/include/__filesystem/file_status.h @@ -26,35 +26,27 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM class _LIBCPP_EXPORTED_FROM_ABI file_status { public: // constructors - _LIBCPP_HIDE_FROM_ABI - file_status() noexcept : file_status(file_type::none) {} - _LIBCPP_HIDE_FROM_ABI - explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept - : __ft_(__ft), - __prms_(__prms) {} + _LIBCPP_HIDE_FROM_ABI file_status() noexcept : file_status(file_type::none) {} + _LIBCPP_HIDE_FROM_ABI explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept + : __ft_(__ft), __prms_(__prms) {} _LIBCPP_HIDE_FROM_ABI file_status(const file_status&) noexcept = default; - _LIBCPP_HIDE_FROM_ABI file_status(file_status&&) noexcept = default; + _LIBCPP_HIDE_FROM_ABI file_status(file_status&&) noexcept = default; - _LIBCPP_HIDE_FROM_ABI - ~file_status() {} + _LIBCPP_HIDE_FROM_ABI ~file_status() {} _LIBCPP_HIDE_FROM_ABI file_status& operator=(const file_status&) noexcept = default; - _LIBCPP_HIDE_FROM_ABI file_status& operator=(file_status&&) noexcept = default; + _LIBCPP_HIDE_FROM_ABI file_status& operator=(file_status&&) noexcept = default; // observers - _LIBCPP_HIDE_FROM_ABI - file_type type() const noexcept { return __ft_; } + _LIBCPP_HIDE_FROM_ABI file_type type() const noexcept { return __ft_; } - _LIBCPP_HIDE_FROM_ABI - perms permissions() const noexcept { return __prms_; } + _LIBCPP_HIDE_FROM_ABI perms permissions() const noexcept { return __prms_; } // modifiers - _LIBCPP_HIDE_FROM_ABI - void type(file_type __ft) noexcept { __ft_ = __ft; } + _LIBCPP_HIDE_FROM_ABI void type(file_type __ft) noexcept { __ft_ = __ft; } - _LIBCPP_HIDE_FROM_ABI - void permissions(perms __p) noexcept { __prms_ = __p; } + _LIBCPP_HIDE_FROM_ABI void permissions(perms __p) noexcept { __prms_ = __p; } # if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__filesystem/file_type.h b/libcxx/include/__filesystem/file_type.h index f456552db5437..c509085d90de0 100644 --- a/libcxx/include/__filesystem/file_type.h +++ b/libcxx/include/__filesystem/file_type.h @@ -24,16 +24,16 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM // On Windows, the library never identifies files as block, character, fifo // or socket. enum class file_type : signed char { - none = 0, + none = 0, not_found = -1, - regular = 1, + regular = 1, directory = 2, - symlink = 3, - block = 4, + symlink = 3, + block = 4, character = 5, - fifo = 6, - socket = 7, - unknown = 8 + fifo = 6, + socket = 7, + unknown = 8 }; _LIBCPP_END_NAMESPACE_FILESYSTEM diff --git a/libcxx/include/__filesystem/operations.h b/libcxx/include/__filesystem/operations.h index ebe816549a9ec..9bb83576f54bc 100644 --- a/libcxx/include/__filesystem/operations.h +++ b/libcxx/include/__filesystem/operations.h @@ -36,15 +36,21 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH _LIBCPP_EXPORTED_FROM_ABI path __absolute(const path&, error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI path __canonical(const path&, error_code* __ec = nullptr); -_LIBCPP_EXPORTED_FROM_ABI bool __copy_file(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr); -_LIBCPP_EXPORTED_FROM_ABI void __copy_symlink(const path& __existing_symlink, const path& __new_symlink, error_code* __ec = nullptr); -_LIBCPP_EXPORTED_FROM_ABI void __copy(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI bool +__copy_file(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI void +__copy_symlink(const path& __existing_symlink, const path& __new_symlink, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI void +__copy(const path& __from, const path& __to, copy_options __opt, error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI bool __create_directories(const path&, error_code* = nullptr); -_LIBCPP_EXPORTED_FROM_ABI void __create_directory_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI void +__create_directory_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI bool __create_directory(const path&, error_code* = nullptr); _LIBCPP_EXPORTED_FROM_ABI bool __create_directory(const path&, const path& __attributes, error_code* = nullptr); -_LIBCPP_EXPORTED_FROM_ABI void __create_hard_link(const path& __to, const path& __new_hard_link, error_code* __ec = nullptr); -_LIBCPP_EXPORTED_FROM_ABI void __create_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI void +__create_hard_link(const path& __to, const path& __new_hard_link, error_code* __ec = nullptr); +_LIBCPP_EXPORTED_FROM_ABI void +__create_symlink(const path& __to, const path& __new_symlink, error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI path __current_path(error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI void __current_path(const path&, error_code* __ec = nullptr); _LIBCPP_EXPORTED_FROM_ABI bool __equivalent(const path&, const path&, error_code* __ec = nullptr); @@ -64,38 +70,85 @@ _LIBCPP_EXPORTED_FROM_ABI path __temp_directory_path(error_code* __ec = nullptr) inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p) { return __absolute(__p); } inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p, error_code& __ec) { return __absolute(__p, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p) { return __canonical(__p); } +inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p) { return __canonical(__p); } inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p, error_code& __ec) { return __canonical(__p, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to) { return __copy_file(__from, __to, copy_options::none); } -inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, error_code& __ec) { return __copy_file(__from, __to, copy_options::none, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, copy_options __opt) { return __copy_file(__from, __to, __opt); } -inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, copy_options __opt, error_code& __ec) { return __copy_file(__from, __to, __opt, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to) { + return __copy_file(__from, __to, copy_options::none); +} +inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, error_code& __ec) { + return __copy_file(__from, __to, copy_options::none, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to, copy_options __opt) { + return __copy_file(__from, __to, __opt); +} +inline _LIBCPP_HIDE_FROM_ABI bool +copy_file(const path& __from, const path& __to, copy_options __opt, error_code& __ec) { + return __copy_file(__from, __to, __opt, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI void copy_symlink(const path& __from, const path& __to) { __copy_symlink(__from, __to); } -inline _LIBCPP_HIDE_FROM_ABI void copy_symlink(const path& __from, const path& __to, error_code& __ec) noexcept { __copy_symlink(__from, __to, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to) { __copy(__from, __to, copy_options::none); } -inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, error_code& __ec) { __copy(__from, __to, copy_options::none, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt) { __copy(__from, __to, __opt); } -inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt, error_code& __ec) { __copy(__from, __to, __opt, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void copy_symlink(const path& __from, const path& __to, error_code& __ec) noexcept { + __copy_symlink(__from, __to, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to) { + __copy(__from, __to, copy_options::none); +} +inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, error_code& __ec) { + __copy(__from, __to, copy_options::none, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt) { + __copy(__from, __to, __opt); +} +inline _LIBCPP_HIDE_FROM_ABI void copy(const path& __from, const path& __to, copy_options __opt, error_code& __ec) { + __copy(__from, __to, __opt, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool create_directories(const path& __p) { return __create_directories(__p); } -inline _LIBCPP_HIDE_FROM_ABI bool create_directories(const path& __p, error_code& __ec) { return __create_directories(__p, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void create_directory_symlink(const path& __target, const path& __link) { __create_directory_symlink(__target, __link); } -inline _LIBCPP_HIDE_FROM_ABI void create_directory_symlink(const path& __target, const path& __link, error_code& __ec) noexcept { __create_directory_symlink(__target, __link, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI bool create_directories(const path& __p, error_code& __ec) { + return __create_directories(__p, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void create_directory_symlink(const path& __target, const path& __link) { + __create_directory_symlink(__target, __link); +} +inline _LIBCPP_HIDE_FROM_ABI void +create_directory_symlink(const path& __target, const path& __link, error_code& __ec) noexcept { + __create_directory_symlink(__target, __link, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p) { return __create_directory(__p); } -inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, error_code& __ec) noexcept { return __create_directory(__p, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs) { return __create_directory(__p, __attrs); } -inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs, error_code& __ec) noexcept { return __create_directory(__p, __attrs, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void create_hard_link(const path& __target, const path& __link) { __create_hard_link(__target, __link); } -inline _LIBCPP_HIDE_FROM_ABI void create_hard_link(const path& __target, const path& __link, error_code& __ec) noexcept { __create_hard_link(__target, __link, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link) { __create_symlink(__target, __link); } -inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link, error_code& __ec) noexcept { return __create_symlink(__target, __link, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, error_code& __ec) noexcept { + return __create_directory(__p, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs) { + return __create_directory(__p, __attrs); +} +inline _LIBCPP_HIDE_FROM_ABI bool create_directory(const path& __p, const path& __attrs, error_code& __ec) noexcept { + return __create_directory(__p, __attrs, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void create_hard_link(const path& __target, const path& __link) { + __create_hard_link(__target, __link); +} +inline _LIBCPP_HIDE_FROM_ABI void +create_hard_link(const path& __target, const path& __link, error_code& __ec) noexcept { + __create_hard_link(__target, __link, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link) { + __create_symlink(__target, __link); +} +inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link, error_code& __ec) noexcept { + return __create_symlink(__target, __link, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI path current_path() { return __current_path(); } inline _LIBCPP_HIDE_FROM_ABI path current_path(error_code& __ec) { return __current_path(&__ec); } inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p) { __current_path(__p); } -inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p, error_code& __ec) noexcept { __current_path(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p, error_code& __ec) noexcept { + __current_path(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2) { return __equivalent(__p1, __p2); } -inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept { return __equivalent(__p1, __p2, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept { + return __equivalent(__p1, __p2, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool status_known(file_status __s) noexcept { return __s.type() != file_type::none; } -inline _LIBCPP_HIDE_FROM_ABI bool exists(file_status __s) noexcept { return status_known(__s) && __s.type() != file_type::not_found; } +inline _LIBCPP_HIDE_FROM_ABI bool exists(file_status __s) noexcept { + return status_known(__s) && __s.type() != file_type::not_found; +} inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p) { return exists(__status(__p)); } inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p, error_code& __ec) noexcept { @@ -106,44 +159,79 @@ inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p, error_code& __ec) noex } inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p) { return __file_size(__p); } -inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p, error_code& __ec) noexcept { return __file_size(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p, error_code& __ec) noexcept { + return __file_size(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p) { return __hard_link_count(__p); } -inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept { return __hard_link_count(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept { + return __hard_link_count(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(file_status __s) noexcept { return __s.type() == file_type::block; } inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p) { return is_block_file(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p, error_code& __ec) noexcept { return is_block_file(__status(__p, &__ec)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(file_status __s) noexcept { return __s.type() == file_type::character; } +inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p, error_code& __ec) noexcept { + return is_block_file(__status(__p, &__ec)); +} +inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(file_status __s) noexcept { + return __s.type() == file_type::character; +} inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p) { return is_character_file(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p, error_code& __ec) noexcept { return is_character_file(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p, error_code& __ec) noexcept { + return is_character_file(__status(__p, &__ec)); +} inline _LIBCPP_HIDE_FROM_ABI bool is_directory(file_status __s) noexcept { return __s.type() == file_type::directory; } inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p) { return is_directory(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p, error_code& __ec) noexcept { return is_directory(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p, error_code& __ec) noexcept { + return is_directory(__status(__p, &__ec)); +} _LIBCPP_EXPORTED_FROM_ABI bool __fs_is_empty(const path& __p, error_code* __ec = nullptr); inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p) { return __fs_is_empty(__p); } inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p, error_code& __ec) { return __fs_is_empty(__p, &__ec); } inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(file_status __s) noexcept { return __s.type() == file_type::fifo; } inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p) { return is_fifo(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p, error_code& __ec) noexcept { return is_fifo(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p, error_code& __ec) noexcept { + return is_fifo(__status(__p, &__ec)); +} inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(file_status __s) noexcept { return __s.type() == file_type::regular; } inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p) { return is_regular_file(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p, error_code& __ec) noexcept { return is_regular_file(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p, error_code& __ec) noexcept { + return is_regular_file(__status(__p, &__ec)); +} inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(file_status __s) noexcept { return __s.type() == file_type::symlink; } inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p) { return is_symlink(__symlink_status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p, error_code& __ec) noexcept { return is_symlink(__symlink_status(__p, &__ec)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_other(file_status __s) noexcept { return exists(__s) && !is_regular_file(__s) && !is_directory(__s) && !is_symlink(__s); } +inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p, error_code& __ec) noexcept { + return is_symlink(__symlink_status(__p, &__ec)); +} +inline _LIBCPP_HIDE_FROM_ABI bool is_other(file_status __s) noexcept { + return exists(__s) && !is_regular_file(__s) && !is_directory(__s) && !is_symlink(__s); +} inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p) { return is_other(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p, error_code& __ec) noexcept { return is_other(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p, error_code& __ec) noexcept { + return is_other(__status(__p, &__ec)); +} inline _LIBCPP_HIDE_FROM_ABI bool is_socket(file_status __s) noexcept { return __s.type() == file_type::socket; } inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p) { return is_socket(__status(__p)); } -inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p, error_code& __ec) noexcept { return is_socket(__status(__p, &__ec)); } +inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p, error_code& __ec) noexcept { + return is_socket(__status(__p, &__ec)); +} inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p) { return __last_write_time(__p); } -inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p, error_code& __ec) noexcept { return __last_write_time(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p, error_code& __ec) noexcept { + return __last_write_time(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t) { __last_write_time(__p, __t); } -inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t, error_code& __ec) noexcept { __last_write_time(__p, __t, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t, error_code& __ec) noexcept { + __last_write_time(__p, __t, &__ec); +} _LIBCPP_EXPORTED_FROM_ABI void __permissions(const path&, perms, perm_options, error_code* = nullptr); -inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, perm_options __opts = perm_options::replace) { __permissions(__p, __prms, __opts); } -inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, error_code& __ec) noexcept { __permissions(__p, __prms, perm_options::replace, &__ec); } -inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, perm_options __opts, error_code& __ec) { __permissions(__p, __prms, __opts, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void +permissions(const path& __p, perms __prms, perm_options __opts = perm_options::replace) { + __permissions(__p, __prms, __opts); +} +inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, error_code& __ec) noexcept { + __permissions(__p, __prms, perm_options::replace, &__ec); +} +inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, perm_options __opts, error_code& __ec) { + __permissions(__p, __prms, __opts, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base, error_code& __ec) { path __tmp = __weakly_canonical(__p, &__ec); @@ -155,8 +243,12 @@ inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base, return __tmp.lexically_proximate(__tmp_base); } -inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, error_code& __ec) { return proximate(__p, current_path(), __ec); } -inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base = current_path()) { return __weakly_canonical(__p).lexically_proximate(__weakly_canonical(__base)); } +inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, error_code& __ec) { + return proximate(__p, current_path(), __ec); +} +inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base = current_path()) { + return __weakly_canonical(__p).lexically_proximate(__weakly_canonical(__base)); +} inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p) { return __read_symlink(__p); } inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p, error_code& __ec) { return __read_symlink(__p, &__ec); } @@ -170,27 +262,45 @@ inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base, return __tmp.lexically_relative(__tmpbase); } -inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, error_code& __ec) { return relative(__p, current_path(), __ec); } -inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base = current_path()) { return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base)); } +inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, error_code& __ec) { + return relative(__p, current_path(), __ec); +} +inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base = current_path()) { + return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base)); +} inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p) { return __remove_all(__p); } -inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p, error_code& __ec) { return __remove_all(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p, error_code& __ec) { + return __remove_all(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI bool remove(const path& __p) { return __remove(__p); } inline _LIBCPP_HIDE_FROM_ABI bool remove(const path& __p, error_code& __ec) noexcept { return __remove(__p, &__ec); } inline _LIBCPP_HIDE_FROM_ABI void rename(const path& __from, const path& __to) { return __rename(__from, __to); } -inline _LIBCPP_HIDE_FROM_ABI void rename(const path& __from, const path& __to, error_code& __ec) noexcept { return __rename(__from, __to, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void rename(const path& __from, const path& __to, error_code& __ec) noexcept { + return __rename(__from, __to, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns) { return __resize_file(__p, __ns); } -inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept { return __resize_file(__p, __ns, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept { + return __resize_file(__p, __ns, &__ec); +} _LIBCPP_EXPORTED_FROM_ABI space_info __space(const path&, error_code* __ec = nullptr); inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p) { return __space(__p); } -inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p, error_code& __ec) noexcept { return __space(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p, error_code& __ec) noexcept { + return __space(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p) { return __status(__p); } -inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p, error_code& __ec) noexcept { return __status(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p, error_code& __ec) noexcept { + return __status(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p) { return __symlink_status(__p); } -inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p, error_code& __ec) noexcept { return __symlink_status(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p, error_code& __ec) noexcept { + return __symlink_status(__p, &__ec); +} inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path() { return __temp_directory_path(); } inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path(error_code& __ec) { return __temp_directory_path(&__ec); } inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p) { return __weakly_canonical(__p); } -inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p, error_code& __ec) { return __weakly_canonical(__p, &__ec); } +inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p, error_code& __ec) { + return __weakly_canonical(__p, &__ec); +} _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h index 466fa25a38ed1..1ff992dd64e6d 100644 --- a/libcxx/include/__filesystem/path.h +++ b/libcxx/include/__filesystem/path.h @@ -28,8 +28,8 @@ #include #if !defined(_LIBCPP_HAS_NO_LOCALIZATION) -# include // for quoted -# include +# include // for quoted +# include #endif #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -51,47 +51,45 @@ struct __can_convert_char : public __can_convert_char<_Tp> {}; template <> struct __can_convert_char { static const bool value = true; - using __char_type = char; + using __char_type = char; }; template <> struct __can_convert_char { static const bool value = true; - using __char_type = wchar_t; + using __char_type = wchar_t; }; -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T template <> struct __can_convert_char { static const bool value = true; - using __char_type = char8_t; + using __char_type = char8_t; }; -#endif +# endif template <> struct __can_convert_char { static const bool value = true; - using __char_type = char16_t; + using __char_type = char16_t; }; template <> struct __can_convert_char { static const bool value = true; - using __char_type = char32_t; + using __char_type = char32_t; }; template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI -bool -__is_separator(_ECharT __e) { -#if defined(_LIBCPP_WIN32API) +_LIBCPP_HIDE_FROM_ABI bool __is_separator(_ECharT __e) { +# if defined(_LIBCPP_WIN32API) return __e == _ECharT('/') || __e == _ECharT('\\'); -#else +# else return __e == _ECharT('/'); -#endif +# endif } -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T typedef u8string __u8_string; -#else +# else typedef string __u8_string; -#endif +# endif struct _NullSentinel {}; @@ -102,99 +100,75 @@ template struct __is_pathable_string : public false_type {}; template -struct __is_pathable_string< - basic_string<_ECharT, _Traits, _Alloc>, - _Void::__char_type> > +struct __is_pathable_string< basic_string<_ECharT, _Traits, _Alloc>, + _Void::__char_type> > : public __can_convert_char<_ECharT> { using _Str = basic_string<_ECharT, _Traits, _Alloc>; - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_end(_Str const& __s) { - return __s.data() + __s.length(); - } + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(_Str const& __s) { return __s.data() + __s.length(); } - _LIBCPP_HIDE_FROM_ABI - static _ECharT __first_or_null(_Str const& __s) { - return __s.empty() ? _ECharT{} : __s[0]; - } + _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Str const& __s) { return __s.empty() ? _ECharT{} : __s[0]; } }; template -struct __is_pathable_string< - basic_string_view<_ECharT, _Traits>, - _Void::__char_type> > +struct __is_pathable_string< basic_string_view<_ECharT, _Traits>, + _Void::__char_type> > : public __can_convert_char<_ECharT> { using _Str = basic_string_view<_ECharT, _Traits>; - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_end(_Str const& __s) { - return __s.data() + __s.length(); - } + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(_Str const& __s) { return __s.data() + __s.length(); } - _LIBCPP_HIDE_FROM_ABI - static _ECharT __first_or_null(_Str const& __s) { - return __s.empty() ? _ECharT{} : __s[0]; - } + _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Str const& __s) { return __s.empty() ? _ECharT{} : __s[0]; } }; -template , - class _UnqualPtrType = - __remove_const_t<__remove_pointer_t<_DS> >, - bool _IsCharPtr = is_pointer<_DS>::value&& - __can_convert_char<_UnqualPtrType>::value> +template , + class _UnqualPtrType = __remove_const_t<__remove_pointer_t<_DS> >, + bool _IsCharPtr = is_pointer<_DS>::value && __can_convert_char<_UnqualPtrType>::value> struct __is_pathable_char_array : false_type {}; template -struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true> - : __can_convert_char<__remove_const_t<_ECharT> > { - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_begin(const _ECharT* __b) { return __b; } - - _LIBCPP_HIDE_FROM_ABI - static _ECharT const* __range_end(const _ECharT* __b) { - using _Iter = const _ECharT*; +struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true> : __can_convert_char<__remove_const_t<_ECharT> > { + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_begin(const _ECharT* __b) { return __b; } + + _LIBCPP_HIDE_FROM_ABI static _ECharT const* __range_end(const _ECharT* __b) { + using _Iter = const _ECharT*; const _ECharT __sentinel = _ECharT{}; - _Iter __e = __b; + _Iter __e = __b; for (; *__e != __sentinel; ++__e) ; return __e; } - _LIBCPP_HIDE_FROM_ABI - static _ECharT __first_or_null(const _ECharT* __b) { return *__b; } + _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(const _ECharT* __b) { return *__b; } }; -template ::value, - class = void> +template ::value, class = void> struct __is_pathable_iter : false_type {}; template struct __is_pathable_iter< - _Iter, true, - _Void::value_type>::__char_type> > + _Iter, + true, + _Void::value_type>::__char_type> > : __can_convert_char::value_type> { using _ECharT = typename iterator_traits<_Iter>::value_type; - _LIBCPP_HIDE_FROM_ABI - static _Iter __range_begin(_Iter __b) { return __b; } + _LIBCPP_HIDE_FROM_ABI static _Iter __range_begin(_Iter __b) { return __b; } - _LIBCPP_HIDE_FROM_ABI - static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; } + _LIBCPP_HIDE_FROM_ABI static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; } - _LIBCPP_HIDE_FROM_ABI - static _ECharT __first_or_null(_Iter __b) { return *__b; } + _LIBCPP_HIDE_FROM_ABI static _ECharT __first_or_null(_Iter __b) { return *__b; } }; -template ::value, +template ::value, bool _IsCharIterT = __is_pathable_char_array<_Tp>::value, - bool _IsIterT = !_IsCharIterT && __is_pathable_iter<_Tp>::value> + bool _IsIterT = !_IsCharIterT && __is_pathable_iter<_Tp>::value> struct __is_pathable : false_type { static_assert(!_IsStringT && !_IsCharIterT && !_IsIterT, "Must all be false"); }; @@ -203,72 +177,64 @@ template struct __is_pathable<_Tp, true, false, false> : __is_pathable_string<_Tp> {}; template -struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> { -}; +struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> {}; template struct __is_pathable<_Tp, false, false, true> : __is_pathable_iter<_Tp> {}; -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) typedef wstring __path_string; typedef wchar_t __path_value; -#else +# else typedef string __path_string; typedef char __path_value; -#endif +# endif -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) _LIBCPP_EXPORTED_FROM_ABI size_t __wide_to_char(const wstring&, char*, size_t); _LIBCPP_EXPORTED_FROM_ABI size_t __char_to_wide(const string&, wchar_t*, size_t); -#endif +# endif template struct _PathCVT; -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# if !defined(_LIBCPP_HAS_NO_LOCALIZATION) template struct _PathCVT { - static_assert(__can_convert_char<_ECharT>::value, - "Char type not convertible"); + static_assert(__can_convert_char<_ECharT>::value, "Char type not convertible"); typedef __narrow_to_utf8 _Narrower; -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) typedef __widen_from_utf8 _Widener; -#endif +# endif - _LIBCPP_HIDE_FROM_ABI - static void __append_range(__path_string& __dest, _ECharT const* __b, - _ECharT const* __e) { -#if defined(_LIBCPP_WIN32API) + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _ECharT const* __b, _ECharT const* __e) { +# if defined(_LIBCPP_WIN32API) string __utf8; _Narrower()(back_inserter(__utf8), __b, __e); _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); -#else +# else _Narrower()(back_inserter(__dest), __b, __e); -#endif +# endif } template - _LIBCPP_HIDE_FROM_ABI - static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload"); if (__b == __e) return; basic_string<_ECharT> __tmp(__b, __e); -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) string __utf8; - _Narrower()(back_inserter(__utf8), __tmp.data(), - __tmp.data() + __tmp.length()); + _Narrower()(back_inserter(__utf8), __tmp.data(), __tmp.data() + __tmp.length()); _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); -#else - _Narrower()(back_inserter(__dest), __tmp.data(), - __tmp.data() + __tmp.length()); -#endif +# else + _Narrower()(back_inserter(__dest), __tmp.data(), __tmp.data() + __tmp.length()); +# endif } template - _LIBCPP_HIDE_FROM_ABI - static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload"); const _ECharT __sentinel = _ECharT{}; if (*__b == __sentinel) @@ -276,94 +242,74 @@ struct _PathCVT { basic_string<_ECharT> __tmp; for (; *__b != __sentinel; ++__b) __tmp.push_back(*__b); -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) string __utf8; - _Narrower()(back_inserter(__utf8), __tmp.data(), - __tmp.data() + __tmp.length()); + _Narrower()(back_inserter(__utf8), __tmp.data(), __tmp.data() + __tmp.length()); _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); -#else - _Narrower()(back_inserter(__dest), __tmp.data(), - __tmp.data() + __tmp.length()); -#endif +# else + _Narrower()(back_inserter(__dest), __tmp.data(), __tmp.data() + __tmp.length()); +# endif } template - _LIBCPP_HIDE_FROM_ABI - static void __append_source(__path_string& __dest, _Source const& __s) { + _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) { using _Traits = __is_pathable<_Source>; - __append_range(__dest, _Traits::__range_begin(__s), - _Traits::__range_end(__s)); + __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s)); } }; -#endif // !_LIBCPP_HAS_NO_LOCALIZATION +# endif // !_LIBCPP_HAS_NO_LOCALIZATION template <> struct _PathCVT<__path_value> { - template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static void - __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { for (; __b != __e; ++__b) __dest.push_back(*__b); } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static void - __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { __dest.append(__b, __e); } template - _LIBCPP_HIDE_FROM_ABI - static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { const char __sentinel = char{}; for (; *__b != __sentinel; ++__b) __dest.push_back(*__b); } template - _LIBCPP_HIDE_FROM_ABI - static void __append_source(__path_string& __dest, _Source const& __s) { + _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) { using _Traits = __is_pathable<_Source>; - __append_range(__dest, _Traits::__range_begin(__s), - _Traits::__range_end(__s)); + __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s)); } }; -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) template <> struct _PathCVT { - - _LIBCPP_HIDE_FROM_ABI - static void - __append_string(__path_string& __dest, const basic_string &__str) { - size_t __size = __char_to_wide(__str, nullptr, 0); - size_t __pos = __dest.size(); - __dest.resize(__pos + __size); - __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size); + _LIBCPP_HIDE_FROM_ABI static void __append_string(__path_string& __dest, const basic_string& __str) { + size_t __size = __char_to_wide(__str, nullptr, 0); + size_t __pos = __dest.size(); + __dest.resize(__pos + __size); + __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size); } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static void - __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { basic_string __tmp(__b, __e); __append_string(__dest, __tmp); } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static void - __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { basic_string __tmp(__b, __e); __append_string(__dest, __tmp); } template - _LIBCPP_HIDE_FROM_ABI - static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + _LIBCPP_HIDE_FROM_ABI static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { const char __sentinel = char{}; basic_string __tmp; for (; *__b != __sentinel; ++__b) @@ -372,11 +318,9 @@ struct _PathCVT { } template - _LIBCPP_HIDE_FROM_ABI - static void __append_source(__path_string& __dest, _Source const& __s) { + _LIBCPP_HIDE_FROM_ABI static void __append_source(__path_string& __dest, _Source const& __s) { using _Traits = __is_pathable<_Source>; - __append_range(__dest, _Traits::__range_begin(__s), - _Traits::__range_end(__s)); + __append_range(__dest, _Traits::__range_begin(__s), _Traits::__range_end(__s)); } }; @@ -386,8 +330,7 @@ struct _PathExport { typedef __widen_from_utf8 _Widener; template - _LIBCPP_HIDE_FROM_ABI - static void __append(_Str& __dest, const __path_string& __src) { + _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) { string __utf8; _Narrower()(back_inserter(__utf8), __src.data(), __src.data() + __src.size()); _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); @@ -397,10 +340,9 @@ struct _PathExport { template <> struct _PathExport { template - _LIBCPP_HIDE_FROM_ABI - static void __append(_Str& __dest, const __path_string& __src) { + _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) { size_t __size = __wide_to_char(__src, nullptr, 0); - size_t __pos = __dest.size(); + size_t __pos = __dest.size(); __dest.resize(__size); __wide_to_char(__src, const_cast(__dest.data()) + __pos, __size); } @@ -409,8 +351,7 @@ struct _PathExport { template <> struct _PathExport { template - _LIBCPP_HIDE_FROM_ABI - static void __append(_Str& __dest, const __path_string& __src) { + _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) { __dest.append(__src.begin(), __src.end()); } }; @@ -418,25 +359,23 @@ struct _PathExport { template <> struct _PathExport { template - _LIBCPP_HIDE_FROM_ABI - static void __append(_Str& __dest, const __path_string& __src) { + _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) { __dest.append(__src.begin(), __src.end()); } }; -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T template <> struct _PathExport { typedef __narrow_to_utf8 _Narrower; template - _LIBCPP_HIDE_FROM_ABI - static void __append(_Str& __dest, const __path_string& __src) { + _LIBCPP_HIDE_FROM_ABI static void __append(_Str& __dest, const __path_string& __src) { _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size()); } }; -#endif /* !_LIBCPP_HAS_NO_CHAR8_T */ -#endif /* _LIBCPP_WIN32API */ +# endif /* !_LIBCPP_HAS_NO_CHAR8_T */ +# endif /* _LIBCPP_WIN32API */ class _LIBCPP_EXPORTED_FROM_ABI path { template @@ -449,101 +388,84 @@ class _LIBCPP_EXPORTED_FROM_ABI path { using _SourceCVT = _PathCVT<_SourceChar<_Tp> >; public: -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) typedef wchar_t value_type; static constexpr value_type preferred_separator = L'\\'; -#else +# else typedef char value_type; static constexpr value_type preferred_separator = '/'; -#endif +# endif typedef basic_string string_type; typedef basic_string_view __string_view; - enum format : unsigned char { - auto_format, - native_format, - generic_format - }; + enum format : unsigned char { auto_format, native_format, generic_format }; // constructors and destructor _LIBCPP_HIDE_FROM_ABI path() noexcept {} _LIBCPP_HIDE_FROM_ABI path(const path& __p) : __pn_(__p.__pn_) {} - _LIBCPP_HIDE_FROM_ABI path(path&& __p) noexcept - : __pn_(std::move(__p.__pn_)) {} + _LIBCPP_HIDE_FROM_ABI path(path&& __p) noexcept : __pn_(std::move(__p.__pn_)) {} - _LIBCPP_HIDE_FROM_ABI - path(string_type&& __s, format = format::auto_format) noexcept - : __pn_(std::move(__s)) {} + _LIBCPP_HIDE_FROM_ABI path(string_type&& __s, format = format::auto_format) noexcept : __pn_(std::move(__s)) {} template > - _LIBCPP_HIDE_FROM_ABI - path(const _Source& __src, format = format::auto_format) { + _LIBCPP_HIDE_FROM_ABI path(const _Source& __src, format = format::auto_format) { _SourceCVT<_Source>::__append_source(__pn_, __src); } template - _LIBCPP_HIDE_FROM_ABI - path(_InputIt __first, _InputIt __last, format = format::auto_format) { + _LIBCPP_HIDE_FROM_ABI path(_InputIt __first, _InputIt __last, format = format::auto_format) { typedef typename iterator_traits<_InputIt>::value_type _ItVal; _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); } -/* -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) - // TODO Implement locale conversions. - template > - path(const _Source& __src, const locale& __loc, format = format::auto_format); - template - path(_InputIt __first, _InputIt _last, const locale& __loc, - format = format::auto_format); -#endif -*/ + /* + #if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + // TODO Implement locale conversions. + template > + path(const _Source& __src, const locale& __loc, format = format::auto_format); + template + path(_InputIt __first, _InputIt _last, const locale& __loc, + format = format::auto_format); + #endif + */ - _LIBCPP_HIDE_FROM_ABI - ~path() = default; + _LIBCPP_HIDE_FROM_ABI ~path() = default; // assignments - _LIBCPP_HIDE_FROM_ABI - path& operator=(const path& __p) { + _LIBCPP_HIDE_FROM_ABI path& operator=(const path& __p) { __pn_ = __p.__pn_; return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator=(path&& __p) noexcept { + _LIBCPP_HIDE_FROM_ABI path& operator=(path&& __p) noexcept { __pn_ = std::move(__p.__pn_); return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator=(string_type&& __s) noexcept { + _LIBCPP_HIDE_FROM_ABI path& operator=(string_type&& __s) noexcept { __pn_ = std::move(__s); return *this; } - _LIBCPP_HIDE_FROM_ABI - path& assign(string_type&& __s) noexcept { + _LIBCPP_HIDE_FROM_ABI path& assign(string_type&& __s) noexcept { __pn_ = std::move(__s); return *this; } template - _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> - operator=(const _Source& __src) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator=(const _Source& __src) { return this->assign(__src); } template - _LIBCPP_HIDE_FROM_ABI - _EnableIfPathable<_Source> assign(const _Source& __src) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> assign(const _Source& __src) { __pn_.clear(); _SourceCVT<_Source>::__append_source(__pn_, __src); return *this; } template - _LIBCPP_HIDE_FROM_ABI - path& assign(_InputIt __first, _InputIt __last) { + _LIBCPP_HIDE_FROM_ABI path& assign(_InputIt __first, _InputIt __last) { typedef typename iterator_traits<_InputIt>::value_type _ItVal; __pn_.clear(); _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); @@ -552,19 +474,17 @@ class _LIBCPP_EXPORTED_FROM_ABI path { public: // appends -#if defined(_LIBCPP_WIN32API) - _LIBCPP_HIDE_FROM_ABI - path& operator/=(const path& __p) { - auto __p_root_name = __p.__root_name(); +# if defined(_LIBCPP_WIN32API) + _LIBCPP_HIDE_FROM_ABI path& operator/=(const path& __p) { + auto __p_root_name = __p.__root_name(); auto __p_root_name_size = __p_root_name.size(); - if (__p.is_absolute() || - (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) { + if (__p.is_absolute() || (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) { __pn_ = __p.__pn_; return *this; } if (__p.has_root_directory()) { path __root_name_str = root_name(); - __pn_ = __root_name_str.native(); + __pn_ = __root_name_str.native(); __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size); return *this; } @@ -574,25 +494,21 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return *this; } template - _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> - operator/=(const _Source& __src) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator/=(const _Source& __src) { return operator/=(path(__src)); } template - _LIBCPP_HIDE_FROM_ABI - _EnableIfPathable<_Source> append(const _Source& __src) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> append(const _Source& __src) { return operator/=(path(__src)); } template - _LIBCPP_HIDE_FROM_ABI - path& append(_InputIt __first, _InputIt __last) { + _LIBCPP_HIDE_FROM_ABI path& append(_InputIt __first, _InputIt __last) { return operator/=(path(__first, __last)); } -#else - _LIBCPP_HIDE_FROM_ABI - path& operator/=(const path& __p) { +# else + _LIBCPP_HIDE_FROM_ABI path& operator/=(const path& __p) { if (__p.is_absolute()) { __pn_ = __p.__pn_; return *this; @@ -607,16 +523,14 @@ class _LIBCPP_EXPORTED_FROM_ABI path { // is known at compile time to be "/' since the user almost certainly intended // to append a separator instead of overwriting the path with "/" template - _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> - operator/=(const _Source& __src) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator/=(const _Source& __src) { return this->append(__src); } template - _LIBCPP_HIDE_FROM_ABI - _EnableIfPathable<_Source> append(const _Source& __src) { - using _Traits = __is_pathable<_Source>; - using _CVT = _PathCVT<_SourceChar<_Source> >; + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> append(const _Source& __src) { + using _Traits = __is_pathable<_Source>; + using _CVT = _PathCVT<_SourceChar<_Source> >; bool __source_is_absolute = filesystem::__is_separator(_Traits::__first_or_null(__src)); if (__source_is_absolute) __pn_.clear(); @@ -627,8 +541,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path { } template - _LIBCPP_HIDE_FROM_ABI - path& append(_InputIt __first, _InputIt __last) { + _LIBCPP_HIDE_FROM_ABI path& append(_InputIt __first, _InputIt __last) { typedef typename iterator_traits<_InputIt>::value_type _ItVal; static_assert(__can_convert_char<_ItVal>::value, "Must convertible"); using _CVT = _PathCVT<_ItVal>; @@ -639,91 +552,76 @@ class _LIBCPP_EXPORTED_FROM_ABI path { _CVT::__append_range(__pn_, __first, __last); return *this; } -#endif +# endif // concatenation - _LIBCPP_HIDE_FROM_ABI - path& operator+=(const path& __x) { + _LIBCPP_HIDE_FROM_ABI path& operator+=(const path& __x) { __pn_ += __x.__pn_; return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator+=(const string_type& __x) { + _LIBCPP_HIDE_FROM_ABI path& operator+=(const string_type& __x) { __pn_ += __x; return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator+=(__string_view __x) { + _LIBCPP_HIDE_FROM_ABI path& operator+=(__string_view __x) { __pn_ += __x; return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator+=(const value_type* __x) { + _LIBCPP_HIDE_FROM_ABI path& operator+=(const value_type* __x) { __pn_ += __x; return *this; } - _LIBCPP_HIDE_FROM_ABI - path& operator+=(value_type __x) { + _LIBCPP_HIDE_FROM_ABI path& operator+=(value_type __x) { __pn_ += __x; return *this; } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - path& - operator+=(_ECharT __x) { - _PathCVT<_ECharT>::__append_source(__pn_, - basic_string_view<_ECharT>(&__x, 1)); + _LIBCPP_HIDE_FROM_ABI path& operator+=(_ECharT __x) { + _PathCVT<_ECharT>::__append_source(__pn_, basic_string_view<_ECharT>(&__x, 1)); return *this; } template - _LIBCPP_HIDE_FROM_ABI - _EnableIfPathable<_Source> operator+=(const _Source& __x) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> operator+=(const _Source& __x) { return this->concat(__x); } template - _LIBCPP_HIDE_FROM_ABI - _EnableIfPathable<_Source> concat(const _Source& __x) { + _LIBCPP_HIDE_FROM_ABI _EnableIfPathable<_Source> concat(const _Source& __x) { _SourceCVT<_Source>::__append_source(__pn_, __x); return *this; } template - _LIBCPP_HIDE_FROM_ABI - path& concat(_InputIt __first, _InputIt __last) { + _LIBCPP_HIDE_FROM_ABI path& concat(_InputIt __first, _InputIt __last) { typedef typename iterator_traits<_InputIt>::value_type _ItVal; _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); return *this; } // modifiers - _LIBCPP_HIDE_FROM_ABI - void clear() noexcept { __pn_.clear(); } + _LIBCPP_HIDE_FROM_ABI void clear() noexcept { __pn_.clear(); } - _LIBCPP_HIDE_FROM_ABI - path& make_preferred() { -#if defined(_LIBCPP_WIN32API) + _LIBCPP_HIDE_FROM_ABI path& make_preferred() { +# if defined(_LIBCPP_WIN32API) std::replace(__pn_.begin(), __pn_.end(), L'/', L'\\'); -#endif +# endif return *this; } - _LIBCPP_HIDE_FROM_ABI - path& remove_filename() { + _LIBCPP_HIDE_FROM_ABI path& remove_filename() { auto __fname = __filename(); if (!__fname.empty()) __pn_.erase(__fname.data() - __pn_.data()); return *this; } - _LIBCPP_HIDE_FROM_ABI - path& replace_filename(const path& __replacement) { + _LIBCPP_HIDE_FROM_ABI path& replace_filename(const path& __replacement) { remove_filename(); return (*this /= __replacement); } @@ -749,7 +647,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path { friend _LIBCPP_HIDE_FROM_ABI bool operator>=(const path& __lhs, const path& __rhs) noexcept { return __lhs.__compare(__rhs.__pn_) >= 0; } -# else // _LIBCPP_STD_VER <= 17 +# else // _LIBCPP_STD_VER <= 17 friend _LIBCPP_HIDE_FROM_ABI strong_ordering operator<=>(const path& __lhs, const path& __rhs) noexcept { return __lhs.__compare(__rhs.__pn_) <=> 0; } @@ -761,39 +659,31 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return __result; } - _LIBCPP_HIDE_FROM_ABI - void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); } + _LIBCPP_HIDE_FROM_ABI void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); } // private helper to allow reserving memory in the path - _LIBCPP_HIDE_FROM_ABI - void __reserve(size_t __s) { __pn_.reserve(__s); } + _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __s) { __pn_.reserve(__s); } // native format observers - _LIBCPP_HIDE_FROM_ABI - const string_type& native() const noexcept { return __pn_; } + _LIBCPP_HIDE_FROM_ABI const string_type& native() const noexcept { return __pn_; } - _LIBCPP_HIDE_FROM_ABI - const value_type* c_str() const noexcept { return __pn_.c_str(); } + _LIBCPP_HIDE_FROM_ABI const value_type* c_str() const noexcept { return __pn_.c_str(); } _LIBCPP_HIDE_FROM_ABI operator string_type() const { return __pn_; } -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return __pn_; } - _LIBCPP_HIDE_FROM_ABI - std::wstring generic_wstring() const { + _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const { std::wstring __s; __s.resize(__pn_.size()); std::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/'); return __s; } -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) - template , - class _Allocator = allocator<_ECharT> > - _LIBCPP_HIDE_FROM_ABI - basic_string<_ECharT, _Traits, _Allocator> - string(const _Allocator& __a = _Allocator()) const { +# if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , class _Allocator = allocator<_ECharT> > + _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const { using _Str = basic_string<_ECharT, _Traits, _Allocator>; _Str __s(__a); __s.reserve(__pn_.size()); @@ -801,9 +691,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return __s; } - _LIBCPP_HIDE_FROM_ABI std::string string() const { - return string(); - } + _LIBCPP_HIDE_FROM_ABI std::string string() const { return string(); } _LIBCPP_HIDE_FROM_ABI __u8_string u8string() const { using _CVT = __narrow_to_utf8; __u8_string __s; @@ -812,54 +700,43 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return __s; } - _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { - return string(); - } - _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { - return string(); - } + _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string(); } + _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string(); } // generic format observers - template , - class _Allocator = allocator<_ECharT> > - _LIBCPP_HIDE_FROM_ABI - basic_string<_ECharT, _Traits, _Allocator> + template , class _Allocator = allocator<_ECharT> > + _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> generic_string(const _Allocator& __a = _Allocator()) const { using _Str = basic_string<_ECharT, _Traits, _Allocator>; - _Str __s = string<_ECharT, _Traits, _Allocator>(__a); + _Str __s = string<_ECharT, _Traits, _Allocator>(__a); // Note: This (and generic_u8string below) is slightly suboptimal as // it iterates twice over the string; once to convert it to the right // character type, and once to replace path delimiters. - std::replace(__s.begin(), __s.end(), - static_cast<_ECharT>('\\'), static_cast<_ECharT>('/')); + std::replace(__s.begin(), __s.end(), static_cast<_ECharT>('\\'), static_cast<_ECharT>('/')); return __s; } _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return generic_string(); } _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return generic_string(); } _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return generic_string(); } - _LIBCPP_HIDE_FROM_ABI - __u8_string generic_u8string() const { + _LIBCPP_HIDE_FROM_ABI __u8_string generic_u8string() const { __u8_string __s = u8string(); std::replace(__s.begin(), __s.end(), '\\', '/'); return __s; } -#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ -#else /* _LIBCPP_WIN32API */ +# endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ +# else /* _LIBCPP_WIN32API */ _LIBCPP_HIDE_FROM_ABI std::string string() const { return __pn_; } -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T _LIBCPP_HIDE_FROM_ABI std::u8string u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); } -#else +# else _LIBCPP_HIDE_FROM_ABI std::string u8string() const { return __pn_; } -#endif +# endif -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) - template , - class _Allocator = allocator<_ECharT> > - _LIBCPP_HIDE_FROM_ABI - basic_string<_ECharT, _Traits, _Allocator> - string(const _Allocator& __a = _Allocator()) const { +# if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , class _Allocator = allocator<_ECharT> > + _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const { using _CVT = __widen_from_utf8; using _Str = basic_string<_ECharT, _Traits, _Allocator>; _Str __s(__a); @@ -868,43 +745,35 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return __s; } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS - _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { - return string(); - } -#endif - _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { - return string(); - } - _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { - return string(); - } -#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return string(); } +# endif + _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string(); } + _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string(); } +# endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ // generic format observers _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return __pn_; } -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T _LIBCPP_HIDE_FROM_ABI std::u8string generic_u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); } -#else +# else _LIBCPP_HIDE_FROM_ABI std::string generic_u8string() const { return __pn_; } -#endif +# endif -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) - template , - class _Allocator = allocator<_ECharT> > - _LIBCPP_HIDE_FROM_ABI - basic_string<_ECharT, _Traits, _Allocator> +# if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , class _Allocator = allocator<_ECharT> > + _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> generic_string(const _Allocator& __a = _Allocator()) const { return string<_ECharT, _Traits, _Allocator>(__a); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const { return string(); } -#endif +# endif _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return string(); } _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return string(); } -#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ -#endif /* !_LIBCPP_WIN32API */ +# endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ +# endif /* !_LIBCPP_WIN32API */ private: int __compare(__string_view) const; @@ -919,80 +788,43 @@ class _LIBCPP_EXPORTED_FROM_ABI path { public: // compare - _LIBCPP_HIDE_FROM_ABI int compare(const path& __p) const noexcept { - return __compare(__p.__pn_); - } - _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { - return __compare(__s); - } - _LIBCPP_HIDE_FROM_ABI int compare(__string_view __s) const { - return __compare(__s); - } - _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { - return __compare(__s); - } + _LIBCPP_HIDE_FROM_ABI int compare(const path& __p) const noexcept { return __compare(__p.__pn_); } + _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return __compare(__s); } + _LIBCPP_HIDE_FROM_ABI int compare(__string_view __s) const { return __compare(__s); } + _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return __compare(__s); } // decomposition - _LIBCPP_HIDE_FROM_ABI path root_name() const { - return string_type(__root_name()); - } - _LIBCPP_HIDE_FROM_ABI path root_directory() const { - return string_type(__root_directory()); - } + _LIBCPP_HIDE_FROM_ABI path root_name() const { return string_type(__root_name()); } + _LIBCPP_HIDE_FROM_ABI path root_directory() const { return string_type(__root_directory()); } _LIBCPP_HIDE_FROM_ABI path root_path() const { -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) return string_type(__root_path_raw()); -#else +# else return root_name().append(string_type(__root_directory())); -#endif - } - _LIBCPP_HIDE_FROM_ABI path relative_path() const { - return string_type(__relative_path()); - } - _LIBCPP_HIDE_FROM_ABI path parent_path() const { - return string_type(__parent_path()); - } - _LIBCPP_HIDE_FROM_ABI path filename() const { - return string_type(__filename()); +# endif } + _LIBCPP_HIDE_FROM_ABI path relative_path() const { return string_type(__relative_path()); } + _LIBCPP_HIDE_FROM_ABI path parent_path() const { return string_type(__parent_path()); } + _LIBCPP_HIDE_FROM_ABI path filename() const { return string_type(__filename()); } _LIBCPP_HIDE_FROM_ABI path stem() const { return string_type(__stem()); } - _LIBCPP_HIDE_FROM_ABI path extension() const { - return string_type(__extension()); - } + _LIBCPP_HIDE_FROM_ABI path extension() const { return string_type(__extension()); } // query - _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI bool - empty() const noexcept { - return __pn_.empty(); - } - - _LIBCPP_HIDE_FROM_ABI bool has_root_name() const { - return !__root_name().empty(); - } - _LIBCPP_HIDE_FROM_ABI bool has_root_directory() const { - return !__root_directory().empty(); - } - _LIBCPP_HIDE_FROM_ABI bool has_root_path() const { - return !__root_path_raw().empty(); - } - _LIBCPP_HIDE_FROM_ABI bool has_relative_path() const { - return !__relative_path().empty(); - } - _LIBCPP_HIDE_FROM_ABI bool has_parent_path() const { - return !__parent_path().empty(); - } - _LIBCPP_HIDE_FROM_ABI bool has_filename() const { - return !__filename().empty(); - } + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __pn_.empty(); } + + _LIBCPP_HIDE_FROM_ABI bool has_root_name() const { return !__root_name().empty(); } + _LIBCPP_HIDE_FROM_ABI bool has_root_directory() const { return !__root_directory().empty(); } + _LIBCPP_HIDE_FROM_ABI bool has_root_path() const { return !__root_path_raw().empty(); } + _LIBCPP_HIDE_FROM_ABI bool has_relative_path() const { return !__relative_path().empty(); } + _LIBCPP_HIDE_FROM_ABI bool has_parent_path() const { return !__parent_path().empty(); } + _LIBCPP_HIDE_FROM_ABI bool has_filename() const { return !__filename().empty(); } _LIBCPP_HIDE_FROM_ABI bool has_stem() const { return !__stem().empty(); } - _LIBCPP_HIDE_FROM_ABI bool has_extension() const { - return !__extension().empty(); - } + _LIBCPP_HIDE_FROM_ABI bool has_extension() const { return !__extension().empty(); } _LIBCPP_HIDE_FROM_ABI bool is_absolute() const { -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) __string_view __root_name_str = __root_name(); - __string_view __root_dir = __root_directory(); + __string_view __root_dir = __root_directory(); if (__root_name_str.size() == 2 && __root_name_str[1] == ':') { // A drive letter with no root directory is relative, e.g. x:example. return !__root_dir.empty(); @@ -1009,9 +841,9 @@ class _LIBCPP_EXPORTED_FROM_ABI path { return false; // Seems to be a server root name return true; -#else +# else return has_root_directory(); -#endif +# endif } _LIBCPP_HIDE_FROM_ABI bool is_relative() const { return !is_absolute(); } @@ -1033,21 +865,23 @@ class _LIBCPP_EXPORTED_FROM_ABI path { iterator begin() const; iterator end() const; -#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) - template ::value && - is_same<_Traits, char_traits >::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI friend - basic_ostream<_CharT, _Traits>& - operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { +# if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template < + class _CharT, + class _Traits, + __enable_if_t::value && is_same<_Traits, char_traits >::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>& + operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { __os << std::__quoted(__p.native()); return __os; } - template ::value || - !is_same<_Traits, char_traits >::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI friend - basic_ostream<_CharT, _Traits>& - operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { + template < + class _CharT, + class _Traits, + __enable_if_t::value || !is_same<_Traits, char_traits >::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI friend basic_ostream<_CharT, _Traits>& + operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { __os << std::__quoted(__p.string<_CharT, _Traits>()); return __os; } @@ -1060,7 +894,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path { __p = __tmp; return __is; } -#endif // !_LIBCPP_HAS_NO_LOCALIZATION +# endif // !_LIBCPP_HAS_NO_LOCALIZATION private: inline _LIBCPP_HIDE_FROM_ABI path& __assign_view(__string_view const& __s) { @@ -1070,9 +904,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path { string_type __pn_; }; -inline _LIBCPP_HIDE_FROM_ABI void swap(path& __lhs, path& __rhs) noexcept { - __lhs.swap(__rhs); -} +inline _LIBCPP_HIDE_FROM_ABI void swap(path& __lhs, path& __rhs) noexcept { __lhs.swap(__rhs); } _LIBCPP_EXPORTED_FROM_ABI size_t hash_value(const path& __p) noexcept; diff --git a/libcxx/include/__filesystem/path_iterator.h b/libcxx/include/__filesystem/path_iterator.h index 598d65d3d72e4..1a9aaf0e7d99e 100644 --- a/libcxx/include/__filesystem/path_iterator.h +++ b/libcxx/include/__filesystem/path_iterator.h @@ -49,49 +49,37 @@ class _LIBCPP_EXPORTED_FROM_ABI path::iterator { typedef path reference; public: - _LIBCPP_HIDE_FROM_ABI - iterator() - : __stashed_elem_(), __path_ptr_(nullptr), __entry_(), - __state_(_Singular) {} + _LIBCPP_HIDE_FROM_ABI iterator() : __stashed_elem_(), __path_ptr_(nullptr), __entry_(), __state_(_Singular) {} _LIBCPP_HIDE_FROM_ABI iterator(const iterator&) = default; - _LIBCPP_HIDE_FROM_ABI ~iterator() = default; + _LIBCPP_HIDE_FROM_ABI ~iterator() = default; _LIBCPP_HIDE_FROM_ABI iterator& operator=(const iterator&) = default; - _LIBCPP_HIDE_FROM_ABI - reference operator*() const { return __stashed_elem_; } + _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __stashed_elem_; } - _LIBCPP_HIDE_FROM_ABI - pointer operator->() const { return &__stashed_elem_; } + _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return &__stashed_elem_; } - _LIBCPP_HIDE_FROM_ABI - iterator& operator++() { - _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, - "attempting to increment a singular iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _AtEnd, - "attempting to increment the end iterator"); + _LIBCPP_HIDE_FROM_ABI iterator& operator++() { + _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, "attempting to increment a singular iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _AtEnd, "attempting to increment the end iterator"); return __increment(); } - _LIBCPP_HIDE_FROM_ABI - iterator operator++(int) { + _LIBCPP_HIDE_FROM_ABI iterator operator++(int) { iterator __it(*this); this->operator++(); return __it; } - _LIBCPP_HIDE_FROM_ABI - iterator& operator--() { - _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, - "attempting to decrement a singular iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(__entry_.data() != __path_ptr_->native().data(), - "attempting to decrement the begin iterator"); + _LIBCPP_HIDE_FROM_ABI iterator& operator--() { + _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, "attempting to decrement a singular iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + __entry_.data() != __path_ptr_->native().data(), "attempting to decrement the begin iterator"); return __decrement(); } - _LIBCPP_HIDE_FROM_ABI - iterator operator--(int) { + _LIBCPP_HIDE_FROM_ABI iterator operator--(int) { iterator __it(*this); this->operator--(); return __it; @@ -100,8 +88,7 @@ class _LIBCPP_EXPORTED_FROM_ABI path::iterator { private: friend class path; - inline _LIBCPP_HIDE_FROM_ABI friend bool operator==(const iterator&, - const iterator&); + inline _LIBCPP_HIDE_FROM_ABI friend bool operator==(const iterator&, const iterator&); iterator& __increment(); iterator& __decrement(); @@ -113,15 +100,12 @@ class _LIBCPP_EXPORTED_FROM_ABI path::iterator { }; _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline _LIBCPP_HIDE_FROM_ABI bool operator==(const path::iterator& __lhs, - const path::iterator& __rhs) { - return __lhs.__path_ptr_ == __rhs.__path_ptr_ && - __lhs.__entry_.data() == __rhs.__entry_.data(); +inline _LIBCPP_HIDE_FROM_ABI bool operator==(const path::iterator& __lhs, const path::iterator& __rhs) { + return __lhs.__path_ptr_ == __rhs.__path_ptr_ && __lhs.__entry_.data() == __rhs.__entry_.data(); } _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const path::iterator& __lhs, - const path::iterator& __rhs) { +inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const path::iterator& __lhs, const path::iterator& __rhs) { return !(__lhs == __rhs); } diff --git a/libcxx/include/__filesystem/perm_options.h b/libcxx/include/__filesystem/perm_options.h index ecf506fa4e8fc..529ef13558e97 100644 --- a/libcxx/include/__filesystem/perm_options.h +++ b/libcxx/include/__filesystem/perm_options.h @@ -21,48 +21,33 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM -enum class perm_options : unsigned char { - replace = 1, - add = 2, - remove = 4, - nofollow = 8 -}; - -_LIBCPP_HIDE_FROM_ABI -inline constexpr perm_options operator&(perm_options __lhs, perm_options __rhs) { - return static_cast(static_cast(__lhs) & - static_cast(__rhs)); +enum class perm_options : unsigned char { replace = 1, add = 2, remove = 4, nofollow = 8 }; + +_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator&(perm_options __lhs, perm_options __rhs) { + return static_cast(static_cast(__lhs) & static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perm_options operator|(perm_options __lhs, perm_options __rhs) { - return static_cast(static_cast(__lhs) | - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator|(perm_options __lhs, perm_options __rhs) { + return static_cast(static_cast(__lhs) | static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perm_options operator^(perm_options __lhs, perm_options __rhs) { - return static_cast(static_cast(__lhs) ^ - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator^(perm_options __lhs, perm_options __rhs) { + return static_cast(static_cast(__lhs) ^ static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perm_options operator~(perm_options __lhs) { +_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator~(perm_options __lhs) { return static_cast(~static_cast(__lhs)); } -_LIBCPP_HIDE_FROM_ABI -inline perm_options& operator&=(perm_options& __lhs, perm_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline perm_options& operator&=(perm_options& __lhs, perm_options __rhs) { return __lhs = __lhs & __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline perm_options& operator|=(perm_options& __lhs, perm_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline perm_options& operator|=(perm_options& __lhs, perm_options __rhs) { return __lhs = __lhs | __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline perm_options& operator^=(perm_options& __lhs, perm_options __rhs) { +_LIBCPP_HIDE_FROM_ABI inline perm_options& operator^=(perm_options& __lhs, perm_options __rhs) { return __lhs = __lhs ^ __rhs; } diff --git a/libcxx/include/__filesystem/perms.h b/libcxx/include/__filesystem/perms.h index 9f45f0c25f034..8f5f9a7e8248e 100644 --- a/libcxx/include/__filesystem/perms.h +++ b/libcxx/include/__filesystem/perms.h @@ -28,61 +28,51 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM enum class perms : unsigned { none = 0, - owner_read = 0400, + owner_read = 0400, owner_write = 0200, - owner_exec = 0100, - owner_all = 0700, + owner_exec = 0100, + owner_all = 0700, - group_read = 040, + group_read = 040, group_write = 020, - group_exec = 010, - group_all = 070, + group_exec = 010, + group_all = 070, - others_read = 04, + others_read = 04, others_write = 02, - others_exec = 01, - others_all = 07, + others_exec = 01, + others_all = 07, all = 0777, - set_uid = 04000, - set_gid = 02000, + set_uid = 04000, + set_gid = 02000, sticky_bit = 01000, - mask = 07777, - unknown = 0xFFFF, + mask = 07777, + unknown = 0xFFFF, }; -_LIBCPP_HIDE_FROM_ABI -inline constexpr perms operator&(perms __lhs, perms __rhs) { - return static_cast(static_cast(__lhs) & - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator&(perms __lhs, perms __rhs) { + return static_cast(static_cast(__lhs) & static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perms operator|(perms __lhs, perms __rhs) { - return static_cast(static_cast(__lhs) | - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator|(perms __lhs, perms __rhs) { + return static_cast(static_cast(__lhs) | static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perms operator^(perms __lhs, perms __rhs) { - return static_cast(static_cast(__lhs) ^ - static_cast(__rhs)); +_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator^(perms __lhs, perms __rhs) { + return static_cast(static_cast(__lhs) ^ static_cast(__rhs)); } -_LIBCPP_HIDE_FROM_ABI -inline constexpr perms operator~(perms __lhs) { +_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator~(perms __lhs) { return static_cast(~static_cast(__lhs)); } -_LIBCPP_HIDE_FROM_ABI -inline perms& operator&=(perms& __lhs, perms __rhs) { return __lhs = __lhs & __rhs; } +_LIBCPP_HIDE_FROM_ABI inline perms& operator&=(perms& __lhs, perms __rhs) { return __lhs = __lhs & __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline perms& operator|=(perms& __lhs, perms __rhs) { return __lhs = __lhs | __rhs; } +_LIBCPP_HIDE_FROM_ABI inline perms& operator|=(perms& __lhs, perms __rhs) { return __lhs = __lhs | __rhs; } -_LIBCPP_HIDE_FROM_ABI -inline perms& operator^=(perms& __lhs, perms __rhs) { return __lhs = __lhs ^ __rhs; } +_LIBCPP_HIDE_FROM_ABI inline perms& operator^=(perms& __lhs, perms __rhs) { return __lhs = __lhs ^ __rhs; } _LIBCPP_END_NAMESPACE_FILESYSTEM diff --git a/libcxx/include/__filesystem/recursive_directory_iterator.h b/libcxx/include/__filesystem/recursive_directory_iterator.h index e0aac636db8c8..7519cc2f2932f 100644 --- a/libcxx/include/__filesystem/recursive_directory_iterator.h +++ b/libcxx/include/__filesystem/recursive_directory_iterator.h @@ -36,40 +36,32 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH class recursive_directory_iterator { public: - using value_type = directory_entry; - using difference_type = ptrdiff_t; - using pointer = directory_entry const*; - using reference = directory_entry const&; + using value_type = directory_entry; + using difference_type = ptrdiff_t; + using pointer = directory_entry const*; + using reference = directory_entry const&; using iterator_category = input_iterator_tag; public: // constructors and destructor - _LIBCPP_HIDE_FROM_ABI - recursive_directory_iterator() noexcept : __rec_(false) {} + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator() noexcept : __rec_(false) {} - _LIBCPP_HIDE_FROM_ABI - explicit recursive_directory_iterator( + _LIBCPP_HIDE_FROM_ABI explicit recursive_directory_iterator( const path& __p, directory_options __xoptions = directory_options::none) : recursive_directory_iterator(__p, __xoptions, nullptr) {} - _LIBCPP_HIDE_FROM_ABI - recursive_directory_iterator(const path& __p, directory_options __xoptions, - error_code& __ec) + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const path& __p, directory_options __xoptions, error_code& __ec) : recursive_directory_iterator(__p, __xoptions, &__ec) {} - _LIBCPP_HIDE_FROM_ABI - recursive_directory_iterator(const path& __p, error_code& __ec) + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const path& __p, error_code& __ec) : recursive_directory_iterator(__p, directory_options::none, &__ec) {} _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(const recursive_directory_iterator&) = default; - _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(recursive_directory_iterator&&) = default; + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator(recursive_directory_iterator&&) = default; - _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& - operator=(const recursive_directory_iterator&) = default; + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator=(const recursive_directory_iterator&) = default; - _LIBCPP_HIDE_FROM_ABI - recursive_directory_iterator& - operator=(recursive_directory_iterator&& __o) noexcept { + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator=(recursive_directory_iterator&& __o) noexcept { // non-default implementation provided to support self-move assign. if (this != &__o) { __imp_ = std::move(__o.__imp_); @@ -80,40 +72,30 @@ class recursive_directory_iterator { _LIBCPP_HIDE_FROM_ABI ~recursive_directory_iterator() = default; - _LIBCPP_HIDE_FROM_ABI - const directory_entry& operator*() const { return __dereference(); } + _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { return __dereference(); } - _LIBCPP_HIDE_FROM_ABI - const directory_entry* operator->() const { return &__dereference(); } + _LIBCPP_HIDE_FROM_ABI const directory_entry* operator->() const { return &__dereference(); } _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& operator++() { return __increment(); } - _LIBCPP_HIDE_FROM_ABI - __dir_element_proxy operator++(int) { + _LIBCPP_HIDE_FROM_ABI __dir_element_proxy operator++(int) { __dir_element_proxy __p(**this); __increment(); return __p; } - _LIBCPP_HIDE_FROM_ABI - recursive_directory_iterator& increment(error_code& __ec) { - return __increment(&__ec); - } + _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } _LIBCPP_EXPORTED_FROM_ABI directory_options options() const; _LIBCPP_EXPORTED_FROM_ABI int depth() const; - _LIBCPP_HIDE_FROM_ABI - void pop() { __pop(); } + _LIBCPP_HIDE_FROM_ABI void pop() { __pop(); } - _LIBCPP_HIDE_FROM_ABI - void pop(error_code& __ec) { __pop(&__ec); } + _LIBCPP_HIDE_FROM_ABI void pop(error_code& __ec) { __pop(&__ec); } - _LIBCPP_HIDE_FROM_ABI - bool recursion_pending() const { return __rec_; } + _LIBCPP_HIDE_FROM_ABI bool recursion_pending() const { return __rec_; } - _LIBCPP_HIDE_FROM_ABI - void disable_recursion_pending() { __rec_ = false; } + _LIBCPP_HIDE_FROM_ABI void disable_recursion_pending() { __rec_ = false; } # if _LIBCPP_STD_VER >= 20 @@ -132,8 +114,7 @@ class recursive_directory_iterator { _LIBCPP_EXPORTED_FROM_ABI void __pop(error_code* __ec = nullptr); inline _LIBCPP_HIDE_FROM_ABI friend bool - operator==(const recursive_directory_iterator&, - const recursive_directory_iterator&) noexcept; + operator==(const recursive_directory_iterator&, const recursive_directory_iterator&) noexcept; struct _LIBCPP_HIDDEN __shared_imp; shared_ptr<__shared_imp> __imp_; @@ -141,24 +122,20 @@ class recursive_directory_iterator { }; // class recursive_directory_iterator inline _LIBCPP_HIDE_FROM_ABI bool -operator==(const recursive_directory_iterator& __lhs, - const recursive_directory_iterator& __rhs) noexcept { +operator==(const recursive_directory_iterator& __lhs, const recursive_directory_iterator& __rhs) noexcept { return __lhs.__imp_ == __rhs.__imp_; } -_LIBCPP_HIDE_FROM_ABI -inline bool operator!=(const recursive_directory_iterator& __lhs, - const recursive_directory_iterator& __rhs) noexcept { +_LIBCPP_HIDE_FROM_ABI inline bool +operator!=(const recursive_directory_iterator& __lhs, const recursive_directory_iterator& __rhs) noexcept { return !(__lhs == __rhs); } // enable recursive_directory_iterator range-based for statements -inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator -begin(recursive_directory_iterator __iter) noexcept { +inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator begin(recursive_directory_iterator __iter) noexcept { return __iter; } -inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator -end(recursive_directory_iterator) noexcept { +inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator end(recursive_directory_iterator) noexcept { return recursive_directory_iterator(); } @@ -166,17 +143,17 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_END_NAMESPACE_FILESYSTEM -#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_STD_VER >= 20 template <> -_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline constexpr bool std::ranges::enable_borrowed_range = true; +_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool + std::ranges::enable_borrowed_range = true; template <> -_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY -inline constexpr bool std::ranges::enable_view = true; +_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool + std::ranges::enable_view = true; -#endif // _LIBCPP_STD_VER >= 20 +# endif // _LIBCPP_STD_VER >= 20 #endif // _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_FILESYSTEM) diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h index a79007e969ca8..bde878054865e 100644 --- a/libcxx/include/__filesystem/u8path.h +++ b/libcxx/include/__filesystem/u8path.h @@ -19,7 +19,7 @@ // Only required on Windows for __widen_from_utf8, and included conservatively // because it requires support for localization. #if defined(_LIBCPP_WIN32API) -# include +# include #endif #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -33,38 +33,34 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T - path - u8path(_InputIt __f, _InputIt __l) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _InputIt __l) { static_assert( -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T is_same::__char_type, char8_t>::value || -#endif - is_same::__char_type, char>::value, +# endif + is_same::__char_type, char>::value, "u8path(Iter, Iter) requires Iter have a value_type of type 'char'" " or 'char8_t'"); -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) string __tmp(__f, __l); using _CVT = __widen_from_utf8; std::wstring __w; __w.reserve(__tmp.size()); _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size()); return path(__w); -#else +# else return path(__f, __l); -#endif /* !_LIBCPP_WIN32API */ +# endif /* !_LIBCPP_WIN32API */ } -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T - path - u8path(_InputIt __f, _NullSentinel) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _NullSentinel) { static_assert( -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T is_same::__char_type, char8_t>::value || -#endif - is_same::__char_type, char>::value, +# endif + is_same::__char_type, char>::value, "u8path(Iter, Iter) requires Iter have a value_type of type 'char'" " or 'char8_t'"); string __tmp; @@ -77,25 +73,23 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size()); return path(__w); } -#endif /* _LIBCPP_WIN32API */ +# endif /* _LIBCPP_WIN32API */ template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T - path - u8path(const _Source& __s) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source& __s) { static_assert( -#ifndef _LIBCPP_HAS_NO_CHAR8_T +# ifndef _LIBCPP_HAS_NO_CHAR8_T is_same::__char_type, char8_t>::value || -#endif - is_same::__char_type, char>::value, +# endif + is_same::__char_type, char>::value, "u8path(Source const&) requires Source have a character type of type " "'char' or 'char8_t'"); -#if defined(_LIBCPP_WIN32API) +# if defined(_LIBCPP_WIN32API) using _Traits = __is_pathable<_Source>; return u8path(std::__unwrap_iter(_Traits::__range_begin(__s)), std::__unwrap_iter(_Traits::__range_end(__s))); -#else +# else return path(__s); -#endif +# endif } _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h index 24608a0b1d200..7ee583d813945 100644 --- a/libcxx/include/__format/buffer.h +++ b/libcxx/include/__format/buffer.h @@ -71,7 +71,7 @@ class _LIBCPP_TEMPLATE_VIS __output_buffer { __obj_(__obj) {} _LIBCPP_HIDE_FROM_ABI void __reset(_CharT* __ptr, size_t __capacity) { - __ptr_ = __ptr; + __ptr_ = __ptr; __capacity_ = __capacity; } @@ -253,19 +253,18 @@ template <__fmt_char_type _CharT> class _LIBCPP_TEMPLATE_VIS __direct_storage {}; template -concept __enable_direct_output = __fmt_char_type<_CharT> && +concept __enable_direct_output = + __fmt_char_type<_CharT> && (same_as<_OutIt, _CharT*> // TODO(hardening): the following check might not apply to hardened iterators and might need to be wrapped in an // `#ifdef`. - || same_as<_OutIt, __wrap_iter<_CharT*>> - ); + || same_as<_OutIt, __wrap_iter<_CharT*>>); /// Write policy for directly writing to the underlying output. template class _LIBCPP_TEMPLATE_VIS __writer_direct { public: - _LIBCPP_HIDE_FROM_ABI explicit __writer_direct(_OutIt __out_it) - : __out_it_(__out_it) {} + _LIBCPP_HIDE_FROM_ABI explicit __writer_direct(_OutIt __out_it) : __out_it_(__out_it) {} _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() { return __out_it_; } @@ -283,8 +282,7 @@ class _LIBCPP_TEMPLATE_VIS __writer_direct { template class _LIBCPP_TEMPLATE_VIS __writer_iterator { public: - _LIBCPP_HIDE_FROM_ABI explicit __writer_iterator(_OutIt __out_it) - : __out_it_{std::move(__out_it)} {} + _LIBCPP_HIDE_FROM_ABI explicit __writer_iterator(_OutIt __out_it) : __out_it_{std::move(__out_it)} {} _LIBCPP_HIDE_FROM_ABI _OutIt __out_it() && { return std::move(__out_it_); } @@ -306,7 +304,8 @@ class _LIBCPP_TEMPLATE_VIS __writer_iterator { template concept __insertable = __enable_insertable<_Container> && __fmt_char_type && - requires(_Container& __t, add_pointer_t __first, + requires(_Container& __t, + add_pointer_t __first, add_pointer_t __last) { __t.insert(__t.end(), __first, __last); }; /// Extract the container type of a \ref back_insert_iterator. @@ -345,28 +344,29 @@ class _LIBCPP_TEMPLATE_VIS __writer_selector { using _Container = typename __back_insert_iterator_container<_OutIt>::type; public: - using type = conditional_t, __writer_container<_Container>, - conditional_t<__enable_direct_output<_OutIt, _CharT>, __writer_direct<_OutIt, _CharT>, - __writer_iterator<_OutIt, _CharT>>>; + using type = + conditional_t, + __writer_container<_Container>, + conditional_t<__enable_direct_output<_OutIt, _CharT>, + __writer_direct<_OutIt, _CharT>, + __writer_iterator<_OutIt, _CharT>>>; }; /// The generic formatting buffer. template -requires(output_iterator<_OutIt, const _CharT&>) class _LIBCPP_TEMPLATE_VIS - __format_buffer { + requires(output_iterator<_OutIt, const _CharT&>) +class _LIBCPP_TEMPLATE_VIS __format_buffer { using _Storage = - conditional_t<__enable_direct_output<_OutIt, _CharT>, - __direct_storage<_CharT>, __internal_storage<_CharT>>; + conditional_t<__enable_direct_output<_OutIt, _CharT>, __direct_storage<_CharT>, __internal_storage<_CharT>>; public: _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it) requires(same_as<_Storage, __internal_storage<_CharT>>) : __output_(__storage_.__begin(), __storage_.__buffer_size, this), __writer_(std::move(__out_it)) {} - _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it) requires( - same_as<_Storage, __direct_storage<_CharT>>) - : __output_(std::__unwrap_iter(__out_it), size_t(-1), this), - __writer_(std::move(__out_it)) {} + _LIBCPP_HIDE_FROM_ABI explicit __format_buffer(_OutIt __out_it) + requires(same_as<_Storage, __direct_storage<_CharT>>) + : __output_(std::__unwrap_iter(__out_it), size_t(-1), this), __writer_(std::move(__out_it)) {} _LIBCPP_HIDE_FROM_ABI auto __make_output_iterator() { return __output_.__make_output_iterator(); } diff --git a/libcxx/include/__format/concepts.h b/libcxx/include/__format/concepts.h index ae96b6a198118..299c5f40ee35b 100644 --- a/libcxx/include/__format/concepts.h +++ b/libcxx/include/__format/concepts.h @@ -74,7 +74,7 @@ concept __fmt_pair_like = __is_specialization_v<_Tp, pair> || (__is_specialization_v<_Tp, tuple> && tuple_size_v<_Tp> == 2); # endif //_LIBCPP_STD_VER >= 23 -#endif //_LIBCPP_STD_VER >= 20 +#endif //_LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h index cc6be7a25660f..280c910824175 100644 --- a/libcxx/include/__format/format_arg.h +++ b/libcxx/include/__format/format_arg.h @@ -73,16 +73,16 @@ enum class __arg_t : uint8_t { }; inline constexpr unsigned __packed_arg_t_bits = 5; -inline constexpr uint8_t __packed_arg_t_mask = 0x1f; +inline constexpr uint8_t __packed_arg_t_mask = 0x1f; inline constexpr unsigned __packed_types_storage_bits = 64; -inline constexpr unsigned __packed_types_max = __packed_types_storage_bits / __packed_arg_t_bits; +inline constexpr unsigned __packed_types_max = __packed_types_storage_bits / __packed_arg_t_bits; -_LIBCPP_HIDE_FROM_ABI -constexpr bool __use_packed_format_arg_store(size_t __size) { return __size <= __packed_types_max; } +_LIBCPP_HIDE_FROM_ABI constexpr bool __use_packed_format_arg_store(size_t __size) { + return __size <= __packed_types_max; +} -_LIBCPP_HIDE_FROM_ABI -constexpr __arg_t __get_packed_type(uint64_t __types, size_t __id) { +_LIBCPP_HIDE_FROM_ABI constexpr __arg_t __get_packed_type(uint64_t __types, size_t __id) { _LIBCPP_ASSERT_UNCATEGORIZED(__id <= __packed_types_max, ""); if (__id > 0) @@ -96,8 +96,7 @@ constexpr __arg_t __get_packed_type(uint64_t __types, size_t __id) { // This function is not user obervable, so it can directly use the non-standard // types of the "variant". See __arg_t for more details. template -_LIBCPP_HIDE_FROM_ABI decltype(auto) -__visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { +_LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { switch (__arg.__type_) { case __format::__arg_t::__none: return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__monostate_); @@ -138,8 +137,8 @@ __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { case __format::__arg_t::__ptr: return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__ptr_); case __format::__arg_t::__handle: - return std::invoke(std::forward<_Visitor>(__vis), - typename basic_format_arg<_Context>::handle{__arg.__value_.__handle_}); + return std::invoke( + std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__arg.__value_.__handle_}); } __libcpp_unreachable(); @@ -224,12 +223,9 @@ class _LIBCPP_TEMPLATE_VIS basic_format_arg { public: class _LIBCPP_TEMPLATE_VIS handle; - _LIBCPP_HIDE_FROM_ABI basic_format_arg() noexcept - : __type_{__format::__arg_t::__none} {} + _LIBCPP_HIDE_FROM_ABI basic_format_arg() noexcept : __type_{__format::__arg_t::__none} {} - _LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { - return __type_ != __format::__arg_t::__none; - } + _LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { return __type_ != __format::__arg_t::__none; } private: using char_type = typename _Context::char_type; @@ -257,8 +253,7 @@ class _LIBCPP_TEMPLATE_VIS basic_format_arg { template class _LIBCPP_TEMPLATE_VIS basic_format_arg<_Context>::handle { public: - _LIBCPP_HIDE_FROM_ABI - void format(basic_format_parse_context& __parse_ctx, _Context& __ctx) const { + _LIBCPP_HIDE_FROM_ABI void format(basic_format_parse_context& __parse_ctx, _Context& __ctx) const { __handle_.__format_(__parse_ctx, __ctx, __handle_.__ptr_); } @@ -272,8 +267,7 @@ class _LIBCPP_TEMPLATE_VIS basic_format_arg<_Context>::handle { // This function is user facing, so it must wrap the non-standard types of // the "variant" in a handle to stay conforming. See __arg_t for more details. template -_LIBCPP_HIDE_FROM_ABI decltype(auto) -visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { +_LIBCPP_HIDE_FROM_ABI decltype(auto) visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { switch (__arg.__type_) { # ifndef _LIBCPP_HAS_NO_INT128 case __format::__arg_t::__i128: { diff --git a/libcxx/include/__format/format_arg_store.h b/libcxx/include/__format/format_arg_store.h index 64ee12440b62f..c481992d2d719 100644 --- a/libcxx/include/__format/format_arg_store.h +++ b/libcxx/include/__format/format_arg_store.h @@ -243,8 +243,7 @@ struct __unpacked_format_arg_store { template struct _LIBCPP_TEMPLATE_VIS __format_arg_store { - _LIBCPP_HIDE_FROM_ABI - __format_arg_store(_Args&... __args) noexcept { + _LIBCPP_HIDE_FROM_ABI __format_arg_store(_Args&... __args) noexcept { if constexpr (sizeof...(_Args) != 0) { if constexpr (__format::__use_packed_format_arg_store(sizeof...(_Args))) __format::__create_packed_storage(__storage.__types_, __storage.__values_, __args...); @@ -253,9 +252,10 @@ struct _LIBCPP_TEMPLATE_VIS __format_arg_store { } } - using _Storage = conditional_t<__format::__use_packed_format_arg_store(sizeof...(_Args)), - __format::__packed_format_arg_store<_Context, sizeof...(_Args)>, - __format::__unpacked_format_arg_store<_Context, sizeof...(_Args)>>; + using _Storage = + conditional_t<__format::__use_packed_format_arg_store(sizeof...(_Args)), + __format::__packed_format_arg_store<_Context, sizeof...(_Args)>, + __format::__unpacked_format_arg_store<_Context, sizeof...(_Args)>>; _Storage __storage; }; diff --git a/libcxx/include/__format/format_args.h b/libcxx/include/__format/format_args.h index defb42a4ab006..9e0afecc0ae96 100644 --- a/libcxx/include/__format/format_args.h +++ b/libcxx/include/__format/format_args.h @@ -37,14 +37,13 @@ class _LIBCPP_TEMPLATE_VIS basic_format_args { if constexpr (sizeof...(_Args) != 0) { if constexpr (__format::__use_packed_format_arg_store(sizeof...(_Args))) { __values_ = __store.__storage.__values_; - __types_ = __store.__storage.__types_; + __types_ = __store.__storage.__types_; } else __args_ = __store.__storage.__args_; } } - _LIBCPP_HIDE_FROM_ABI - basic_format_arg<_Context> get(size_t __id) const noexcept { + _LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> get(size_t __id) const noexcept { if (__id >= __size_) return basic_format_arg<_Context>{}; diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h index 4e3d15ec862e7..5b252b81f691b 100644 --- a/libcxx/include/__format/format_context.h +++ b/libcxx/include/__format/format_context.h @@ -27,8 +27,8 @@ #include #ifndef _LIBCPP_HAS_NO_LOCALIZATION -#include -#include +# include +# include #endif #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -40,10 +40,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -requires output_iterator<_OutIt, const _CharT&> + requires output_iterator<_OutIt, const _CharT&> class _LIBCPP_TEMPLATE_VIS basic_format_context; -#ifndef _LIBCPP_HAS_NO_LOCALIZATION +# ifndef _LIBCPP_HAS_NO_LOCALIZATION /** * Helper to create a basic_format_context. * @@ -51,32 +51,26 @@ class _LIBCPP_TEMPLATE_VIS basic_format_context; */ template _LIBCPP_HIDE_FROM_ABI basic_format_context<_OutIt, _CharT> -__format_context_create( - _OutIt __out_it, - basic_format_args> __args, - optional&& __loc = nullopt) { +__format_context_create(_OutIt __out_it, + basic_format_args> __args, + optional&& __loc = nullopt) { return std::basic_format_context(std::move(__out_it), __args, std::move(__loc)); } -#else +# else template _LIBCPP_HIDE_FROM_ABI basic_format_context<_OutIt, _CharT> -__format_context_create( - _OutIt __out_it, - basic_format_args> __args) { +__format_context_create(_OutIt __out_it, basic_format_args> __args) { return std::basic_format_context(std::move(__out_it), __args); } -#endif +# endif -using format_context = - basic_format_context>, - char>; -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS -using wformat_context = basic_format_context< - back_insert_iterator<__format::__output_buffer>, wchar_t>; -#endif +using format_context = basic_format_context>, char>; +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +using wformat_context = basic_format_context< back_insert_iterator<__format::__output_buffer>, wchar_t>; +# endif template -requires output_iterator<_OutIt, const _CharT&> + requires output_iterator<_OutIt, const _CharT&> class // clang-format off _LIBCPP_TEMPLATE_VIS @@ -85,29 +79,28 @@ class // clang-format on basic_format_context { public: - using iterator = _OutIt; + using iterator = _OutIt; using char_type = _CharT; template using formatter_type = formatter<_Tp, _CharT>; - _LIBCPP_HIDE_FROM_ABI basic_format_arg - arg(size_t __id) const noexcept { + _LIBCPP_HIDE_FROM_ABI basic_format_arg arg(size_t __id) const noexcept { return __args_.get(__id); } -#ifndef _LIBCPP_HAS_NO_LOCALIZATION +# ifndef _LIBCPP_HAS_NO_LOCALIZATION _LIBCPP_HIDE_FROM_ABI std::locale locale() { if (!__loc_) __loc_ = std::locale{}; return *__loc_; } -#endif +# endif _LIBCPP_HIDE_FROM_ABI iterator out() { return std::move(__out_it_); } _LIBCPP_HIDE_FROM_ABI void advance_to(iterator __it) { __out_it_ = std::move(__it); } private: iterator __out_it_; basic_format_args __args_; -#ifndef _LIBCPP_HAS_NO_LOCALIZATION +# ifndef _LIBCPP_HAS_NO_LOCALIZATION // The Standard doesn't specify how the locale is stored. // [format.context]/6 @@ -121,27 +114,21 @@ class optional __loc_; template - friend _LIBCPP_HIDE_FROM_ABI basic_format_context<_OtherOutIt, _OtherCharT> - __format_context_create(_OtherOutIt, basic_format_args>, - optional&&); + friend _LIBCPP_HIDE_FROM_ABI basic_format_context<_OtherOutIt, _OtherCharT> __format_context_create( + _OtherOutIt, basic_format_args>, optional&&); // Note: the Standard doesn't specify the required constructors. - _LIBCPP_HIDE_FROM_ABI - explicit basic_format_context(_OutIt __out_it, - basic_format_args __args, - optional&& __loc) - : __out_it_(std::move(__out_it)), __args_(__args), - __loc_(std::move(__loc)) {} -#else + _LIBCPP_HIDE_FROM_ABI explicit basic_format_context( + _OutIt __out_it, basic_format_args __args, optional&& __loc) + : __out_it_(std::move(__out_it)), __args_(__args), __loc_(std::move(__loc)) {} +# else template friend _LIBCPP_HIDE_FROM_ABI basic_format_context<_OtherOutIt, _OtherCharT> __format_context_create(_OtherOutIt, basic_format_args>); - _LIBCPP_HIDE_FROM_ABI - explicit basic_format_context(_OutIt __out_it, - basic_format_args __args) + _LIBCPP_HIDE_FROM_ABI explicit basic_format_context(_OutIt __out_it, basic_format_args __args) : __out_it_(std::move(__out_it)), __args_(__args) {} -#endif +# endif }; // A specialization for __retarget_buffer @@ -161,8 +148,7 @@ class // Here the width of an element in input is determined dynamically. // Note when the top-level element has no width the retargeting is not needed. template -class _LIBCPP_TEMPLATE_VIS - basic_format_context::__iterator, _CharT> { +class _LIBCPP_TEMPLATE_VIS basic_format_context::__iterator, _CharT> { public: using iterator = typename __format::__retarget_buffer<_CharT>::__iterator; using char_type = _CharT; diff --git a/libcxx/include/__format/format_error.h b/libcxx/include/__format/format_error.h index 51d6c58230910..ed40e395d6af7 100644 --- a/libcxx/include/__format/format_error.h +++ b/libcxx/include/__format/format_error.h @@ -26,24 +26,21 @@ _LIBCPP_DIAGNOSTIC_PUSH _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables") class _LIBCPP_EXPORTED_FROM_ABI format_error : public runtime_error { public: - _LIBCPP_HIDE_FROM_ABI explicit format_error(const string& __s) - : runtime_error(__s) {} - _LIBCPP_HIDE_FROM_ABI explicit format_error(const char* __s) - : runtime_error(__s) {} - _LIBCPP_HIDE_FROM_ABI format_error(const format_error&) = default; + _LIBCPP_HIDE_FROM_ABI explicit format_error(const string& __s) : runtime_error(__s) {} + _LIBCPP_HIDE_FROM_ABI explicit format_error(const char* __s) : runtime_error(__s) {} + _LIBCPP_HIDE_FROM_ABI format_error(const format_error&) = default; _LIBCPP_HIDE_FROM_ABI format_error& operator=(const format_error&) = default; _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~format_error() noexcept override = default; }; _LIBCPP_DIAGNOSTIC_POP -_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void -__throw_format_error(const char* __s) { -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS +_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_format_error(const char* __s) { +# ifndef _LIBCPP_HAS_NO_EXCEPTIONS throw format_error(__s); -#else +# else _LIBCPP_VERBOSE_ABORT("format_error was thrown in -fno-exceptions mode with message \"%s\"", __s); -#endif +# endif } #endif //_LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__format/format_functions.h b/libcxx/include/__format/format_functions.h index 8b2111f0e287c..015bff70f51d9 100644 --- a/libcxx/include/__format/format_functions.h +++ b/libcxx/include/__format/format_functions.h @@ -41,7 +41,7 @@ #include #ifndef _LIBCPP_HAS_NO_LOCALIZATION -#include +# include #endif #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -58,9 +58,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD // to do this optimization now. using format_args = basic_format_args; -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS using wformat_args = basic_format_args; -#endif +# endif template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&... __args) { @@ -188,9 +188,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_validate_argument( // This function is not user facing, so it can directly use the non-standard types of the "variant". template -_LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(basic_format_parse_context<_CharT>& __parse_ctx, - __compile_time_basic_format_context<_CharT>& __ctx, - __arg_t __type) { +_LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg( + basic_format_parse_context<_CharT>& __parse_ctx, + __compile_time_basic_format_context<_CharT>& __ctx, + __arg_t __type) { switch (__type) { case __arg_t::__none: std::__throw_format_error("Invalid argument"); @@ -203,22 +204,22 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(basic_forma case __arg_t::__long_long: return __format::__compile_time_validate_argument<_CharT, long long>(__parse_ctx, __ctx); case __arg_t::__i128: -# ifndef _LIBCPP_HAS_NO_INT128 +# ifndef _LIBCPP_HAS_NO_INT128 return __format::__compile_time_validate_argument<_CharT, __int128_t>(__parse_ctx, __ctx); -# else +# else std::__throw_format_error("Invalid argument"); -# endif +# endif return; case __arg_t::__unsigned: return __format::__compile_time_validate_argument<_CharT, unsigned>(__parse_ctx, __ctx); case __arg_t::__unsigned_long_long: return __format::__compile_time_validate_argument<_CharT, unsigned long long>(__parse_ctx, __ctx); case __arg_t::__u128: -# ifndef _LIBCPP_HAS_NO_INT128 +# ifndef _LIBCPP_HAS_NO_INT128 return __format::__compile_time_validate_argument<_CharT, __uint128_t>(__parse_ctx, __ctx); -# else +# else std::__throw_format_error("Invalid argument"); -# endif +# endif return; case __arg_t::__float: return __format::__compile_time_validate_argument<_CharT, float, true>(__parse_ctx, __ctx); @@ -240,9 +241,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(basic_forma template _LIBCPP_HIDE_FROM_ABI constexpr _Iterator -__handle_replacement_field(_Iterator __begin, _Iterator __end, - _ParseCtx& __parse_ctx, _Ctx& __ctx) { - using _CharT = iter_value_t<_Iterator>; +__handle_replacement_field(_Iterator __begin, _Iterator __end, _ParseCtx& __parse_ctx, _Ctx& __ctx) { + using _CharT = iter_value_t<_Iterator>; __format::__parse_number_result __r = __format::__parse_arg_id(__begin, __end, __parse_ctx); if (__r.__last == __end) @@ -294,13 +294,12 @@ __handle_replacement_field(_Iterator __begin, _Iterator __end, } template -_LIBCPP_HIDE_FROM_ABI constexpr typename _Ctx::iterator -__vformat_to(_ParseCtx&& __parse_ctx, _Ctx&& __ctx) { +_LIBCPP_HIDE_FROM_ABI constexpr typename _Ctx::iterator __vformat_to(_ParseCtx&& __parse_ctx, _Ctx&& __ctx) { using _CharT = typename _ParseCtx::char_type; static_assert(same_as); - auto __begin = __parse_ctx.begin(); - auto __end = __parse_ctx.end(); + auto __begin = __parse_ctx.begin(); + auto __end = __parse_ctx.end(); typename _Ctx::iterator __out_it = __ctx.out(); while (__begin != __end) { switch (*__begin) { @@ -311,8 +310,7 @@ __vformat_to(_ParseCtx&& __parse_ctx, _Ctx&& __ctx) { if (*__begin != _CharT('{')) [[likely]] { __ctx.advance_to(std::move(__out_it)); - __begin = - __format::__handle_replacement_field(__begin, __end, __parse_ctx, __ctx); + __begin = __format::__handle_replacement_field(__begin, __end, __parse_ctx, __ctx); __out_it = __ctx.out(); // The output is written and __begin points to the next character. So @@ -371,9 +369,7 @@ struct _LIBCPP_TEMPLATE_VIS basic_format_string { _Context{__types_.data(), __handles_.data(), sizeof...(_Args)}); } - _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<_CharT> get() const noexcept { - return __str_; - } + _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<_CharT> get() const noexcept { return __str_; } # if _LIBCPP_STD_VER >= 26 _LIBCPP_HIDE_FROM_ABI basic_format_string(__runtime_format_string<_CharT> __s) noexcept : __str_(__s.__str_) {} # endif @@ -399,23 +395,23 @@ struct _LIBCPP_TEMPLATE_VIS basic_format_string { template using format_string = basic_format_string...>; -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template using wformat_string = basic_format_string...>; -#endif +# endif template -requires(output_iterator<_OutIt, const _CharT&>) _LIBCPP_HIDE_FROM_ABI _OutIt - __vformat_to( - _OutIt __out_it, basic_string_view<_CharT> __fmt, - basic_format_args> __args) { + requires(output_iterator<_OutIt, const _CharT&>) +_LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to(_OutIt __out_it, + basic_string_view<_CharT> __fmt, + basic_format_args> __args) { if constexpr (same_as<_OutIt, _FormatOutIt>) - return std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()}, - std::__format_context_create(std::move(__out_it), __args)); + return std::__format::__vformat_to( + basic_format_parse_context{__fmt, __args.__size()}, std::__format_context_create(std::move(__out_it), __args)); else { __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)}; std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()}, - std::__format_context_create(__buffer.__make_output_iterator(), __args)); + std::__format_context_create(__buffer.__make_output_iterator(), __args)); return std::move(__buffer).__out_it(); } } @@ -424,34 +420,31 @@ requires(output_iterator<_OutIt, const _CharT&>) _LIBCPP_HIDE_FROM_ABI _OutIt // https://reviews.llvm.org/D110499#inline-1180704 // TODO FMT Evaluate whether we want to file a Clang bug report regarding this. template _OutIt> -_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt -vformat_to(_OutIt __out_it, string_view __fmt, format_args __args) { +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt vformat_to(_OutIt __out_it, string_view __fmt, format_args __args) { return std::__vformat_to(std::move(__out_it), __fmt, __args); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt> _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt vformat_to(_OutIt __out_it, wstring_view __fmt, wformat_args __args) { return std::__vformat_to(std::move(__out_it), __fmt, __args); } -#endif +# endif template _OutIt, class... _Args> _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt format_to(_OutIt __out_it, format_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat_to(std::move(__out_it), __fmt.get(), - std::make_format_args(__args...)); + return std::vformat_to(std::move(__out_it), __fmt.get(), std::make_format_args(__args...)); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt, class... _Args> _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt format_to(_OutIt __out_it, wformat_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat_to(std::move(__out_it), __fmt.get(), - std::make_wformat_args(__args...)); + return std::vformat_to(std::move(__out_it), __fmt.get(), std::make_wformat_args(__args...)); } -#endif +# endif // TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup // fires too eagerly, see http://llvm.org/PR61563. @@ -490,12 +483,14 @@ format(wformat_string<_Args...> __fmt, _Args&&... __args) { # endif template -_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __vformat_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, - basic_string_view<_CharT> __fmt, - basic_format_args<_Context> __args) { +_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> +__vformat_to_n(_OutIt __out_it, + iter_difference_t<_OutIt> __n, + basic_string_view<_CharT> __fmt, + basic_format_args<_Context> __args) { __format::__format_to_n_buffer<_OutIt, _CharT> __buffer{std::move(__out_it), __n}; std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()}, - std::__format_context_create(__buffer.__make_output_iterator(), __args)); + std::__format_context_create(__buffer.__make_output_iterator(), __args)); return std::move(__buffer).__result(); } @@ -505,20 +500,19 @@ format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, format_string<_Args. return std::__vformat_to_n(std::move(__out_it), __n, __fmt.get(), std::make_format_args(__args...)); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt, class... _Args> _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> -format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, wformat_string<_Args...> __fmt, - _Args&&... __args) { +format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, wformat_string<_Args...> __fmt, _Args&&... __args) { return std::__vformat_to_n(std::move(__out_it), __n, __fmt.get(), std::make_wformat_args(__args...)); } -#endif +# endif template _LIBCPP_HIDE_FROM_ABI size_t __vformatted_size(basic_string_view<_CharT> __fmt, auto __args) { __format::__formatted_size_buffer<_CharT> __buffer; std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()}, - std::__format_context_create(__buffer.__make_output_iterator(), __args)); + std::__format_context_create(__buffer.__make_output_iterator(), __args)); return std::move(__buffer).__result(); } @@ -539,14 +533,15 @@ formatted_size(wformat_string<_Args...> __fmt, _Args&&... __args) { # ifndef _LIBCPP_HAS_NO_LOCALIZATION template -requires(output_iterator<_OutIt, const _CharT&>) _LIBCPP_HIDE_FROM_ABI _OutIt - __vformat_to( - _OutIt __out_it, locale __loc, basic_string_view<_CharT> __fmt, - basic_format_args> __args) { + requires(output_iterator<_OutIt, const _CharT&>) +_LIBCPP_HIDE_FROM_ABI _OutIt __vformat_to( + _OutIt __out_it, + locale __loc, + basic_string_view<_CharT> __fmt, + basic_format_args> __args) { if constexpr (same_as<_OutIt, _FormatOutIt>) - return std::__format::__vformat_to( - basic_format_parse_context{__fmt, __args.__size()}, - std::__format_context_create(std::move(__out_it), __args, std::move(__loc))); + return std::__format::__vformat_to(basic_format_parse_context{__fmt, __args.__size()}, + std::__format_context_create(std::move(__out_it), __args, std::move(__loc))); else { __format::__format_buffer<_OutIt, _CharT> __buffer{std::move(__out_it)}; std::__format::__vformat_to( @@ -557,36 +552,32 @@ requires(output_iterator<_OutIt, const _CharT&>) _LIBCPP_HIDE_FROM_ABI _OutIt } template _OutIt> -_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt vformat_to( - _OutIt __out_it, locale __loc, string_view __fmt, format_args __args) { - return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, - __args); +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt +vformat_to(_OutIt __out_it, locale __loc, string_view __fmt, format_args __args) { + return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, __args); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt> -_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt vformat_to( - _OutIt __out_it, locale __loc, wstring_view __fmt, wformat_args __args) { - return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, - __args); +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt +vformat_to(_OutIt __out_it, locale __loc, wstring_view __fmt, wformat_args __args) { + return std::__vformat_to(std::move(__out_it), std::move(__loc), __fmt, __args); } -#endif +# endif template _OutIt, class... _Args> _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt format_to(_OutIt __out_it, locale __loc, format_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), - std::make_format_args(__args...)); + return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), std::make_format_args(__args...)); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt, class... _Args> _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _OutIt format_to(_OutIt __out_it, locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), - std::make_wformat_args(__args...)); + return std::vformat_to(std::move(__out_it), std::move(__loc), __fmt.get(), std::make_wformat_args(__args...)); } -#endif +# endif // TODO FMT This needs to be a template or std::to_chars(floating-point) availability markup // fires too eagerly, see http://llvm.org/PR61563. @@ -594,8 +585,7 @@ template _LIBCPP_NODISCARD_EXT _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI string vformat(locale __loc, string_view __fmt, format_args __args) { string __res; - std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, - __args); + std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args); return __res; } @@ -606,8 +596,7 @@ template _LIBCPP_NODISCARD_EXT _LIBCPP_ALWAYS_INLINE inline _LIBCPP_HIDE_FROM_ABI wstring vformat(locale __loc, wstring_view __fmt, wformat_args __args) { wstring __res; - std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, - __args); + std::vformat_to(std::back_inserter(__res), std::move(__loc), __fmt, __args); return __res; } # endif @@ -615,23 +604,24 @@ vformat(locale __loc, wstring_view __fmt, wformat_args __args) { template _LIBCPP_NODISCARD_EXT _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI string format(locale __loc, format_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat(std::move(__loc), __fmt.get(), - std::make_format_args(__args...)); + return std::vformat(std::move(__loc), __fmt.get(), std::make_format_args(__args...)); } # ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _LIBCPP_NODISCARD_EXT _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI wstring format(locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) { - return std::vformat(std::move(__loc), __fmt.get(), - std::make_wformat_args(__args...)); + return std::vformat(std::move(__loc), __fmt.get(), std::make_wformat_args(__args...)); } # endif template -_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __vformat_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, - locale __loc, basic_string_view<_CharT> __fmt, - basic_format_args<_Context> __args) { +_LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __vformat_to_n( + _OutIt __out_it, + iter_difference_t<_OutIt> __n, + locale __loc, + basic_string_view<_CharT> __fmt, + basic_format_args<_Context> __args) { __format::__format_to_n_buffer<_OutIt, _CharT> __buffer{std::move(__out_it), __n}; std::__format::__vformat_to( basic_format_parse_context{__fmt, __args.__size()}, @@ -640,22 +630,20 @@ _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> __vformat_to_n(_OutIt __out_it, } template _OutIt, class... _Args> -_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> -format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, format_string<_Args...> __fmt, - _Args&&... __args) { - return std::__vformat_to_n(std::move(__out_it), __n, std::move(__loc), __fmt.get(), - std::make_format_args(__args...)); +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> format_to_n( + _OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, format_string<_Args...> __fmt, _Args&&... __args) { + return std::__vformat_to_n( + std::move(__out_it), __n, std::move(__loc), __fmt.get(), std::make_format_args(__args...)); } -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _OutIt, class... _Args> -_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> -format_to_n(_OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, wformat_string<_Args...> __fmt, - _Args&&... __args) { - return std::__vformat_to_n(std::move(__out_it), __n, std::move(__loc), __fmt.get(), - std::make_wformat_args(__args...)); +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI format_to_n_result<_OutIt> format_to_n( + _OutIt __out_it, iter_difference_t<_OutIt> __n, locale __loc, wformat_string<_Args...> __fmt, _Args&&... __args) { + return std::__vformat_to_n( + std::move(__out_it), __n, std::move(__loc), __fmt.get(), std::make_wformat_args(__args...)); } -#endif +# endif template _LIBCPP_HIDE_FROM_ABI size_t __vformatted_size(locale __loc, basic_string_view<_CharT> __fmt, auto __args) { diff --git a/libcxx/include/__format/format_parse_context.h b/libcxx/include/__format/format_parse_context.h index 79f53f77d4a05..aefcd5497f3b9 100644 --- a/libcxx/include/__format/format_parse_context.h +++ b/libcxx/include/__format/format_parse_context.h @@ -26,32 +26,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS basic_format_parse_context { public: - using char_type = _CharT; + using char_type = _CharT; using const_iterator = typename basic_string_view<_CharT>::const_iterator; - using iterator = const_iterator; + using iterator = const_iterator; - _LIBCPP_HIDE_FROM_ABI - constexpr explicit basic_format_parse_context(basic_string_view<_CharT> __fmt, - size_t __num_args = 0) noexcept + _LIBCPP_HIDE_FROM_ABI constexpr explicit basic_format_parse_context( + basic_string_view<_CharT> __fmt, size_t __num_args = 0) noexcept : __begin_(__fmt.begin()), __end_(__fmt.end()), __indexing_(__unknown), __next_arg_id_(0), __num_args_(__num_args) {} - basic_format_parse_context(const basic_format_parse_context&) = delete; - basic_format_parse_context& - operator=(const basic_format_parse_context&) = delete; + basic_format_parse_context(const basic_format_parse_context&) = delete; + basic_format_parse_context& operator=(const basic_format_parse_context&) = delete; - _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept { - return __begin_; - } - _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { - return __end_; - } - _LIBCPP_HIDE_FROM_ABI constexpr void advance_to(const_iterator __it) { - __begin_ = __it; - } + _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept { return __begin_; } + _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { return __end_; } + _LIBCPP_HIDE_FROM_ABI constexpr void advance_to(const_iterator __it) { __begin_ = __it; } _LIBCPP_HIDE_FROM_ABI constexpr size_t next_arg_id() { if (__indexing_ == __manual) @@ -102,9 +94,9 @@ class _LIBCPP_TEMPLATE_VIS basic_format_parse_context { _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(basic_format_parse_context); using format_parse_context = basic_format_parse_context; -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS using wformat_parse_context = basic_format_parse_context; -#endif +# endif #endif //_LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__format/format_string.h b/libcxx/include/__format/format_string.h index 2e1c71b3d01b2..bdf3cff7f49b1 100644 --- a/libcxx/include/__format/format_string.h +++ b/libcxx/include/__format/format_string.h @@ -38,8 +38,7 @@ template __parse_number_result(_Iterator, uint32_t) -> __parse_number_result<_Iterator>; template -_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator> -__parse_number(_Iterator __begin, _Iterator __end); +_LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator> __parse_number(_Iterator __begin, _Iterator __end); /** * The maximum value of a numeric argument. @@ -66,9 +65,7 @@ template _LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator> __parse_automatic(_Iterator __begin, _Iterator, auto& __parse_ctx) { size_t __value = __parse_ctx.next_arg_id(); - _LIBCPP_ASSERT_UNCATEGORIZED( - __value <= __number_max, - "Compilers don't support this number of arguments"); + _LIBCPP_ASSERT_UNCATEGORIZED(__value <= __number_max, "Compilers don't support this number of arguments"); return {__begin, uint32_t(__value)}; } @@ -93,8 +90,7 @@ template _LIBCPP_HIDE_FROM_ABI constexpr __parse_number_result<_Iterator> __parse_number(_Iterator __begin, _Iterator __end_input) { using _CharT = iter_value_t<_Iterator>; - static_assert(__format::__number_max == INT32_MAX, - "The algorithm is implemented based on this value."); + static_assert(__format::__number_max == INT32_MAX, "The algorithm is implemented based on this value."); /* * Limit the input to 9 digits, otherwise we need two checks during every * iteration: @@ -102,7 +98,7 @@ __parse_number(_Iterator __begin, _Iterator __end_input) { * - Does the value exceed width of an uint32_t? (Switching to uint64_t would * have the same issue, but with a higher maximum.) */ - _Iterator __end = __end_input - __begin > 9 ? __begin + 9 : __end_input; + _Iterator __end = __end_input - __begin > 9 ? __begin + 9 : __end_input; uint32_t __value = *__begin - _CharT('0'); while (++__begin != __end) { if (*__begin < _CharT('0') || *__begin > _CharT('9')) @@ -111,9 +107,7 @@ __parse_number(_Iterator __begin, _Iterator __end_input) { __value = __value * 10 + *__begin - _CharT('0'); } - if (__begin != __end_input && *__begin >= _CharT('0') && - *__begin <= _CharT('9')) { - + if (__begin != __end_input && *__begin >= _CharT('0') && *__begin <= _CharT('9')) { /* * There are more than 9 digits, do additional validations: * - Does the 10th digit exceed the maximum allowed value? @@ -121,9 +115,7 @@ __parse_number(_Iterator __begin, _Iterator __end_input) { * (More than 10 digits always overflows the maximum.) */ uint64_t __v = uint64_t(__value) * 10 + *__begin++ - _CharT('0'); - if (__v > __number_max || - (__begin != __end_input && *__begin >= _CharT('0') && - *__begin <= _CharT('9'))) + if (__v > __number_max || (__begin != __end_input && *__begin >= _CharT('0') && *__begin <= _CharT('9'))) std::__throw_format_error("The numeric value of the format specifier is too large"); __value = __v; diff --git a/libcxx/include/__format/formatter.h b/libcxx/include/__format/formatter.h index 172b2d5f7b8a1..079befc5bd9ca 100644 --- a/libcxx/include/__format/formatter.h +++ b/libcxx/include/__format/formatter.h @@ -33,8 +33,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD /// - is_move_assignable. template struct _LIBCPP_TEMPLATE_VIS formatter { - formatter() = delete; - formatter(const formatter&) = delete; + formatter() = delete; + formatter(const formatter&) = delete; formatter& operator=(const formatter&) = delete; }; diff --git a/libcxx/include/__format/formatter_floating_point.h b/libcxx/include/__format/formatter_floating_point.h index b45c2a6ebd39b..33cc2a4ed6612 100644 --- a/libcxx/include/__format/formatter_floating_point.h +++ b/libcxx/include/__format/formatter_floating_point.h @@ -116,8 +116,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr size_t __float_buffer_size(int __precision) { template <> struct __traits { - static constexpr int __max_integral = 38; - static constexpr int __max_fractional = 149; + static constexpr int __max_integral = 38; + static constexpr int __max_fractional = 149; static constexpr int __max_fractional_value = 3; static constexpr size_t __stack_buffer_size = 256; @@ -126,8 +126,8 @@ struct __traits { template <> struct __traits { - static constexpr int __max_integral = 308; - static constexpr int __max_fractional = 1074; + static constexpr int __max_integral = 308; + static constexpr int __max_fractional = 1074; static constexpr int __max_fractional_value = 4; static constexpr size_t __stack_buffer_size = 1024; @@ -153,7 +153,6 @@ class _LIBCPP_TEMPLATE_VIS __float_buffer { // required. explicit _LIBCPP_HIDE_FROM_ABI __float_buffer(int __precision) : __precision_(__precision != -1 ? __precision : _Traits::__max_fractional) { - // When the precision is larger than _Traits::__max_fractional the digits in // the range (_Traits::__max_fractional, precision] will contain the value // zero. There's no need to request to_chars to write these zeros: @@ -165,7 +164,7 @@ class _LIBCPP_TEMPLATE_VIS __float_buffer { // to be converted from a char to a wchar_t. if (__precision_ > _Traits::__max_fractional) { __num_trailing_zeros_ = __precision_ - _Traits::__max_fractional; - __precision_ = _Traits::__max_fractional; + __precision_ = _Traits::__max_fractional; } __size_ = __formatter::__float_buffer_size<_Fp>(__precision_); @@ -180,7 +179,7 @@ class _LIBCPP_TEMPLATE_VIS __float_buffer { if (__size_ > _Traits::__stack_buffer_size) allocator{}.deallocate(__begin_, __size_); } - _LIBCPP_HIDE_FROM_ABI __float_buffer(const __float_buffer&) = delete; + _LIBCPP_HIDE_FROM_ABI __float_buffer(const __float_buffer&) = delete; _LIBCPP_HIDE_FROM_ABI __float_buffer& operator=(const __float_buffer&) = delete; _LIBCPP_HIDE_FROM_ABI char* begin() const { return __begin_; } @@ -234,8 +233,8 @@ constexpr inline _LIBCPP_HIDE_FROM_ABI char* __find_exponent(char* __first, char } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_default(const __float_buffer<_Fp>& __buffer, _Tp __value, - char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result +__format_buffer_default(const __float_buffer<_Fp>& __buffer, _Tp __value, char* __integral) { __float_result __result; __result.__integral = __integral; __result.__last = __formatter::__to_buffer(__integral, __buffer.end(), __value); @@ -263,9 +262,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_default(const __float_buffe } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case(const __float_buffer<_Fp>& __buffer, - _Tp __value, int __precision, - char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case( + const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result; __result.__integral = __integral; if (__precision == -1) @@ -297,12 +295,12 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case(cons // 0123456789 static_assert(__traits<_Fp>::__hex_precision_digits <= 4, "Guard against possible underflow."); - char* __last = __result.__last - 2; - __first = __last - __traits<_Fp>::__hex_precision_digits; + char* __last = __result.__last - 2; + __first = __last - __traits<_Fp>::__hex_precision_digits; __result.__exponent = std::find(__first, __last, 'p'); } else { __result.__radix_point = __result.__last; - __result.__exponent = __first; + __result.__exponent = __first; } // clang-format off @@ -316,9 +314,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case(cons } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_upper_case(const __float_buffer<_Fp>& __buffer, - _Tp __value, int __precision, - char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_upper_case( + const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result = __formatter::__format_buffer_hexadecimal_lower_case(__buffer, __value, __precision, __integral); std::transform(__result.__integral, __result.__exponent, __result.__integral, __hex_to_upper); @@ -327,9 +324,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_upper_case(cons } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case(const __float_buffer<_Fp>& __buffer, - _Tp __value, int __precision, - char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case( + const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result; __result.__integral = __integral; __result.__last = @@ -342,7 +338,7 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case(const __result.__exponent = __formatter::__find_exponent(__first + 1, __result.__last); } else { __result.__radix_point = __result.__last; - __result.__exponent = __first; + __result.__exponent = __first; } // clang-format off @@ -355,9 +351,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case(const } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_upper_case(const __float_buffer<_Fp>& __buffer, - _Tp __value, int __precision, - char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_upper_case( + const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result = __formatter::__format_buffer_scientific_lower_case(__buffer, __value, __precision, __integral); *__result.__exponent = 'E'; @@ -365,8 +360,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_upper_case(const } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_fixed(const __float_buffer<_Fp>& __buffer, _Tp __value, - int __precision, char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result +__format_buffer_fixed(const __float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result; __result.__integral = __integral; __result.__last = __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::fixed, __precision); @@ -376,7 +371,7 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_fixed(const __float_buffer< // By converting __precision to a bool the subtraction can be done // unconditionally. __result.__radix_point = __result.__last - (__precision + bool(__precision)); - __result.__exponent = __result.__last; + __result.__exponent = __result.__last; // clang-format off _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && @@ -388,9 +383,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_fixed(const __float_buffer< } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_general_lower_case(__float_buffer<_Fp>& __buffer, _Tp __value, - int __precision, char* __integral) { - +_LIBCPP_HIDE_FROM_ABI __float_result +__format_buffer_general_lower_case(__float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __buffer.__remove_trailing_zeros(); __float_result __result; @@ -400,7 +394,7 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_general_lower_case(__float_ char* __first = __integral + 1; if (__first == __result.__last) { __result.__radix_point = __result.__last; - __result.__exponent = __result.__last; + __result.__exponent = __result.__last; } else { __result.__exponent = __formatter::__find_exponent(__first, __result.__last); if (__result.__exponent != __result.__last) @@ -426,8 +420,8 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_general_lower_case(__float_ } template -_LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_general_upper_case(__float_buffer<_Fp>& __buffer, _Tp __value, - int __precision, char* __integral) { +_LIBCPP_HIDE_FROM_ABI __float_result +__format_buffer_general_upper_case(__float_buffer<_Fp>& __buffer, _Tp __value, int __precision, char* __integral) { __float_result __result = __formatter::__format_buffer_general_lower_case(__buffer, __value, __precision, __integral); if (__result.__exponent != __result.__last) *__result.__exponent = 'E'; @@ -504,9 +498,9 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __format_locale_specific_form( const __float_result& __result, std::locale __loc, __format_spec::__parsed_specifications<_CharT> __specs) { - const auto& __np = std::use_facet>(__loc); + const auto& __np = std::use_facet>(__loc); string __grouping = __np.grouping(); - char* __first = __result.__integral; + char* __first = __result.__integral; // When no radix point or exponent are present __last will be __result.__last. char* __last = std::min(__result.__radix_point, __result.__exponent); @@ -524,11 +518,11 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __format_locale_specific_form( __grouping.size() - // Grouping contains one !__grouping.empty(); // additional character - __formatter::__padding_size_result __padding = {0, 0}; - bool __zero_padding = __specs.__alignment_ == __format_spec::__alignment::__zero_padding; + __formatter::__padding_size_result __padding = {0, 0}; + bool __zero_padding = __specs.__alignment_ == __format_spec::__alignment::__zero_padding; if (__size < __specs.__width_) { if (__zero_padding) { - __specs.__alignment_ = __format_spec::__alignment::__right; + __specs.__alignment_ = __format_spec::__alignment::__right; __specs.__fill_.__data[0] = _CharT('0'); } @@ -546,8 +540,8 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __format_locale_specific_form( if (__grouping.empty()) { __out_it = __formatter::__copy(__first, __digits, std::move(__out_it)); } else { - auto __r = __grouping.rbegin(); - auto __e = __grouping.rend() - 1; + auto __r = __grouping.rbegin(); + auto __e = __grouping.rend() - 1; _CharT __sep = __np.thousands_sep(); // The output is divided in small groups of numbers to write: // - A group before the first separator. @@ -627,8 +621,8 @@ _LIBCPP_HIDE_FROM_ABI auto __write_using_trailing_zeros( const _CharT* __exponent, size_t __num_trailing_zeros) -> decltype(__out_it) { _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "Not a valid range"); - _LIBCPP_ASSERT_UNCATEGORIZED(__num_trailing_zeros > 0, - "The overload not writing trailing zeros should have been used"); + _LIBCPP_ASSERT_UNCATEGORIZED( + __num_trailing_zeros > 0, "The overload not writing trailing zeros should have been used"); __padding_size_result __padding = __formatter::__padding_size(__size + __num_trailing_zeros, __specs.__width_, __specs.__alignment_); @@ -639,7 +633,6 @@ _LIBCPP_HIDE_FROM_ABI auto __write_using_trailing_zeros( return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_); } - template _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_floating_point(_Tp __value, _FormatContext& __ctx, __format_spec::__parsed_specifications<_CharT> __specs) { @@ -743,7 +736,7 @@ __format_floating_point(_Tp __value, _FormatContext& __ctx, __format_spec::__par *__out_it++ = *__first++; // After the sign is written, zero padding is the same a right alignment // with '0'. - __specs.__alignment_ = __format_spec::__alignment::__right; + __specs.__alignment_ = __format_spec::__alignment::__right; __specs.__fill_.__data[0] = _CharT('0'); } @@ -775,14 +768,11 @@ struct _LIBCPP_TEMPLATE_VIS __formatter_floating_point { }; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_floating_point<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_floating_point<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_floating_point<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_floating_point<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_floating_point<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_floating_point<_CharT> {}; #endif //_LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__format/formatter_integer.h b/libcxx/include/__format/formatter_integer.h index 5590bff5aa98b..d57082b3881ba 100644 --- a/libcxx/include/__format/formatter_integer.h +++ b/libcxx/include/__format/formatter_integer.h @@ -26,13 +26,12 @@ # pragma GCC system_header #endif - _LIBCPP_BEGIN_NAMESPACE_STD +_LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 - template <__fmt_char_type _CharT> - struct _LIBCPP_TEMPLATE_VIS __formatter_integer { - +template <__fmt_char_type _CharT> +struct _LIBCPP_TEMPLATE_VIS __formatter_integer { public: template _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) { @@ -60,44 +59,34 @@ // Signed integral types. template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> { -}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; # ifndef _LIBCPP_HAS_NO_INT128 template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter<__int128_t, _CharT> - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter<__int128_t, _CharT> : public __formatter_integer<_CharT> {}; # endif // Unsigned integral types. template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_integer<_CharT> {}; # ifndef _LIBCPP_HAS_NO_INT128 template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter<__uint128_t, _CharT> - : public __formatter_integer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter<__uint128_t, _CharT> : public __formatter_integer<_CharT> {}; # endif #endif //_LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__format/formatter_integral.h b/libcxx/include/__format/formatter_integral.h index cbb3505bca2f2..ca66e26ede107 100644 --- a/libcxx/include/__format/formatter_integral.h +++ b/libcxx/include/__format/formatter_integral.h @@ -90,9 +90,10 @@ _LIBCPP_HIDE_FROM_ABI inline _Iterator __insert_sign(_Iterator __buf, bool __neg * regardless whether the @c std::numpunct's type is @c char or @c wchar_t. */ _LIBCPP_HIDE_FROM_ABI inline string __determine_grouping(ptrdiff_t __size, const string& __grouping) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__grouping.empty() && __size > __grouping[0], - "The slow grouping formatting is used while there will be no " - "separators written"); + _LIBCPP_ASSERT_UNCATEGORIZED( + !__grouping.empty() && __size > __grouping[0], + "The slow grouping formatting is used while there will be no " + "separators written"); string __r; auto __end = __grouping.end() - 1; auto __ptr = __grouping.begin(); @@ -124,10 +125,10 @@ _LIBCPP_HIDE_FROM_ABI inline string __determine_grouping(ptrdiff_t __size, const // template <__fmt_char_type _CharT> -_LIBCPP_HIDE_FROM_ABI auto __format_char( - integral auto __value, - output_iterator auto __out_it, - __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) { +_LIBCPP_HIDE_FROM_ABI auto +__format_char(integral auto __value, + output_iterator auto __out_it, + __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) { using _Tp = decltype(__value); if constexpr (!same_as<_CharT, _Tp>) { // cmp_less and cmp_greater can't be used for character types. @@ -212,9 +213,14 @@ consteval size_t __buffer_size() noexcept template requires same_as> -_LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, _Iterator __begin, _Iterator __first, - _Iterator __last, string&& __grouping, _CharT __sep, - __format_spec::__parsed_specifications<_CharT> __specs) { +_LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators( + _OutIt __out_it, + _Iterator __begin, + _Iterator __first, + _Iterator __last, + string&& __grouping, + _CharT __sep, + __format_spec::__parsed_specifications<_CharT> __specs) { int __size = (__first - __begin) + // [sign][prefix] (__last - __first) + // data (__grouping.size() - 1); // number of separator characters @@ -242,8 +248,10 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, _ auto __r = __grouping.rbegin(); auto __e = __grouping.rend() - 1; - _LIBCPP_ASSERT_UNCATEGORIZED(__r != __e, "The slow grouping formatting is used while " - "there will be no separators written."); + _LIBCPP_ASSERT_UNCATEGORIZED( + __r != __e, + "The slow grouping formatting is used while " + "there will be no separators written."); // The output is divided in small groups of numbers to write: // - A group before the first separator. // - A separator and a group, repeated for the number of separators. @@ -257,9 +265,9 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, _ // hoisting the invariant is worth the effort. while (true) { if (__specs.__std_.__type_ == __format_spec::__type::__hexadecimal_upper_case) { - __last = __first + *__r; + __last = __first + *__r; __out_it = __formatter::__transform(__first, __last, std::move(__out_it), __hex_to_upper); - __first = __last; + __first = __last; } else { __out_it = __formatter::__copy(__first, *__r, std::move(__out_it)); __first += *__r; @@ -275,8 +283,6 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators(_OutIt __out_it, _ return __formatter::__fill(std::move(__out_it), __padding.__after_, __specs.__fill_); } - - template requires same_as> _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_integer( @@ -324,10 +330,10 @@ _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator __format_integer( // The zero padding is done like: // - Write [sign][prefix] // - Write data right aligned with '0' as fill character. - __out_it = __formatter::__copy(__begin, __first, std::move(__out_it)); - __specs.__alignment_ = __format_spec::__alignment::__right; + __out_it = __formatter::__copy(__begin, __first, std::move(__out_it)); + __specs.__alignment_ = __format_spec::__alignment::__right; __specs.__fill_.__data[0] = _CharT('0'); - int32_t __size = __first - __begin; + int32_t __size = __first - __begin; __specs.__width_ -= std::min(__size, __specs.__width_); } diff --git a/libcxx/include/__format/formatter_output.h b/libcxx/include/__format/formatter_output.h index 6c7892d86900d..89854f67f5fc1 100644 --- a/libcxx/include/__format/formatter_output.h +++ b/libcxx/include/__format/formatter_output.h @@ -102,8 +102,7 @@ _LIBCPP_HIDE_FROM_ABI auto __copy(basic_string_view<_CharT> __str, output_iterat if constexpr (std::same_as>>) { __out_it.__get_container()->__copy(__str); return __out_it; - } else if constexpr (std::same_as::__iterator>) { + } else if constexpr (std::same_as::__iterator>) { __out_it.__buffer_->__copy(__str); return __out_it; } else { @@ -134,16 +133,13 @@ template ::value_type, __fmt_char_type _OutCharT = _CharT, class _UnaryOperation> -_LIBCPP_HIDE_FROM_ABI auto -__transform(_Iterator __first, - _Iterator __last, - output_iterator auto __out_it, - _UnaryOperation __operation) -> decltype(__out_it) { +_LIBCPP_HIDE_FROM_ABI auto __transform( + _Iterator __first, _Iterator __last, output_iterator auto __out_it, _UnaryOperation __operation) + -> decltype(__out_it) { if constexpr (std::same_as>>) { __out_it.__get_container()->__transform(__first, __last, std::move(__operation)); return __out_it; - } else if constexpr (std::same_as::__iterator>) { + } else if constexpr (std::same_as::__iterator>) { __out_it.__buffer_->__transform(__first, __last, std::move(__operation)); return __out_it; } else { @@ -271,10 +267,12 @@ template ::value_type, class _ParserCharT, class _UnaryOperation> -_LIBCPP_HIDE_FROM_ABI auto __write_transformed(_Iterator __first, _Iterator __last, - output_iterator auto __out_it, - __format_spec::__parsed_specifications<_ParserCharT> __specs, - _UnaryOperation __op) -> decltype(__out_it) { +_LIBCPP_HIDE_FROM_ABI auto __write_transformed( + _Iterator __first, + _Iterator __last, + output_iterator auto __out_it, + __format_spec::__parsed_specifications<_ParserCharT> __specs, + _UnaryOperation __op) -> decltype(__out_it) { _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "Not a valid range"); ptrdiff_t __size = __last - __first; diff --git a/libcxx/include/__format/formatter_pointer.h b/libcxx/include/__format/formatter_pointer.h index a2212611df488..3373996ec3d5f 100644 --- a/libcxx/include/__format/formatter_pointer.h +++ b/libcxx/include/__format/formatter_pointer.h @@ -60,14 +60,11 @@ struct _LIBCPP_TEMPLATE_VIS __formatter_pointer { // - template<> struct formatter; // - template<> struct formatter; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_pointer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_pointer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_pointer<_CharT> { -}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_pointer<_CharT> {}; template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_pointer<_CharT> {}; +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_pointer<_CharT> {}; #endif //_LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__format/formatter_string.h b/libcxx/include/__format/formatter_string.h index ba64a64af80a7..4ba5617a49c8d 100644 --- a/libcxx/include/__format/formatter_string.h +++ b/libcxx/include/__format/formatter_string.h @@ -59,14 +59,15 @@ struct _LIBCPP_TEMPLATE_VIS __formatter_string { // Formatter const char*. template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter - : public __formatter_string<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter : public __formatter_string<_CharT> { using _Base = __formatter_string<_CharT>; template _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _CharT* __str, _FormatContext& __ctx) const { - _LIBCPP_ASSERT_UNCATEGORIZED(__str, "The basic_format_arg constructor should have " - "prevented an invalid pointer."); + _LIBCPP_ASSERT_UNCATEGORIZED( + __str, + "The basic_format_arg constructor should have " + "prevented an invalid pointer."); __format_spec::__parsed_specifications<_CharT> __specs = _Base::__parser_.__get_parsed_std_specifications(__ctx); # if _LIBCPP_STD_VER >= 23 @@ -98,8 +99,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter // Formatter char*. template <__fmt_char_type _CharT> -struct _LIBCPP_TEMPLATE_VIS formatter<_CharT*, _CharT> - : public formatter { +struct _LIBCPP_TEMPLATE_VIS formatter<_CharT*, _CharT> : public formatter { using _Base = formatter; template @@ -110,8 +110,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter<_CharT*, _CharT> // Formatter char[]. template <__fmt_char_type _CharT, size_t _Size> -struct _LIBCPP_TEMPLATE_VIS formatter<_CharT[_Size], _CharT> - : public __formatter_string<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter<_CharT[_Size], _CharT> : public __formatter_string<_CharT> { using _Base = __formatter_string<_CharT>; template @@ -137,8 +136,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter, // Formatter std::string_view. template <__fmt_char_type _CharT, class _Traits> -struct _LIBCPP_TEMPLATE_VIS formatter, _CharT> - : public __formatter_string<_CharT> { +struct _LIBCPP_TEMPLATE_VIS formatter, _CharT> : public __formatter_string<_CharT> { using _Base = __formatter_string<_CharT>; template diff --git a/libcxx/include/__format/parser_std_format_spec.h b/libcxx/include/__format/parser_std_format_spec.h index ea5dfdf30511c..9a91179fdfb52 100644 --- a/libcxx/include/__format/parser_std_format_spec.h +++ b/libcxx/include/__format/parser_std_format_spec.h @@ -82,8 +82,7 @@ __parse_arg_id(_Iterator __begin, _Iterator __end, _ParseContext& __ctx) { } template -_LIBCPP_HIDE_FROM_ABI constexpr uint32_t -__substitute_arg_id(basic_format_arg<_Context> __format_arg) { +_LIBCPP_HIDE_FROM_ABI constexpr uint32_t __substitute_arg_id(basic_format_arg<_Context> __format_arg) { // [format.string.std]/8 // If the corresponding formatting argument is not of integral type... // This wording allows char and bool too. LWG-3720 changes the wording to @@ -240,22 +239,22 @@ inline constexpr uint32_t __type_mask_integer = __create_type_mask(__type::__hexadecimal_upper_case); struct __std { - __alignment __alignment_ : 3; - __sign __sign_ : 2; - bool __alternate_form_ : 1; + __alignment __alignment_ : 3; + __sign __sign_ : 2; + bool __alternate_form_ : 1; bool __locale_specific_form_ : 1; __type __type_; }; struct __chrono { - __alignment __alignment_ : 3; + __alignment __alignment_ : 3; bool __locale_specific_form_ : 1; bool __hour_ : 1; - bool __weekday_name_ : 1; + bool __weekday_name_ : 1; bool __weekday_ : 1; bool __day_of_year_ : 1; bool __week_of_year_ : 1; - bool __month_name_ : 1; + bool __month_name_ : 1; }; // The fill UCS scalar value. @@ -495,8 +494,7 @@ class _LIBCPP_TEMPLATE_VIS __parser { } /// \returns the `__parsed_specifications` with the resolved dynamic sizes.. - _LIBCPP_HIDE_FROM_ABI - __parsed_specifications<_CharT> __get_parsed_std_specifications(auto& __ctx) const { + _LIBCPP_HIDE_FROM_ABI __parsed_specifications<_CharT> __get_parsed_std_specifications(auto& __ctx) const { return __parsed_specifications<_CharT>{ .__std_ = __std{.__alignment_ = __alignment_, .__sign_ = __sign_, @@ -524,9 +522,9 @@ class _LIBCPP_TEMPLATE_VIS __parser { .__fill_{__fill_}}; } - __alignment __alignment_ : 3 {__alignment::__default}; - __sign __sign_ : 2 {__sign::__default}; - bool __alternate_form_ : 1 {false}; + __alignment __alignment_ : 3 {__alignment::__default}; + __sign __sign_ : 2 {__sign::__default}; + bool __alternate_form_ : 1 {false}; bool __locale_specific_form_ : 1 {false}; bool __clear_brackets_ : 1 {false}; __type __type_{__type::__default}; @@ -547,7 +545,7 @@ class _LIBCPP_TEMPLATE_VIS __parser { uint8_t __reserved_1_ : 6 {0}; // These two flags are only used internally and not part of the // __parsed_specifications. Therefore put them at the end. - bool __width_as_arg_ : 1 {false}; + bool __width_as_arg_ : 1 {false}; bool __precision_as_arg_ : 1 {false}; /// The requested width, either the value or the arg-id. @@ -593,9 +591,10 @@ class _LIBCPP_TEMPLATE_VIS __parser { || (same_as<_CharT, wchar_t> && sizeof(wchar_t) == 2) # endif _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end, bool __use_range_fill) { - _LIBCPP_ASSERT_UNCATEGORIZED(__begin != __end, - "when called with an empty input the function will cause " - "undefined behavior by evaluating data not in the input"); + _LIBCPP_ASSERT_UNCATEGORIZED( + __begin != __end, + "when called with an empty input the function will cause " + "undefined behavior by evaluating data not in the input"); __unicode::__code_point_view<_CharT> __view{__begin, __end}; __unicode::__consume_result __consumed = __view.__consume(); if (__consumed.__status != __unicode::__consume_result::__ok) @@ -625,9 +624,10 @@ class _LIBCPP_TEMPLATE_VIS __parser { template requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 4) _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end, bool __use_range_fill) { - _LIBCPP_ASSERT_UNCATEGORIZED(__begin != __end, - "when called with an empty input the function will cause " - "undefined behavior by evaluating data not in the input"); + _LIBCPP_ASSERT_UNCATEGORIZED( + __begin != __end, + "when called with an empty input the function will cause " + "undefined behavior by evaluating data not in the input"); if (__begin + 1 != __end && __parse_alignment(*(__begin + 1))) { if (!__unicode::__is_scalar_value(*__begin)) std::__throw_format_error("The fill option contains an invalid value"); @@ -652,9 +652,10 @@ class _LIBCPP_TEMPLATE_VIS __parser { // range-fill and tuple-fill are identical template _LIBCPP_HIDE_FROM_ABI constexpr bool __parse_fill_align(_Iterator& __begin, _Iterator __end, bool __use_range_fill) { - _LIBCPP_ASSERT_UNCATEGORIZED(__begin != __end, - "when called with an empty input the function will cause " - "undefined behavior by evaluating data not in the input"); + _LIBCPP_ASSERT_UNCATEGORIZED( + __begin != __end, + "when called with an empty input the function will cause " + "undefined behavior by evaluating data not in the input"); if (__begin + 1 != __end) { if (__parse_alignment(*(__begin + 1))) { __validate_fill_character(*__begin, __use_range_fill); @@ -721,9 +722,9 @@ class _LIBCPP_TEMPLATE_VIS __parser { if (*__begin == _CharT('{')) { __format::__parse_number_result __r = __format_spec::__parse_arg_id(++__begin, __end, __ctx); - __width_as_arg_ = true; - __width_ = __r.__value; - __begin = __r.__last; + __width_as_arg_ = true; + __width_ = __r.__value; + __begin = __r.__last; return true; } @@ -731,9 +732,11 @@ class _LIBCPP_TEMPLATE_VIS __parser { return false; __format::__parse_number_result __r = __format::__parse_number(__begin, __end); - __width_ = __r.__value; - _LIBCPP_ASSERT_UNCATEGORIZED(__width_ != 0, "A zero value isn't allowed and should be impossible, " - "due to validations in this function"); + __width_ = __r.__value; + _LIBCPP_ASSERT_UNCATEGORIZED( + __width_ != 0, + "A zero value isn't allowed and should be impossible, " + "due to validations in this function"); __begin = __r.__last; return true; } @@ -749,9 +752,9 @@ class _LIBCPP_TEMPLATE_VIS __parser { if (*__begin == _CharT('{')) { __format::__parse_number_result __arg_id = __format_spec::__parse_arg_id(++__begin, __end, __ctx); - __precision_as_arg_ = true; - __precision_ = __arg_id.__value; - __begin = __arg_id.__last; + __precision_as_arg_ = true; + __precision_ = __arg_id.__value; + __begin = __arg_id.__last; return true; } @@ -759,9 +762,9 @@ class _LIBCPP_TEMPLATE_VIS __parser { std::__throw_format_error("The precision option does not contain a value or an argument index"); __format::__parse_number_result __r = __format::__parse_number(__begin, __end); - __precision_ = __r.__value; - __precision_as_arg_ = false; - __begin = __r.__last; + __precision_ = __r.__value; + __precision_as_arg_ = false; + __begin = __r.__last; return true; } @@ -858,16 +861,14 @@ class _LIBCPP_TEMPLATE_VIS __parser { ++__begin; } - _LIBCPP_HIDE_FROM_ABI - int32_t __get_width(auto& __ctx) const { + _LIBCPP_HIDE_FROM_ABI int32_t __get_width(auto& __ctx) const { if (!__width_as_arg_) return __width_; return __format_spec::__substitute_arg_id(__ctx.arg(__width_)); } - _LIBCPP_HIDE_FROM_ABI - int32_t __get_precision(auto& __ctx) const { + _LIBCPP_HIDE_FROM_ABI int32_t __get_precision(auto& __ctx) const { if (!__precision_as_arg_) return __precision_; diff --git a/libcxx/include/__format/unicode.h b/libcxx/include/__format/unicode.h index 8666f80c78a28..8e1e7bb192a00 100644 --- a/libcxx/include/__format/unicode.h +++ b/libcxx/include/__format/unicode.h @@ -319,9 +319,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __at_extended_grapheme_cluster_break( return true; // *** Do not break Hangul syllable sequences. *** - if (__prev == __property::__L && - (__next == __property::__L || __next == __property::__V || __next == __property::__LV || - __next == __property::__LVT)) // GB6 + if (__prev == __property::__L && (__next == __property::__L || __next == __property::__V || + __next == __property::__LV || __next == __property::__LVT)) // GB6 return false; if ((__prev == __property::__LV || __prev == __property::__V) && @@ -402,9 +401,8 @@ class __extended_grapheme_cluster_view { }; _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() { - _LIBCPP_ASSERT_UNCATEGORIZED( - __next_prop_ != __extended_grapheme_custer_property_boundary::__property::__eot, - "can't move beyond the end of input"); + _LIBCPP_ASSERT_UNCATEGORIZED(__next_prop_ != __extended_grapheme_custer_property_boundary::__property::__eot, + "can't move beyond the end of input"); char32_t __code_point = __next_code_point_; if (!__code_point_view_.__at_end()) diff --git a/libcxx/include/__format/write_escaped.h b/libcxx/include/__format/write_escaped.h index 51bae3cb238fb..15141eebc0292 100644 --- a/libcxx/include/__format/write_escaped.h +++ b/libcxx/include/__format/write_escaped.h @@ -32,7 +32,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD - namespace __formatter { #if _LIBCPP_STD_VER >= 20 @@ -42,10 +41,10 @@ namespace __formatter { /// \note When \c _LIBCPP_HAS_NO_UNICODE is defined the function assumes the /// input is ASCII. template -_LIBCPP_HIDE_FROM_ABI auto __write_string( - basic_string_view<_CharT> __str, - output_iterator auto __out_it, - __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) { +_LIBCPP_HIDE_FROM_ABI auto +__write_string(basic_string_view<_CharT> __str, + output_iterator auto __out_it, + __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) { if (!__specs.__has_precision()) return __formatter::__write_string_no_precision(__str, std::move(__out_it), __specs); @@ -54,8 +53,8 @@ _LIBCPP_HIDE_FROM_ABI auto __write_string( return __formatter::__write(__str.begin(), __str.end(), std::move(__out_it), __specs, __size); } -# endif // _LIBCPP_STD_VER >= 20 -# if _LIBCPP_STD_VER >= 23 +#endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 23 struct __nul_terminator {}; @@ -100,11 +99,11 @@ _LIBCPP_HIDE_FROM_ABI void __write_escape_ill_formed_code_unit(basic_string<_Cha template [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool __is_escaped_sequence_written(basic_string<_CharT>& __str, char32_t __value) { -# ifdef _LIBCPP_HAS_NO_UNICODE +# ifdef _LIBCPP_HAS_NO_UNICODE // For ASCII assume everything above 127 is printable. if (__value > 127) return false; -# endif +# endif if (!__escaped_output_table::__needs_escape(__value)) return false; @@ -213,7 +212,7 @@ __format_escaped_string(basic_string_view<_CharT> __values, return __formatter::__write_string(basic_string_view{__str}, std::move(__out_it), __specs); } -# endif // _LIBCPP_STD_VER >= 23 +#endif // _LIBCPP_STD_VER >= 23 } // namespace __formatter diff --git a/libcxx/include/__functional/binary_function.h b/libcxx/include/__functional/binary_function.h index fdedb8b177dae..ddee3b170311f 100644 --- a/libcxx/include/__functional/binary_function.h +++ b/libcxx/include/__functional/binary_function.h @@ -21,20 +21,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) template -struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binary_function -{ - typedef _Arg1 first_argument_type; - typedef _Arg2 second_argument_type; - typedef _Result result_type; +struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binary_function { + typedef _Arg1 first_argument_type; + typedef _Arg2 second_argument_type; + typedef _Result result_type; }; #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) -template struct __binary_function_keep_layout_base { +template +struct __binary_function_keep_layout_base { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using first_argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg1; + using first_argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg1; using second_argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg2; - using result_type _LIBCPP_DEPRECATED_IN_CXX17 = _Result; + using result_type _LIBCPP_DEPRECATED_IN_CXX17 = _Result; #endif }; diff --git a/libcxx/include/__functional/binary_negate.h b/libcxx/include/__functional/binary_negate.h index 3180284fc3822..ce52b5ae9fc49 100644 --- a/libcxx/include/__functional/binary_negate.h +++ b/libcxx/include/__functional/binary_negate.h @@ -25,23 +25,24 @@ template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 binary_negate : public __binary_function -{ - _Predicate __pred_; + bool> { + _Predicate __pred_; + public: - _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR_SINCE_CXX14 - binary_negate(const _Predicate& __pred) : __pred_(__pred) {} + _LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR_SINCE_CXX14 binary_negate(const _Predicate& __pred) + : __pred_(__pred) {} - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const typename _Predicate::first_argument_type& __x, - const typename _Predicate::second_argument_type& __y) const - {return !__pred_(__x, __y);} + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()( + const typename _Predicate::first_argument_type& __x, const typename _Predicate::second_argument_type& __y) const { + return !__pred_(__x, __y); + } }; template -_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI -binary_negate<_Predicate> -not2(const _Predicate& __pred) {return binary_negate<_Predicate>(__pred);} +_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI binary_negate<_Predicate> +not2(const _Predicate& __pred) { + return binary_negate<_Predicate>(__pred); +} #endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS) diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h index 61d99e1fd564f..19e7d82155ec9 100644 --- a/libcxx/include/__functional/bind.h +++ b/libcxx/include/__functional/bind.h @@ -25,34 +25,30 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -struct is_bind_expression : _If< - _IsSame<_Tp, __remove_cvref_t<_Tp> >::value, - false_type, - is_bind_expression<__remove_cvref_t<_Tp> > -> {}; +template +struct is_bind_expression + : _If< _IsSame<_Tp, __remove_cvref_t<_Tp> >::value, false_type, is_bind_expression<__remove_cvref_t<_Tp> > > {}; #if _LIBCPP_STD_VER >= 17 template inline constexpr bool is_bind_expression_v = is_bind_expression<_Tp>::value; #endif -template -struct is_placeholder : _If< - _IsSame<_Tp, __remove_cvref_t<_Tp> >::value, - integral_constant, - is_placeholder<__remove_cvref_t<_Tp> > -> {}; +template +struct is_placeholder + : _If< _IsSame<_Tp, __remove_cvref_t<_Tp> >::value, + integral_constant, + is_placeholder<__remove_cvref_t<_Tp> > > {}; #if _LIBCPP_STD_VER >= 17 template inline constexpr int is_placeholder_v = is_placeholder<_Tp>::value; #endif -namespace placeholders -{ +namespace placeholders { -template struct __ph {}; +template +struct __ph {}; // C++17 recommends that we implement placeholders as `inline constexpr`, but allows // implementing them as `extern `. Libc++ implements them as @@ -62,291 +58,234 @@ template struct __ph {}; // // In practice, since placeholders are empty, `extern const` is almost impossible // to distinguish from `inline constexpr` from a usage stand point. -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<1> _1; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<2> _2; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<3> _3; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<4> _4; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<5> _5; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<6> _6; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<7> _7; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<8> _8; -_LIBCPP_EXPORTED_FROM_ABI extern const __ph<9> _9; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<1> _1; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<2> _2; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<3> _3; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<4> _4; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<5> _5; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<6> _6; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<7> _7; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<8> _8; +_LIBCPP_EXPORTED_FROM_ABI extern const __ph<9> _9; _LIBCPP_EXPORTED_FROM_ABI extern const __ph<10> _10; } // namespace placeholders -template -struct is_placeholder > - : public integral_constant {}; - +template +struct is_placeholder > : public integral_constant {}; #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_Tp& -__mu(reference_wrapper<_Tp> __t, _Uj&) -{ - return __t.get(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __mu(reference_wrapper<_Tp> __t, _Uj&) { + return __t.get(); } -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -typename __invoke_of<_Ti&, _Uj...>::type -__mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>) -{ - return __ti(std::forward<_Uj>(std::get<_Indx>(__uj))...); +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of<_Ti&, _Uj...>::type +__mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>) { + return __ti(std::forward<_Uj>(std::get<_Indx>(__uj))...); } -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -typename __invoke_of<_Ti&, _Uj...>::type -__mu(_Ti& __ti, tuple<_Uj...>& __uj) -{ - typedef typename __make_tuple_indices::type __indices; - return std::__mu_expand(__ti, __uj, __indices()); +template ::value, int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of<_Ti&, _Uj...>::type +__mu(_Ti& __ti, tuple<_Uj...>& __uj) { + typedef typename __make_tuple_indices::type __indices; + return std::__mu_expand(__ti, __uj, __indices()); } template struct __mu_return2 {}; template -struct __mu_return2 -{ - typedef typename tuple_element::value - 1, _Uj>::type type; +struct __mu_return2 { + typedef typename tuple_element::value - 1, _Uj>::type type; }; template ::value, int> = 0> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -typename __mu_return2<0 < is_placeholder<_Ti>::value, _Ti, _Uj>::type -__mu(_Ti&, _Uj& __uj) -{ - const size_t __indx = is_placeholder<_Ti>::value - 1; - return std::forward::type>(std::get<__indx>(__uj)); + typename __mu_return2<0 < is_placeholder<_Ti>::value, _Ti, _Uj>::type + __mu(_Ti&, _Uj& __uj) { + const size_t __indx = is_placeholder<_Ti>::value - 1; + return std::forward::type>(std::get<__indx>(__uj)); } -template ::value && - is_placeholder<_Ti>::value == 0 && - !__is_reference_wrapper<_Ti>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -_Ti& -__mu(_Ti& __ti, _Uj&) -{ - return __ti; +template ::value && is_placeholder<_Ti>::value == 0 && + !__is_reference_wrapper<_Ti>::value, + int> = 0> +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ti& __mu(_Ti& __ti, _Uj&) { + return __ti; } -template +template struct __mu_return_impl; -template -struct __mu_return_invokable // false +template +struct __mu_return_invokable // false { - typedef __nat type; + typedef __nat type; }; -template -struct __mu_return_invokable -{ - typedef typename __invoke_of<_Ti&, _Uj...>::type type; +template +struct __mu_return_invokable { + typedef typename __invoke_of<_Ti&, _Uj...>::type type; }; -template +template struct __mu_return_impl<_Ti, false, true, false, tuple<_Uj...> > - : public __mu_return_invokable<__invokable<_Ti&, _Uj...>::value, _Ti, _Uj...> -{ -}; + : public __mu_return_invokable<__invokable<_Ti&, _Uj...>::value, _Ti, _Uj...> {}; template -struct __mu_return_impl<_Ti, false, false, true, _TupleUj> -{ - typedef typename tuple_element::value - 1, - _TupleUj>::type&& type; +struct __mu_return_impl<_Ti, false, false, true, _TupleUj> { + typedef typename tuple_element::value - 1, _TupleUj>::type&& type; }; template -struct __mu_return_impl<_Ti, true, false, false, _TupleUj> -{ - typedef typename _Ti::type& type; +struct __mu_return_impl<_Ti, true, false, false, _TupleUj> { + typedef typename _Ti::type& type; }; template -struct __mu_return_impl<_Ti, false, false, false, _TupleUj> -{ - typedef _Ti& type; +struct __mu_return_impl<_Ti, false, false, false, _TupleUj> { + typedef _Ti& type; }; template struct __mu_return - : public __mu_return_impl<_Ti, - __is_reference_wrapper<_Ti>::value, - is_bind_expression<_Ti>::value, - 0 < is_placeholder<_Ti>::value && - is_placeholder<_Ti>::value <= tuple_size<_TupleUj>::value, - _TupleUj> -{ -}; + : public __mu_return_impl< + _Ti, + __is_reference_wrapper<_Ti>::value, + is_bind_expression<_Ti>::value, + 0 < is_placeholder<_Ti>::value && is_placeholder<_Ti>::value <= tuple_size<_TupleUj>::value, + _TupleUj> {}; template -struct __is_valid_bind_return -{ - static const bool value = false; +struct __is_valid_bind_return { + static const bool value = false; }; -template -struct __is_valid_bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj> -{ - static const bool value = __invokable<_Fp, - typename __mu_return<_BoundArgs, _TupleUj>::type...>::value; +template +struct __is_valid_bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj> { + static const bool value = __invokable<_Fp, typename __mu_return<_BoundArgs, _TupleUj>::type...>::value; }; -template -struct __is_valid_bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj> -{ - static const bool value = __invokable<_Fp, - typename __mu_return::type...>::value; +template +struct __is_valid_bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj> { + static const bool value = __invokable<_Fp, typename __mu_return::type...>::value; }; -template ::value> +template ::value> struct __bind_return; -template -struct __bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj, true> -{ - typedef typename __invoke_of - < - _Fp&, - typename __mu_return - < - _BoundArgs, - _TupleUj - >::type... - >::type type; +template +struct __bind_return<_Fp, tuple<_BoundArgs...>, _TupleUj, true> { + typedef typename __invoke_of< _Fp&, typename __mu_return< _BoundArgs, _TupleUj >::type... >::type type; }; -template -struct __bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj, true> -{ - typedef typename __invoke_of - < - _Fp&, - typename __mu_return - < - const _BoundArgs, - _TupleUj - >::type... - >::type type; +template +struct __bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj, true> { + typedef typename __invoke_of< _Fp&, typename __mu_return< const _BoundArgs, _TupleUj >::type... >::type type; }; -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -typename __bind_return<_Fp, _BoundArgs, _Args>::type -__apply_functor(_Fp& __f, _BoundArgs& __bound_args, __tuple_indices<_Indx...>, - _Args&& __args) -{ - return std::__invoke(__f, std::__mu(std::get<_Indx>(__bound_args), __args)...); +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fp, _BoundArgs, _Args>::type +__apply_functor(_Fp& __f, _BoundArgs& __bound_args, __tuple_indices<_Indx...>, _Args&& __args) { + return std::__invoke(__f, std::__mu(std::get<_Indx>(__bound_args), __args)...); } -template -class __bind : public __weak_result_type<__decay_t<_Fp> > -{ +template +class __bind : public __weak_result_type<__decay_t<_Fp> > { protected: - using _Fd = __decay_t<_Fp>; - typedef tuple<__decay_t<_BoundArgs>...> _Td; + using _Fd = __decay_t<_Fp>; + typedef tuple<__decay_t<_BoundArgs>...> _Td; + private: - _Fd __f_; - _Td __bound_args_; + _Fd __f_; + _Td __bound_args_; + + typedef typename __make_tuple_indices::type __indices; - typedef typename __make_tuple_indices::type __indices; public: - template ::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - explicit __bind(_Gp&& __f, _BA&& ...__bound_args) - : __f_(std::forward<_Gp>(__f)), - __bound_args_(std::forward<_BA>(__bound_args)...) {} - - template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - typename __bind_return<_Fd, _Td, tuple<_Args&&...> >::type - operator()(_Args&& ...__args) - { - return std::__apply_functor(__f_, __bound_args_, __indices(), - tuple<_Args&&...>(std::forward<_Args>(__args)...)); - } - - template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - typename __bind_return >::type - operator()(_Args&& ...__args) const - { - return std::__apply_functor(__f_, __bound_args_, __indices(), - tuple<_Args&&...>(std::forward<_Args>(__args)...)); - } + template < + class _Gp, + class... _BA, + __enable_if_t::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind>::value, + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bind(_Gp&& __f, _BA&&... __bound_args) + : __f_(std::forward<_Gp>(__f)), __bound_args_(std::forward<_BA>(__bound_args)...) {} + + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fd, _Td, tuple<_Args&&...> >::type + operator()(_Args&&... __args) { + return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...)); + } + + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + typename __bind_return >::type + operator()(_Args&&... __args) const { + return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...)); + } }; -template +template struct is_bind_expression<__bind<_Fp, _BoundArgs...> > : public true_type {}; -template -class __bind_r - : public __bind<_Fp, _BoundArgs...> -{ - typedef __bind<_Fp, _BoundArgs...> base; - typedef typename base::_Fd _Fd; - typedef typename base::_Td _Td; +template +class __bind_r : public __bind<_Fp, _BoundArgs...> { + typedef __bind<_Fp, _BoundArgs...> base; + typedef typename base::_Fd _Fd; + typedef typename base::_Td _Td; + public: - typedef _Rp result_type; - - - template ::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind_r>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - explicit __bind_r(_Gp&& __f, _BA&& ...__bound_args) - : base(std::forward<_Gp>(__f), - std::forward<_BA>(__bound_args)...) {} - - template >::type, - result_type>::value || is_void<_Rp>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - result_type - operator()(_Args&& ...__args) - { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(static_cast(*this), std::forward<_Args>(__args)...); - } - - template >::type, - result_type>::value || is_void<_Rp>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - result_type - operator()(_Args&& ...__args) const - { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(static_cast(*this), std::forward<_Args>(__args)...); - } + typedef _Rp result_type; + + template < + class _Gp, + class... _BA, + __enable_if_t::value && !is_same<__libcpp_remove_reference_t<_Gp>, __bind_r>::value, + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bind_r(_Gp&& __f, _BA&&... __bound_args) + : base(std::forward<_Gp>(__f), std::forward<_BA>(__bound_args)...) {} + + template < + class... _Args, + __enable_if_t >::type, result_type>::value || + is_void<_Rp>::value, + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 result_type operator()(_Args&&... __args) { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(static_cast(*this), std::forward<_Args>(__args)...); + } + + template >::type, + result_type>::value || + is_void<_Rp>::value, + int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 result_type operator()(_Args&&... __args) const { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(static_cast(*this), std::forward<_Args>(__args)...); + } }; -template +template struct is_bind_expression<__bind_r<_Rp, _Fp, _BoundArgs...> > : public true_type {}; -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -__bind<_Fp, _BoundArgs...> -bind(_Fp&& __f, _BoundArgs&&... __bound_args) -{ - typedef __bind<_Fp, _BoundArgs...> type; - return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...); +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind<_Fp, _BoundArgs...> +bind(_Fp&& __f, _BoundArgs&&... __bound_args) { + typedef __bind<_Fp, _BoundArgs...> type; + return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...); } -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -__bind_r<_Rp, _Fp, _BoundArgs...> -bind(_Fp&& __f, _BoundArgs&&... __bound_args) -{ - typedef __bind_r<_Rp, _Fp, _BoundArgs...> type; - return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...); +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind_r<_Rp, _Fp, _BoundArgs...> +bind(_Fp&& __f, _BoundArgs&&... __bound_args) { + typedef __bind_r<_Rp, _Fp, _BoundArgs...> type; + return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...); } #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__functional/bind_back.h b/libcxx/include/__functional/bind_back.h index 5b5b40520933b..ce26d3b70630f 100644 --- a/libcxx/include/__functional/bind_back.h +++ b/libcxx/include/__functional/bind_back.h @@ -29,28 +29,38 @@ _LIBCPP_BEGIN_NAMESPACE_STD template > struct __bind_back_op; -template +template struct __bind_back_op<_NBound, index_sequence<_Ip...>> { - template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f, _BoundArgs&& __bound_args, _Args&&... __args) const - noexcept(noexcept(std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)..., std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...))) - -> decltype( std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)..., std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...)) - { return std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)..., std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...); } + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f, _BoundArgs&& __bound_args, _Args&&... __args) const + noexcept(noexcept(std::invoke(std::forward<_Fn>(__f), + std::forward<_Args>(__args)..., + std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...))) + -> decltype(std::invoke(std::forward<_Fn>(__f), + std::forward<_Args>(__args)..., + std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...)) { + return std::invoke(std::forward<_Fn>(__f), + std::forward<_Args>(__args)..., + std::get<_Ip>(std::forward<_BoundArgs>(__bound_args))...); + } }; template struct __bind_back_t : __perfect_forward<__bind_back_op>, _Fn, _BoundArgs> { - using __perfect_forward<__bind_back_op>, _Fn, _BoundArgs>::__perfect_forward; + using __perfect_forward<__bind_back_op>, _Fn, _BoundArgs>::__perfect_forward; }; template requires is_constructible_v, _Fn> && is_move_constructible_v> && (is_constructible_v, _Args> && ...) && (is_move_constructible_v> && ...) -_LIBCPP_HIDE_FROM_ABI -constexpr auto __bind_back(_Fn&& __f, _Args&&... __args) - noexcept(noexcept(__bind_back_t, tuple...>>(std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...)))) - -> decltype( __bind_back_t, tuple...>>(std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...))) - { return __bind_back_t, tuple...>>(std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...)); } +_LIBCPP_HIDE_FROM_ABI constexpr auto __bind_back(_Fn&& __f, _Args&&... __args) noexcept( + noexcept(__bind_back_t, tuple...>>( + std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...)))) + -> decltype(__bind_back_t, tuple...>>( + std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...))) { + return __bind_back_t, tuple...>>( + std::forward<_Fn>(__f), std::forward_as_tuple(std::forward<_Args>(__args)...)); +} #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__functional/bind_front.h b/libcxx/include/__functional/bind_front.h index d476243438768..30dda533615b2 100644 --- a/libcxx/include/__functional/bind_front.h +++ b/libcxx/include/__functional/bind_front.h @@ -29,25 +29,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 struct __bind_front_op { - template - _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()(_Args&& ...__args) const - noexcept(noexcept(std::invoke(std::forward<_Args>(__args)...))) - -> decltype( std::invoke(std::forward<_Args>(__args)...)) - { return std::invoke(std::forward<_Args>(__args)...); } + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const + noexcept(noexcept(std::invoke(std::forward<_Args>(__args)...))) + -> decltype(std::invoke(std::forward<_Args>(__args)...)) { + return std::invoke(std::forward<_Args>(__args)...); + } }; -template +template struct __bind_front_t : __perfect_forward<__bind_front_op, _Fn, _BoundArgs...> { - using __perfect_forward<__bind_front_op, _Fn, _BoundArgs...>::__perfect_forward; + using __perfect_forward<__bind_front_op, _Fn, _BoundArgs...>::__perfect_forward; }; template requires is_constructible_v, _Fn> && is_move_constructible_v> && (is_constructible_v, _Args> && ...) && (is_move_constructible_v> && ...) -_LIBCPP_HIDE_FROM_ABI -constexpr auto bind_front(_Fn&& __f, _Args&&... __args) { - return __bind_front_t, decay_t<_Args>...>(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); +_LIBCPP_HIDE_FROM_ABI constexpr auto bind_front(_Fn&& __f, _Args&&... __args) { + return __bind_front_t, decay_t<_Args>...>(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); } #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__functional/binder1st.h b/libcxx/include/__functional/binder1st.h index 9f0dc08ba6b1d..04b51fefab70a 100644 --- a/libcxx/include/__functional/binder1st.h +++ b/libcxx/include/__functional/binder1st.h @@ -23,28 +23,29 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder1st - : public __unary_function -{ + : public __unary_function { protected: - _Operation op; - typename _Operation::first_argument_type value; + _Operation op; + typename _Operation::first_argument_type value; + public: - _LIBCPP_HIDE_FROM_ABI binder1st(const _Operation& __x, - const typename _Operation::first_argument_type __y) - : op(__x), value(__y) {} - _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type operator() - (typename _Operation::second_argument_type& __x) const - {return op(value, __x);} - _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type operator() - (const typename _Operation::second_argument_type& __x) const - {return op(value, __x);} + _LIBCPP_HIDE_FROM_ABI binder1st(const _Operation& __x, const typename _Operation::first_argument_type __y) + : op(__x), value(__y) {} + _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type + operator()(typename _Operation::second_argument_type& __x) const { + return op(value, __x); + } + _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type + operator()(const typename _Operation::second_argument_type& __x) const { + return op(value, __x); + } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -binder1st<_Operation> -bind1st(const _Operation& __op, const _Tp& __x) - {return binder1st<_Operation>(__op, __x);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI binder1st<_Operation> +bind1st(const _Operation& __op, const _Tp& __x) { + return binder1st<_Operation>(__op, __x); +} #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) diff --git a/libcxx/include/__functional/binder2nd.h b/libcxx/include/__functional/binder2nd.h index cf80a55bc6a3d..9d22e4430b1b3 100644 --- a/libcxx/include/__functional/binder2nd.h +++ b/libcxx/include/__functional/binder2nd.h @@ -23,28 +23,29 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 binder2nd - : public __unary_function -{ + : public __unary_function { protected: - _Operation op; - typename _Operation::second_argument_type value; + _Operation op; + typename _Operation::second_argument_type value; + public: - _LIBCPP_HIDE_FROM_ABI - binder2nd(const _Operation& __x, const typename _Operation::second_argument_type __y) - : op(__x), value(__y) {} - _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type operator() - ( typename _Operation::first_argument_type& __x) const - {return op(__x, value);} - _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type operator() - (const typename _Operation::first_argument_type& __x) const - {return op(__x, value);} + _LIBCPP_HIDE_FROM_ABI binder2nd(const _Operation& __x, const typename _Operation::second_argument_type __y) + : op(__x), value(__y) {} + _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type + operator()(typename _Operation::first_argument_type& __x) const { + return op(__x, value); + } + _LIBCPP_HIDE_FROM_ABI typename _Operation::result_type + operator()(const typename _Operation::first_argument_type& __x) const { + return op(__x, value); + } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -binder2nd<_Operation> -bind2nd(const _Operation& __op, const _Tp& __x) - {return binder2nd<_Operation>(__op, __x);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI binder2nd<_Operation> +bind2nd(const _Operation& __op, const _Tp& __x) { + return binder2nd<_Operation>(__op, __x); +} #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) diff --git a/libcxx/include/__functional/boyer_moore_searcher.h b/libcxx/include/__functional/boyer_moore_searcher.h index 0165e062810fe..648b60c505219 100644 --- a/libcxx/include/__functional/boyer_moore_searcher.h +++ b/libcxx/include/__functional/boyer_moore_searcher.h @@ -29,39 +29,29 @@ #if _LIBCPP_STD_VER >= 17 _LIBCPP_PUSH_MACROS -#include <__undef_macros> +# include <__undef_macros> _LIBCPP_BEGIN_NAMESPACE_STD -template +template class _BMSkipTable; // General case for BM data searching; use a map -template +template class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, false> { private: using value_type = _Value; - using key_type = _Key; + using key_type = _Key; const value_type __default_value_; unordered_map<_Key, _Value, _Hash, _BinaryPredicate> __table_; public: - _LIBCPP_HIDE_FROM_ABI - explicit _BMSkipTable(size_t __sz, value_type __default_value, _Hash __hash, _BinaryPredicate __pred) - : __default_value_(__default_value), - __table_(__sz, __hash, __pred) {} + _LIBCPP_HIDE_FROM_ABI explicit _BMSkipTable( + size_t __sz, value_type __default_value, _Hash __hash, _BinaryPredicate __pred) + : __default_value_(__default_value), __table_(__sz, __hash, __pred) {} - _LIBCPP_HIDE_FROM_ABI void insert(const key_type& __key, value_type __val) { - __table_[__key] = __val; - } + _LIBCPP_HIDE_FROM_ABI void insert(const key_type& __key, value_type __val) { __table_[__key] = __val; } _LIBCPP_HIDE_FROM_ABI value_type operator[](const key_type& __key) const { auto __it = __table_.find(__key); @@ -70,14 +60,11 @@ class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, false> { }; // Special case small numeric values; use an array -template +template class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, true> { private: using value_type = _Value; - using key_type = _Key; + using key_type = _Key; using unsigned_key_type = make_unsigned_t; std::array __table_; @@ -98,34 +85,33 @@ class _BMSkipTable<_Key, _Value, _Hash, _BinaryPredicate, true> { }; template ::value_type>, + class _Hash = hash::value_type>, class _BinaryPredicate = equal_to<>> class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher { private: using difference_type = typename std::iterator_traits<_RandomAccessIterator1>::difference_type; - using value_type = typename std::iterator_traits<_RandomAccessIterator1>::value_type; - using __skip_table_type = _BMSkipTable - && sizeof(value_type) == 1 - && is_same_v<_Hash, hash> - && is_same_v<_BinaryPredicate, equal_to<>>>; + using value_type = typename std::iterator_traits<_RandomAccessIterator1>::value_type; + using __skip_table_type = + _BMSkipTable && sizeof(value_type) == 1 && is_same_v<_Hash, hash> && + is_same_v<_BinaryPredicate, equal_to<>>>; public: - _LIBCPP_HIDE_FROM_ABI - boyer_moore_searcher(_RandomAccessIterator1 __first, - _RandomAccessIterator1 __last, - _Hash __hash = _Hash(), - _BinaryPredicate __pred = _BinaryPredicate()) - : __first_(__first), - __last_(__last), - __pred_(__pred), - __pattern_length_(__last - __first), - __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, -1, __hash, __pred_)), - __suffix_(std::__allocate_shared_unbounded_array( - allocator(), __pattern_length_ + 1)) { + _LIBCPP_HIDE_FROM_ABI boyer_moore_searcher( + _RandomAccessIterator1 __first, + _RandomAccessIterator1 __last, + _Hash __hash = _Hash(), + _BinaryPredicate __pred = _BinaryPredicate()) + : __first_(__first), + __last_(__last), + __pred_(__pred), + __pattern_length_(__last - __first), + __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, -1, __hash, __pred_)), + __suffix_(std::__allocate_shared_unbounded_array( + allocator(), __pattern_length_ + 1)) { difference_type __i = 0; while (__first != __last) { __skip_table_->insert(*__first, __i); @@ -162,8 +148,8 @@ class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher { template _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2> __search(_RandomAccessIterator2 __f, _RandomAccessIterator2 __l) const { - _RandomAccessIterator2 __current = __f; - const _RandomAccessIterator2 __last = __l - __pattern_length_; + _RandomAccessIterator2 __current = __f; + const _RandomAccessIterator2 __last = __l - __pattern_length_; const __skip_table_type& __skip_table = *__skip_table_; while (__current <= __last) { @@ -190,7 +176,7 @@ class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher { const size_t __count = __last - __first; __prefix[0] = 0; - size_t __k = 0; + size_t __k = 0; for (size_t __i = 1; __i != __count; ++__i) { while (__k > 0 && !__pred(__first[__k], __first[__i])) @@ -219,7 +205,7 @@ class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher { __compute_bm_prefix(_ReverseIter(__last), _ReverseIter(__first), __pred, __scratch); for (size_t __i = 0; __i != __count; ++__i) { - const size_t __j = __count - __scratch[__i]; + const size_t __j = __count - __scratch[__i]; const difference_type __k = __i - __scratch[__i] + 1; if (__suffix_[__j] > __k) @@ -230,31 +216,31 @@ class _LIBCPP_TEMPLATE_VIS boyer_moore_searcher { _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(boyer_moore_searcher); template ::value_type>, + class _Hash = hash::value_type>, class _BinaryPredicate = equal_to<>> class _LIBCPP_TEMPLATE_VIS boyer_moore_horspool_searcher { private: using difference_type = typename iterator_traits<_RandomAccessIterator1>::difference_type; - using value_type = typename iterator_traits<_RandomAccessIterator1>::value_type; - using __skip_table_type = _BMSkipTable - && sizeof(value_type) == 1 - && is_same_v<_Hash, hash> - && is_same_v<_BinaryPredicate, equal_to<>>>; + using value_type = typename iterator_traits<_RandomAccessIterator1>::value_type; + using __skip_table_type = + _BMSkipTable && sizeof(value_type) == 1 && is_same_v<_Hash, hash> && + is_same_v<_BinaryPredicate, equal_to<>>>; + public: - _LIBCPP_HIDE_FROM_ABI - boyer_moore_horspool_searcher(_RandomAccessIterator1 __first, - _RandomAccessIterator1 __last, - _Hash __hash = _Hash(), - _BinaryPredicate __pred = _BinaryPredicate()) - : __first_(__first), - __last_(__last), - __pred_(__pred), - __pattern_length_(__last - __first), - __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, __pattern_length_, __hash, __pred_)) { + _LIBCPP_HIDE_FROM_ABI boyer_moore_horspool_searcher( + _RandomAccessIterator1 __first, + _RandomAccessIterator1 __last, + _Hash __hash = _Hash(), + _BinaryPredicate __pred = _BinaryPredicate()) + : __first_(__first), + __last_(__last), + __pred_(__pred), + __pattern_length_(__last - __first), + __skip_table_(std::make_shared<__skip_table_type>(__pattern_length_, __pattern_length_, __hash, __pred_)) { if (__first == __last) return; --__last; @@ -293,8 +279,8 @@ class _LIBCPP_TEMPLATE_VIS boyer_moore_horspool_searcher { template _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2> __search(_RandomAccessIterator2 __f, _RandomAccessIterator2 __l) const { - _RandomAccessIterator2 __current = __f; - const _RandomAccessIterator2 __last = __l - __pattern_length_; + _RandomAccessIterator2 __current = __f; + const _RandomAccessIterator2 __last = __l - __pattern_length_; const __skip_table_type& __skip_table = *__skip_table_; while (__current <= __last) { diff --git a/libcxx/include/__functional/compose.h b/libcxx/include/__functional/compose.h index af207e7cbafbf..4b86dd37cd48a 100644 --- a/libcxx/include/__functional/compose.h +++ b/libcxx/include/__functional/compose.h @@ -25,25 +25,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 struct __compose_op { - template - _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()(_Fn1&& __f1, _Fn2&& __f2, _Args&&... __args) const - noexcept(noexcept(std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...)))) - -> decltype( std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...))) - { return std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...)); } + template + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn1&& __f1, _Fn2&& __f2, _Args&&... __args) const noexcept(noexcept( + std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...)))) + -> decltype(std::invoke(std::forward<_Fn1>(__f1), + std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...))) { + return std::invoke(std::forward<_Fn1>(__f1), std::invoke(std::forward<_Fn2>(__f2), std::forward<_Args>(__args)...)); + } }; template struct __compose_t : __perfect_forward<__compose_op, _Fn1, _Fn2> { - using __perfect_forward<__compose_op, _Fn1, _Fn2>::__perfect_forward; + using __perfect_forward<__compose_op, _Fn1, _Fn2>::__perfect_forward; }; template -_LIBCPP_HIDE_FROM_ABI -constexpr auto __compose(_Fn1&& __f1, _Fn2&& __f2) - noexcept(noexcept(__compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2)))) - -> decltype( __compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2))) - { return __compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2)); } +_LIBCPP_HIDE_FROM_ABI constexpr auto __compose(_Fn1&& __f1, _Fn2&& __f2) noexcept( + noexcept(__compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2)))) + -> decltype(__compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2))) { + return __compose_t, decay_t<_Fn2>>(std::forward<_Fn1>(__f1), std::forward<_Fn2>(__f2)); +} #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__functional/default_searcher.h b/libcxx/include/__functional/default_searcher.h index 21b6de9c34c84..db89d10757c1b 100644 --- a/libcxx/include/__functional/default_searcher.h +++ b/libcxx/include/__functional/default_searcher.h @@ -26,27 +26,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 // default searcher -template> +template > class _LIBCPP_TEMPLATE_VIS default_searcher { public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - default_searcher(_ForwardIterator __f, _ForwardIterator __l, - _BinaryPredicate __p = _BinaryPredicate()) - : __first_(__f), __last_(__l), __pred_(__p) {} - - template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - pair<_ForwardIterator2, _ForwardIterator2> - operator () (_ForwardIterator2 __f, _ForwardIterator2 __l) const - { - auto __proj = __identity(); - return std::__search_impl(__f, __l, __first_, __last_, __pred_, __proj, __proj); - } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + default_searcher(_ForwardIterator __f, _ForwardIterator __l, _BinaryPredicate __p = _BinaryPredicate()) + : __first_(__f), __last_(__l), __pred_(__p) {} + + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_ForwardIterator2, _ForwardIterator2> + operator()(_ForwardIterator2 __f, _ForwardIterator2 __l) const { + auto __proj = __identity(); + return std::__search_impl(__f, __l, __first_, __last_, __pred_, __proj, __proj); + } private: - _ForwardIterator __first_; - _ForwardIterator __last_; - _BinaryPredicate __pred_; + _ForwardIterator __first_; + _ForwardIterator __last_; + _BinaryPredicate __pred_; }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(default_searcher); diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h index 7cc4cccc26cdc..6505bb5871739 100644 --- a/libcxx/include/__functional/function.h +++ b/libcxx/include/__functional/function.h @@ -53,86 +53,77 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables") -class _LIBCPP_EXPORTED_FROM_ABI bad_function_call - : public exception -{ +class _LIBCPP_EXPORTED_FROM_ABI bad_function_call : public exception { public: - _LIBCPP_HIDE_FROM_ABI bad_function_call() _NOEXCEPT = default; - _LIBCPP_HIDE_FROM_ABI bad_function_call(const bad_function_call&) _NOEXCEPT = default; - _LIBCPP_HIDE_FROM_ABI bad_function_call& operator=(const bad_function_call&) _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI bad_function_call() _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI bad_function_call(const bad_function_call&) _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI bad_function_call& operator=(const bad_function_call&) _NOEXCEPT = default; // Note that when a key function is not used, every translation unit that uses // bad_function_call will end up containing a weak definition of the vtable and // typeinfo. -#ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION - ~bad_function_call() _NOEXCEPT override; -#else - _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_function_call() _NOEXCEPT override {} -#endif - -#ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE - const char* what() const _NOEXCEPT override; -#endif +# ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION + ~bad_function_call() _NOEXCEPT override; +# else + _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_function_call() _NOEXCEPT override {} +# endif + +# ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE + const char* what() const _NOEXCEPT override; +# endif }; _LIBCPP_DIAGNOSTIC_POP -_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI -void __throw_bad_function_call() -{ -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - throw bad_function_call(); -#else - _LIBCPP_VERBOSE_ABORT("bad_function_call was thrown in -fno-exceptions mode"); -#endif +_LIBCPP_NORETURN inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_function_call() { +# ifndef _LIBCPP_HAS_NO_EXCEPTIONS + throw bad_function_call(); +# else + _LIBCPP_VERBOSE_ABORT("bad_function_call was thrown in -fno-exceptions mode"); +# endif } -template class _LIBCPP_TEMPLATE_VIS function; // undefined +template +class _LIBCPP_TEMPLATE_VIS function; // undefined -namespace __function -{ +namespace __function { -template -struct __maybe_derive_from_unary_function -{ -}; +template +struct __maybe_derive_from_unary_function {}; -template -struct __maybe_derive_from_unary_function<_Rp(_A1)> - : public __unary_function<_A1, _Rp> -{ -}; +template +struct __maybe_derive_from_unary_function<_Rp(_A1)> : public __unary_function<_A1, _Rp> {}; -template -struct __maybe_derive_from_binary_function -{ -}; +template +struct __maybe_derive_from_binary_function {}; -template -struct __maybe_derive_from_binary_function<_Rp(_A1, _A2)> - : public __binary_function<_A1, _A2, _Rp> -{ -}; +template +struct __maybe_derive_from_binary_function<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {}; template -_LIBCPP_HIDE_FROM_ABI -bool __not_null(_Fp const&) { return true; } +_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp const&) { + return true; +} template -_LIBCPP_HIDE_FROM_ABI -bool __not_null(_Fp* __ptr) { return __ptr; } +_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp* __ptr) { + return __ptr; +} template -_LIBCPP_HIDE_FROM_ABI -bool __not_null(_Ret _Class::*__ptr) { return __ptr; } +_LIBCPP_HIDE_FROM_ABI bool __not_null(_Ret _Class::*__ptr) { + return __ptr; +} template -_LIBCPP_HIDE_FROM_ABI -bool __not_null(function<_Fp> const& __f) { return !!__f; } +_LIBCPP_HIDE_FROM_ABI bool __not_null(function<_Fp> const& __f) { + return !!__f; +} -#ifdef _LIBCPP_HAS_EXTENSION_BLOCKS -template -_LIBCPP_HIDE_FROM_ABI -bool __not_null(_Rp (^__p)(_Args...)) { return __p; } -#endif +# ifdef _LIBCPP_HAS_EXTENSION_BLOCKS +template +_LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) { + return __p; +} +# endif } // namespace __function @@ -140,84 +131,60 @@ namespace __function { // __alloc_func holds a functor and an allocator. -template class __alloc_func; +template +class __alloc_func; template class __default_alloc_func; template -class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> -{ - __compressed_pair<_Fp, _Ap> __f_; - - public: - typedef _LIBCPP_NODEBUG _Fp _Target; - typedef _LIBCPP_NODEBUG _Ap _Alloc; - - _LIBCPP_HIDE_FROM_ABI - const _Target& __target() const { return __f_.first(); } - - // WIN32 APIs may define __allocator, so use __get_allocator instead. - _LIBCPP_HIDE_FROM_ABI - const _Alloc& __get_allocator() const { return __f_.second(); } - - _LIBCPP_HIDE_FROM_ABI - explicit __alloc_func(_Target&& __f) - : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), - std::forward_as_tuple()) - { - } +class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> { + __compressed_pair<_Fp, _Ap> __f_; - _LIBCPP_HIDE_FROM_ABI - explicit __alloc_func(const _Target& __f, const _Alloc& __a) - : __f_(piecewise_construct, std::forward_as_tuple(__f), - std::forward_as_tuple(__a)) - { - } +public: + typedef _LIBCPP_NODEBUG _Fp _Target; + typedef _LIBCPP_NODEBUG _Ap _Alloc; - _LIBCPP_HIDE_FROM_ABI - explicit __alloc_func(const _Target& __f, _Alloc&& __a) - : __f_(piecewise_construct, std::forward_as_tuple(__f), - std::forward_as_tuple(std::move(__a))) - { - } + _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_.first(); } - _LIBCPP_HIDE_FROM_ABI - explicit __alloc_func(_Target&& __f, _Alloc&& __a) - : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), - std::forward_as_tuple(std::move(__a))) - { - } + // WIN32 APIs may define __allocator, so use __get_allocator instead. + _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __f_.second(); } - _LIBCPP_HIDE_FROM_ABI - _Rp operator()(_ArgTypes&&... __arg) - { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(__f_.first(), - std::forward<_ArgTypes>(__arg)...); - } + _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f) + : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple()) {} - _LIBCPP_HIDE_FROM_ABI - __alloc_func* __clone() const - { - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA; - _AA __a(__f_.second()); - typedef __allocator_destructor<_AA> _Dp; - unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a)); - return __hold.release(); - } + _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a) + : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(__a)) {} - _LIBCPP_HIDE_FROM_ABI - void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); } + _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a) + : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(std::move(__a))) {} - _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) { - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __rebind_alloc<__alloc_traits, __alloc_func> _FunAlloc; - _FunAlloc __a(__f->__get_allocator()); - __f->destroy(); - __a.deallocate(__f, 1); - } + _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a) + : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple(std::move(__a))) {} + + _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(__f_.first(), std::forward<_ArgTypes>(__arg)...); + } + + _LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA; + _AA __a(__f_.second()); + typedef __allocator_destructor<_AA> _Dp; + unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a)); + return __hold.release(); + } + + _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); } + + _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __rebind_alloc<__alloc_traits, __alloc_func> _FunAlloc; + _FunAlloc __a(__f->__get_allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } }; template @@ -227,454 +194,377 @@ class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> { public: typedef _LIBCPP_NODEBUG _Fp _Target; - _LIBCPP_HIDE_FROM_ABI - const _Target& __target() const { return __f_; } + _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_; } - _LIBCPP_HIDE_FROM_ABI - explicit __default_alloc_func(_Target&& __f) : __f_(std::move(__f)) {} + _LIBCPP_HIDE_FROM_ABI explicit __default_alloc_func(_Target&& __f) : __f_(std::move(__f)) {} - _LIBCPP_HIDE_FROM_ABI - explicit __default_alloc_func(const _Target& __f) : __f_(__f) {} + _LIBCPP_HIDE_FROM_ABI explicit __default_alloc_func(const _Target& __f) : __f_(__f) {} - _LIBCPP_HIDE_FROM_ABI - _Rp operator()(_ArgTypes&&... __arg) { + _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) { typedef __invoke_void_return_wrapper<_Rp> _Invoker; return _Invoker::__call(__f_, std::forward<_ArgTypes>(__arg)...); } - _LIBCPP_HIDE_FROM_ABI - __default_alloc_func* __clone() const { - __builtin_new_allocator::__holder_t __hold = - __builtin_new_allocator::__allocate_type<__default_alloc_func>(1); - __default_alloc_func* __res = - ::new ((void*)__hold.get()) __default_alloc_func(__f_); + _LIBCPP_HIDE_FROM_ABI __default_alloc_func* __clone() const { + __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<__default_alloc_func>(1); + __default_alloc_func* __res = ::new ((void*)__hold.get()) __default_alloc_func(__f_); (void)__hold.release(); return __res; } - _LIBCPP_HIDE_FROM_ABI - void destroy() _NOEXCEPT { __f_.~_Target(); } + _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~_Target(); } _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__default_alloc_func* __f) { __f->destroy(); - __builtin_new_allocator::__deallocate_type<__default_alloc_func>(__f, 1); + __builtin_new_allocator::__deallocate_type<__default_alloc_func>(__f, 1); } }; // __base provides an abstract interface for copyable functors. -template class _LIBCPP_TEMPLATE_VIS __base; +template +class _LIBCPP_TEMPLATE_VIS __base; + +template +class __base<_Rp(_ArgTypes...)> { + __base(const __base&); + __base& operator=(const __base&); -template -class __base<_Rp(_ArgTypes...)> -{ - __base(const __base&); - __base& operator=(const __base&); public: - _LIBCPP_HIDE_FROM_ABI __base() {} - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual ~__base() {} - virtual __base* __clone() const = 0; - virtual void __clone(__base*) const = 0; - virtual void destroy() _NOEXCEPT = 0; - virtual void destroy_deallocate() _NOEXCEPT = 0; - virtual _Rp operator()(_ArgTypes&& ...) = 0; -#ifndef _LIBCPP_HAS_NO_RTTI - virtual const void* target(const type_info&) const _NOEXCEPT = 0; - virtual const std::type_info& target_type() const _NOEXCEPT = 0; -#endif // _LIBCPP_HAS_NO_RTTI + _LIBCPP_HIDE_FROM_ABI __base() {} + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual ~__base() {} + virtual __base* __clone() const = 0; + virtual void __clone(__base*) const = 0; + virtual void destroy() _NOEXCEPT = 0; + virtual void destroy_deallocate() _NOEXCEPT = 0; + virtual _Rp operator()(_ArgTypes&&...) = 0; +# ifndef _LIBCPP_HAS_NO_RTTI + virtual const void* target(const type_info&) const _NOEXCEPT = 0; + virtual const std::type_info& target_type() const _NOEXCEPT = 0; +# endif // _LIBCPP_HAS_NO_RTTI }; // __func implements __base for a given functor type. -template class __func; +template +class __func; + +template +class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> { + __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_; -template -class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> - : public __base<_Rp(_ArgTypes...)> -{ - __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_; public: - _LIBCPP_HIDE_FROM_ABI - explicit __func(_Fp&& __f) - : __f_(std::move(__f)) {} - - _LIBCPP_HIDE_FROM_ABI - explicit __func(const _Fp& __f, const _Alloc& __a) - : __f_(__f, __a) {} - - _LIBCPP_HIDE_FROM_ABI - explicit __func(const _Fp& __f, _Alloc&& __a) - : __f_(__f, std::move(__a)) {} - - _LIBCPP_HIDE_FROM_ABI - explicit __func(_Fp&& __f, _Alloc&& __a) - : __f_(std::move(__f), std::move(__a)) {} - - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const; - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __clone(__base<_Rp(_ArgTypes...)>*) const; - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg); -#ifndef _LIBCPP_HAS_NO_RTTI - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(const type_info&) const _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT; -#endif // _LIBCPP_HAS_NO_RTTI + _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f) : __f_(std::move(__f)) {} + + _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, const _Alloc& __a) : __f_(__f, __a) {} + + _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, _Alloc&& __a) : __f_(__f, std::move(__a)) {} + + _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f, _Alloc&& __a) : __f_(std::move(__f), std::move(__a)) {} + + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const; + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __clone(__base<_Rp(_ArgTypes...)>*) const; + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg); +# ifndef _LIBCPP_HAS_NO_RTTI + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(const type_info&) const _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT; +# endif // _LIBCPP_HAS_NO_RTTI }; -template -__base<_Rp(_ArgTypes...)>* -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const -{ - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __rebind_alloc<__alloc_traits, __func> _Ap; - _Ap __a(__f_.__get_allocator()); - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a)); - return __hold.release(); +template +__base<_Rp(_ArgTypes...)>* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __rebind_alloc<__alloc_traits, __func> _Ap; + _Ap __a(__f_.__get_allocator()); + typedef __allocator_destructor<_Ap> _Dp; + unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a)); + return __hold.release(); } -template -void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const -{ - ::new ((void*)__p) __func(__f_.__target(), __f_.__get_allocator()); +template +void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const { + ::new ((void*)__p) __func(__f_.__target(), __f_.__get_allocator()); } -template -void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT -{ - __f_.destroy(); +template +void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT { + __f_.destroy(); } -template -void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT -{ - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __rebind_alloc<__alloc_traits, __func> _Ap; - _Ap __a(__f_.__get_allocator()); - __f_.destroy(); - __a.deallocate(this, 1); +template +void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __rebind_alloc<__alloc_traits, __func> _Ap; + _Ap __a(__f_.__get_allocator()); + __f_.destroy(); + __a.deallocate(this, 1); } -template -_Rp -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) -{ - return __f_(std::forward<_ArgTypes>(__arg)...); +template +_Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&&... __arg) { + return __f_(std::forward<_ArgTypes>(__arg)...); } -#ifndef _LIBCPP_HAS_NO_RTTI +# ifndef _LIBCPP_HAS_NO_RTTI -template -const void* -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT -{ - if (__ti == typeid(_Fp)) - return std::addressof(__f_.__target()); - return nullptr; +template +const void* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT { + if (__ti == typeid(_Fp)) + return std::addressof(__f_.__target()); + return nullptr; } -template -const std::type_info& -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT -{ - return typeid(_Fp); +template +const std::type_info& __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { + return typeid(_Fp); } -#endif // _LIBCPP_HAS_NO_RTTI +# endif // _LIBCPP_HAS_NO_RTTI // __value_func creates a value-type from a __func. -template class __value_func; +template +class __value_func; -template class __value_func<_Rp(_ArgTypes...)> -{ - _LIBCPP_SUPPRESS_DEPRECATED_PUSH - typename aligned_storage<3 * sizeof(void*)>::type __buf_; - _LIBCPP_SUPPRESS_DEPRECATED_POP +template +class __value_func<_Rp(_ArgTypes...)> { + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + typename aligned_storage<3 * sizeof(void*)>::type __buf_; + _LIBCPP_SUPPRESS_DEPRECATED_POP - typedef __base<_Rp(_ArgTypes...)> __func; - __func* __f_; + typedef __base<_Rp(_ArgTypes...)> __func; + __func* __f_; - _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI static __func* __as_base(void* __p) - { - return reinterpret_cast<__func*>(__p); - } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI static __func* __as_base(void* __p) { return reinterpret_cast<__func*>(__p); } - public: - _LIBCPP_HIDE_FROM_ABI - __value_func() _NOEXCEPT : __f_(nullptr) {} - - template - _LIBCPP_HIDE_FROM_ABI __value_func(_Fp&& __f, const _Alloc& __a) - : __f_(nullptr) - { - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; - typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc; - - if (__function::__not_null(__f)) - { - _FunAlloc __af(__a); - if (sizeof(_Fun) <= sizeof(__buf_) && - is_nothrow_copy_constructible<_Fp>::value && - is_nothrow_copy_constructible<_FunAlloc>::value) - { - __f_ = - ::new ((void*)&__buf_) _Fun(std::move(__f), _Alloc(__af)); - } - else - { - typedef __allocator_destructor<_FunAlloc> _Dp; - unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); - ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__a)); - __f_ = __hold.release(); - } - } - } +public: + _LIBCPP_HIDE_FROM_ABI __value_func() _NOEXCEPT : __f_(nullptr) {} - template , __value_func>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) - : __value_func(std::forward<_Fp>(__f), allocator<_Fp>()) {} - - _LIBCPP_HIDE_FROM_ABI - __value_func(const __value_func& __f) - { - if (__f.__f_ == nullptr) - __f_ = nullptr; - else if ((void*)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); + template + _LIBCPP_HIDE_FROM_ABI __value_func(_Fp&& __f, const _Alloc& __a) : __f_(nullptr) { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc; + + if (__function::__not_null(__f)) { + _FunAlloc __af(__a); + if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value && + is_nothrow_copy_constructible<_FunAlloc>::value) { + __f_ = ::new ((void*)&__buf_) _Fun(std::move(__f), _Alloc(__af)); + } else { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__a)); + __f_ = __hold.release(); + } } + } - _LIBCPP_HIDE_FROM_ABI - __value_func(__value_func&& __f) _NOEXCEPT - { - if (__f.__f_ == nullptr) - __f_ = nullptr; - else if ((void*)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = nullptr; - } - } + template , __value_func>::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) : __value_func(std::forward<_Fp>(__f), allocator<_Fp>()) {} + + _LIBCPP_HIDE_FROM_ABI __value_func(const __value_func& __f) { + if (__f.__f_ == nullptr) + __f_ = nullptr; + else if ((void*)__f.__f_ == &__f.__buf_) { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } else + __f_ = __f.__f_->__clone(); + } - _LIBCPP_HIDE_FROM_ABI - ~__value_func() - { - if ((void*)__f_ == &__buf_) - __f_->destroy(); - else if (__f_) - __f_->destroy_deallocate(); + _LIBCPP_HIDE_FROM_ABI __value_func(__value_func&& __f) _NOEXCEPT { + if (__f.__f_ == nullptr) + __f_ = nullptr; + else if ((void*)__f.__f_ == &__f.__buf_) { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } else { + __f_ = __f.__f_; + __f.__f_ = nullptr; } + } - _LIBCPP_HIDE_FROM_ABI - __value_func& operator=(__value_func&& __f) - { - *this = nullptr; - if (__f.__f_ == nullptr) - __f_ = nullptr; - else if ((void*)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = nullptr; - } - return *this; - } + _LIBCPP_HIDE_FROM_ABI ~__value_func() { + if ((void*)__f_ == &__buf_) + __f_->destroy(); + else if (__f_) + __f_->destroy_deallocate(); + } - _LIBCPP_HIDE_FROM_ABI - __value_func& operator=(nullptr_t) - { - __func* __f = __f_; - __f_ = nullptr; - if ((void*)__f == &__buf_) - __f->destroy(); - else if (__f) - __f->destroy_deallocate(); - return *this; + _LIBCPP_HIDE_FROM_ABI __value_func& operator=(__value_func&& __f) { + *this = nullptr; + if (__f.__f_ == nullptr) + __f_ = nullptr; + else if ((void*)__f.__f_ == &__f.__buf_) { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } else { + __f_ = __f.__f_; + __f.__f_ = nullptr; } + return *this; + } - _LIBCPP_HIDE_FROM_ABI - _Rp operator()(_ArgTypes&&... __args) const - { - if (__f_ == nullptr) - __throw_bad_function_call(); - return (*__f_)(std::forward<_ArgTypes>(__args)...); - } + _LIBCPP_HIDE_FROM_ABI __value_func& operator=(nullptr_t) { + __func* __f = __f_; + __f_ = nullptr; + if ((void*)__f == &__buf_) + __f->destroy(); + else if (__f) + __f->destroy_deallocate(); + return *this; + } - _LIBCPP_HIDE_FROM_ABI - void swap(__value_func& __f) _NOEXCEPT - { - if (&__f == this) - return; - if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) - { - _LIBCPP_SUPPRESS_DEPRECATED_PUSH - typename aligned_storage::type __tempbuf; - _LIBCPP_SUPPRESS_DEPRECATED_POP - __func* __t = __as_base(&__tempbuf); - __f_->__clone(__t); - __f_->destroy(); - __f_ = nullptr; - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = nullptr; - __f_ = __as_base(&__buf_); - __t->__clone(__as_base(&__f.__buf_)); - __t->destroy(); - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void*)__f_ == &__buf_) - { - __f_->__clone(__as_base(&__f.__buf_)); - __f_->destroy(); - __f_ = __f.__f_; - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void*)__f.__f_ == &__f.__buf_) - { - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = __f_; - __f_ = __as_base(&__buf_); - } - else - std::swap(__f_, __f.__f_); - } + _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __args) const { + if (__f_ == nullptr) + __throw_bad_function_call(); + return (*__f_)(std::forward<_ArgTypes>(__args)...); + } - _LIBCPP_HIDE_FROM_ABI - explicit operator bool() const _NOEXCEPT { return __f_ != nullptr; } + _LIBCPP_HIDE_FROM_ABI void swap(__value_func& __f) _NOEXCEPT { + if (&__f == this) + return; + if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) { + _LIBCPP_SUPPRESS_DEPRECATED_PUSH + typename aligned_storage::type __tempbuf; + _LIBCPP_SUPPRESS_DEPRECATED_POP + __func* __t = __as_base(&__tempbuf); + __f_->__clone(__t); + __f_->destroy(); + __f_ = nullptr; + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = nullptr; + __f_ = __as_base(&__buf_); + __t->__clone(__as_base(&__f.__buf_)); + __t->destroy(); + __f.__f_ = __as_base(&__f.__buf_); + } else if ((void*)__f_ == &__buf_) { + __f_->__clone(__as_base(&__f.__buf_)); + __f_->destroy(); + __f_ = __f.__f_; + __f.__f_ = __as_base(&__f.__buf_); + } else if ((void*)__f.__f_ == &__f.__buf_) { + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = __f_; + __f_ = __as_base(&__buf_); + } else + std::swap(__f_, __f.__f_); + } -#ifndef _LIBCPP_HAS_NO_RTTI - _LIBCPP_HIDE_FROM_ABI - const std::type_info& target_type() const _NOEXCEPT - { - if (__f_ == nullptr) - return typeid(void); - return __f_->target_type(); - } + _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __f_ != nullptr; } - template - _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT - { - if (__f_ == nullptr) - return nullptr; - return (const _Tp*)__f_->target(typeid(_Tp)); - } -#endif // _LIBCPP_HAS_NO_RTTI +# ifndef _LIBCPP_HAS_NO_RTTI + _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT { + if (__f_ == nullptr) + return typeid(void); + return __f_->target_type(); + } + + template + _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT { + if (__f_ == nullptr) + return nullptr; + return (const _Tp*)__f_->target(typeid(_Tp)); + } +# endif // _LIBCPP_HAS_NO_RTTI }; // Storage for a functor object, to be used with __policy to manage copy and // destruction. -union __policy_storage -{ - mutable char __small[sizeof(void*) * 2]; - void* __large; +union __policy_storage { + mutable char __small[sizeof(void*) * 2]; + void* __large; }; // True if _Fun can safely be held in __policy_storage.__small. template struct __use_small_storage : public integral_constant< - bool, sizeof(_Fun) <= sizeof(__policy_storage) && - _LIBCPP_ALIGNOF(_Fun) <= _LIBCPP_ALIGNOF(__policy_storage) && - is_trivially_copy_constructible<_Fun>::value && - is_trivially_destructible<_Fun>::value> {}; + bool, + sizeof(_Fun) <= sizeof(__policy_storage)&& _LIBCPP_ALIGNOF(_Fun) <= _LIBCPP_ALIGNOF(__policy_storage) && + is_trivially_copy_constructible<_Fun>::value && is_trivially_destructible<_Fun>::value> {}; // Policy contains information about how to copy, destroy, and move the // underlying functor. You can think of it as a vtable of sorts. -struct __policy -{ - // Used to copy or destroy __large values. null for trivial objects. - void* (*const __clone)(const void*); - void (*const __destroy)(void*); - - // True if this is the null policy (no value). - const bool __is_null; - - // The target type. May be null if RTTI is disabled. - const std::type_info* const __type_info; - - // Returns a pointer to a static policy object suitable for the functor - // type. - template - _LIBCPP_HIDE_FROM_ABI static const __policy* __create() - { - return __choose_policy<_Fun>(__use_small_storage<_Fun>()); - } +struct __policy { + // Used to copy or destroy __large values. null for trivial objects. + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + const bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* const __type_info; + + // Returns a pointer to a static policy object suitable for the functor + // type. + template + _LIBCPP_HIDE_FROM_ABI static const __policy* __create() { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } - _LIBCPP_HIDE_FROM_ABI - static const __policy* __create_empty() - { - static const _LIBCPP_CONSTEXPR __policy __policy = {nullptr, nullptr, - true, -#ifndef _LIBCPP_HAS_NO_RTTI - &typeid(void) -#else - nullptr -#endif - }; - return &__policy; - } + _LIBCPP_HIDE_FROM_ABI static const __policy* __create_empty() { + static const _LIBCPP_CONSTEXPR __policy __policy = { + nullptr, + nullptr, + true, +# ifndef _LIBCPP_HAS_NO_RTTI + &typeid(void) +# else + nullptr +# endif + }; + return &__policy; + } - private: - template - _LIBCPP_HIDE_FROM_ABI static void* __large_clone(const void* __s) - { - const _Fun* __f = static_cast(__s); - return __f->__clone(); - } +private: + template + _LIBCPP_HIDE_FROM_ABI static void* __large_clone(const void* __s) { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + } - template - _LIBCPP_HIDE_FROM_ABI static void __large_destroy(void* __s) { - _Fun::__destroy_and_delete(static_cast<_Fun*>(__s)); - } + template + _LIBCPP_HIDE_FROM_ABI static void __large_destroy(void* __s) { + _Fun::__destroy_and_delete(static_cast<_Fun*>(__s)); + } - template - _LIBCPP_HIDE_FROM_ABI static const __policy* - __choose_policy(/* is_small = */ false_type) { - static const _LIBCPP_CONSTEXPR __policy __policy = { - &__large_clone<_Fun>, &__large_destroy<_Fun>, false, -#ifndef _LIBCPP_HAS_NO_RTTI - &typeid(typename _Fun::_Target) -#else - nullptr -#endif - }; - return &__policy; - } + template + _LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ false_type) { + static const _LIBCPP_CONSTEXPR __policy __policy = { + &__large_clone<_Fun>, + &__large_destroy<_Fun>, + false, +# ifndef _LIBCPP_HAS_NO_RTTI + &typeid(typename _Fun::_Target) +# else + nullptr +# endif + }; + return &__policy; + } - template - _LIBCPP_HIDE_FROM_ABI static const __policy* - __choose_policy(/* is_small = */ true_type) - { - static const _LIBCPP_CONSTEXPR __policy __policy = { - nullptr, nullptr, false, -#ifndef _LIBCPP_HAS_NO_RTTI - &typeid(typename _Fun::_Target) -#else - nullptr -#endif - }; - return &__policy; - } + template + _LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ true_type) { + static const _LIBCPP_CONSTEXPR __policy __policy = { + nullptr, + nullptr, + false, +# ifndef _LIBCPP_HAS_NO_RTTI + &typeid(typename _Fun::_Target) +# else + nullptr +# endif + }; + return &__policy; + } }; // Used to choose between perfect forwarding or pass-by-value. Pass-by-value is @@ -684,548 +574,459 @@ using __fast_forward = __conditional_t::value, _Tp, _Tp&&>; // __policy_invoker calls an instance of __alloc_func held in __policy_storage. -template struct __policy_invoker; +template +struct __policy_invoker; template -struct __policy_invoker<_Rp(_ArgTypes...)> -{ - typedef _Rp (*__Call)(const __policy_storage*, - __fast_forward<_ArgTypes>...); - - __Call __call_; - - // Creates an invoker that throws bad_function_call. - _LIBCPP_HIDE_FROM_ABI - __policy_invoker() : __call_(&__call_empty) {} - - // Creates an invoker that calls the given instance of __func. - template - _LIBCPP_HIDE_FROM_ABI static __policy_invoker __create() - { - return __policy_invoker(&__call_impl<_Fun>); - } +struct __policy_invoker<_Rp(_ArgTypes...)> { + typedef _Rp (*__Call)(const __policy_storage*, __fast_forward<_ArgTypes>...); - private: - _LIBCPP_HIDE_FROM_ABI - explicit __policy_invoker(__Call __c) : __call_(__c) {} + __Call __call_; - _LIBCPP_HIDE_FROM_ABI static _Rp __call_empty(const __policy_storage*, - __fast_forward<_ArgTypes>...) - { - __throw_bad_function_call(); - } + // Creates an invoker that throws bad_function_call. + _LIBCPP_HIDE_FROM_ABI __policy_invoker() : __call_(&__call_empty) {} - template - _LIBCPP_HIDE_FROM_ABI static _Rp __call_impl(const __policy_storage* __buf, - __fast_forward<_ArgTypes>... __args) - { - _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value - ? &__buf->__small - : __buf->__large); - return (*__f)(std::forward<_ArgTypes>(__args)...); - } + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_HIDE_FROM_ABI static __policy_invoker __create() { + return __policy_invoker(&__call_impl<_Fun>); + } + +private: + _LIBCPP_HIDE_FROM_ABI explicit __policy_invoker(__Call __c) : __call_(__c) {} + + _LIBCPP_HIDE_FROM_ABI static _Rp __call_empty(const __policy_storage*, __fast_forward<_ArgTypes>...) { + __throw_bad_function_call(); + } + + template + _LIBCPP_HIDE_FROM_ABI static _Rp __call_impl(const __policy_storage* __buf, __fast_forward<_ArgTypes>... __args) { + _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value ? &__buf->__small : __buf->__large); + return (*__f)(std::forward<_ArgTypes>(__args)...); + } }; // __policy_func uses a __policy and __policy_invoker to create a type-erased, // copyable functor. -template class __policy_func; - -template class __policy_func<_Rp(_ArgTypes...)> -{ - // Inline storage for small objects. - __policy_storage __buf_; - - // Calls the value stored in __buf_. This could technically be part of - // policy, but storing it here eliminates a level of indirection inside - // operator(). - typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; - __invoker __invoker_; - - // The policy that describes how to move / copy / destroy __buf_. Never - // null, even if the function is empty. - const __policy* __policy_; - - public: - _LIBCPP_HIDE_FROM_ABI - __policy_func() : __policy_(__policy::__create_empty()) {} - - template - _LIBCPP_HIDE_FROM_ABI __policy_func(_Fp&& __f, const _Alloc& __a) - : __policy_(__policy::__create_empty()) - { - typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; - typedef allocator_traits<_Alloc> __alloc_traits; - typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc; - - if (__function::__not_null(__f)) - { - __invoker_ = __invoker::template __create<_Fun>(); - __policy_ = __policy::__create<_Fun>(); - - _FunAlloc __af(__a); - if (__use_small_storage<_Fun>()) - { - ::new ((void*)&__buf_.__small) - _Fun(std::move(__f), _Alloc(__af)); - } - else - { - typedef __allocator_destructor<_FunAlloc> _Dp; - unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); - ::new ((void*)__hold.get()) - _Fun(std::move(__f), _Alloc(__af)); - __buf_.__large = __hold.release(); - } - } - } +template +class __policy_func; + +template +class __policy_func<_Rp(_ArgTypes...)> { + // Inline storage for small objects. + __policy_storage __buf_; + + // Calls the value stored in __buf_. This could technically be part of + // policy, but storing it here eliminates a level of indirection inside + // operator(). + typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; - template , __policy_func>::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI explicit __policy_func(_Fp&& __f) - : __policy_(__policy::__create_empty()) { - typedef __default_alloc_func<_Fp, _Rp(_ArgTypes...)> _Fun; - - if (__function::__not_null(__f)) { - __invoker_ = __invoker::template __create<_Fun>(); - __policy_ = __policy::__create<_Fun>(); - if (__use_small_storage<_Fun>()) { - ::new ((void*)&__buf_.__small) _Fun(std::move(__f)); - } else { - __builtin_new_allocator::__holder_t __hold = - __builtin_new_allocator::__allocate_type<_Fun>(1); - __buf_.__large = ::new ((void*)__hold.get()) _Fun(std::move(__f)); - (void)__hold.release(); - } + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + +public: + _LIBCPP_HIDE_FROM_ABI __policy_func() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_HIDE_FROM_ABI __policy_func(_Fp&& __f, const _Alloc& __a) : __policy_(__policy::__create_empty()) { + typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc; + + if (__function::__not_null(__f)) { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) { + ::new ((void*)&__buf_.__small) _Fun(std::move(__f), _Alloc(__af)); + } else { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); } } + } - _LIBCPP_HIDE_FROM_ABI - __policy_func(const __policy_func& __f) - : __buf_(__f.__buf_), __invoker_(__f.__invoker_), - __policy_(__f.__policy_) - { - if (__policy_->__clone) - __buf_.__large = __policy_->__clone(__f.__buf_.__large); + template , __policy_func>::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI explicit __policy_func(_Fp&& __f) : __policy_(__policy::__create_empty()) { + typedef __default_alloc_func<_Fp, _Rp(_ArgTypes...)> _Fun; + + if (__function::__not_null(__f)) { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + if (__use_small_storage<_Fun>()) { + ::new ((void*)&__buf_.__small) _Fun(std::move(__f)); + } else { + __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<_Fun>(1); + __buf_.__large = ::new ((void*)__hold.get()) _Fun(std::move(__f)); + (void)__hold.release(); + } } + } - _LIBCPP_HIDE_FROM_ABI - __policy_func(__policy_func&& __f) - : __buf_(__f.__buf_), __invoker_(__f.__invoker_), - __policy_(__f.__policy_) - { - if (__policy_->__destroy) - { - __f.__policy_ = __policy::__create_empty(); - __f.__invoker_ = __invoker(); - } - } + _LIBCPP_HIDE_FROM_ABI __policy_func(const __policy_func& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), __policy_(__f.__policy_) { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } - _LIBCPP_HIDE_FROM_ABI - ~__policy_func() - { - if (__policy_->__destroy) - __policy_->__destroy(__buf_.__large); + _LIBCPP_HIDE_FROM_ABI __policy_func(__policy_func&& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), __policy_(__f.__policy_) { + if (__policy_->__destroy) { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); } + } - _LIBCPP_HIDE_FROM_ABI - __policy_func& operator=(__policy_func&& __f) - { - *this = nullptr; - __buf_ = __f.__buf_; - __invoker_ = __f.__invoker_; - __policy_ = __f.__policy_; - __f.__policy_ = __policy::__create_empty(); - __f.__invoker_ = __invoker(); - return *this; - } + _LIBCPP_HIDE_FROM_ABI ~__policy_func() { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } - _LIBCPP_HIDE_FROM_ABI - __policy_func& operator=(nullptr_t) - { - const __policy* __p = __policy_; - __policy_ = __policy::__create_empty(); - __invoker_ = __invoker(); - if (__p->__destroy) - __p->__destroy(__buf_.__large); - return *this; - } + _LIBCPP_HIDE_FROM_ABI __policy_func& operator=(__policy_func&& __f) { + *this = nullptr; + __buf_ = __f.__buf_; + __invoker_ = __f.__invoker_; + __policy_ = __f.__policy_; + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + return *this; + } - _LIBCPP_HIDE_FROM_ABI - _Rp operator()(_ArgTypes&&... __args) const - { - return __invoker_.__call_(std::addressof(__buf_), - std::forward<_ArgTypes>(__args)...); - } + _LIBCPP_HIDE_FROM_ABI __policy_func& operator=(nullptr_t) { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + return *this; + } - _LIBCPP_HIDE_FROM_ABI - void swap(__policy_func& __f) - { - std::swap(__invoker_, __f.__invoker_); - std::swap(__policy_, __f.__policy_); - std::swap(__buf_, __f.__buf_); - } + _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __args) const { + return __invoker_.__call_(std::addressof(__buf_), std::forward<_ArgTypes>(__args)...); + } - _LIBCPP_HIDE_FROM_ABI - explicit operator bool() const _NOEXCEPT - { - return !__policy_->__is_null; - } + _LIBCPP_HIDE_FROM_ABI void swap(__policy_func& __f) { + std::swap(__invoker_, __f.__invoker_); + std::swap(__policy_, __f.__policy_); + std::swap(__buf_, __f.__buf_); + } -#ifndef _LIBCPP_HAS_NO_RTTI - _LIBCPP_HIDE_FROM_ABI - const std::type_info& target_type() const _NOEXCEPT - { - return *__policy_->__type_info; - } + _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return !__policy_->__is_null; } - template - _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT - { - if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) - return nullptr; - if (__policy_->__clone) // Out of line storage. - return reinterpret_cast(__buf_.__large); - else - return reinterpret_cast(&__buf_.__small); - } -#endif // _LIBCPP_HAS_NO_RTTI +# ifndef _LIBCPP_HAS_NO_RTTI + _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT { return *__policy_->__type_info; } + + template + _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) + return nullptr; + if (__policy_->__clone) // Out of line storage. + return reinterpret_cast(__buf_.__large); + else + return reinterpret_cast(&__buf_.__small); + } +# endif // _LIBCPP_HAS_NO_RTTI }; -#if defined(_LIBCPP_HAS_BLOCKS_RUNTIME) +# if defined(_LIBCPP_HAS_BLOCKS_RUNTIME) -extern "C" void *_Block_copy(const void *); -extern "C" void _Block_release(const void *); +extern "C" void* _Block_copy(const void*); +extern "C" void _Block_release(const void*); -template -class __func<_Rp1(^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> - : public __base<_Rp(_ArgTypes...)> -{ - typedef _Rp1(^__block_type)(_ArgTypes1...); - __block_type __f_; +template +class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> { + typedef _Rp1 (^__block_type)(_ArgTypes1...); + __block_type __f_; public: - _LIBCPP_HIDE_FROM_ABI - explicit __func(__block_type const& __f) -#ifdef _LIBCPP_HAS_OBJC_ARC - : __f_(__f) -#else - : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr)) -#endif - { } + _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type const& __f) +# ifdef _LIBCPP_HAS_OBJC_ARC + : __f_(__f) +# else + : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr)) +# endif + { + } - // [TODO] add && to save on a retain + // [TODO] add && to save on a retain - _LIBCPP_HIDE_FROM_ABI - explicit __func(__block_type __f, const _Alloc& /* unused */) -#ifdef _LIBCPP_HAS_OBJC_ARC - : __f_(__f) -#else - : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr)) -#endif - { } - - virtual __base<_Rp(_ArgTypes...)>* __clone() const { - _LIBCPP_ASSERT_INTERNAL(false, - "Block pointers are just pointers, so they should always fit into " - "std::function's small buffer optimization. This function should " - "never be invoked."); - return nullptr; - } + _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type __f, const _Alloc& /* unused */) +# ifdef _LIBCPP_HAS_OBJC_ARC + : __f_(__f) +# else + : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr)) +# endif + { + } - virtual void __clone(__base<_Rp(_ArgTypes...)>* __p) const { - ::new ((void*)__p) __func(__f_); - } + virtual __base<_Rp(_ArgTypes...)>* __clone() const { + _LIBCPP_ASSERT_INTERNAL( + false, + "Block pointers are just pointers, so they should always fit into " + "std::function's small buffer optimization. This function should " + "never be invoked."); + return nullptr; + } - virtual void destroy() _NOEXCEPT { -#ifndef _LIBCPP_HAS_OBJC_ARC - if (__f_) - _Block_release(__f_); -#endif - __f_ = 0; - } + virtual void __clone(__base<_Rp(_ArgTypes...)>* __p) const { ::new ((void*)__p) __func(__f_); } - virtual void destroy_deallocate() _NOEXCEPT { - _LIBCPP_ASSERT_INTERNAL(false, - "Block pointers are just pointers, so they should always fit into " - "std::function's small buffer optimization. This function should " - "never be invoked."); - } + virtual void destroy() _NOEXCEPT { +# ifndef _LIBCPP_HAS_OBJC_ARC + if (__f_) + _Block_release(__f_); +# endif + __f_ = 0; + } - virtual _Rp operator()(_ArgTypes&& ... __arg) { - return std::__invoke(__f_, std::forward<_ArgTypes>(__arg)...); - } + virtual void destroy_deallocate() _NOEXCEPT { + _LIBCPP_ASSERT_INTERNAL( + false, + "Block pointers are just pointers, so they should always fit into " + "std::function's small buffer optimization. This function should " + "never be invoked."); + } -#ifndef _LIBCPP_HAS_NO_RTTI - virtual const void* target(type_info const& __ti) const _NOEXCEPT { - if (__ti == typeid(__func::__block_type)) - return &__f_; - return (const void*)nullptr; - } + virtual _Rp operator()(_ArgTypes&&... __arg) { return std::__invoke(__f_, std::forward<_ArgTypes>(__arg)...); } - virtual const std::type_info& target_type() const _NOEXCEPT { - return typeid(__func::__block_type); - } -#endif // _LIBCPP_HAS_NO_RTTI +# ifndef _LIBCPP_HAS_NO_RTTI + virtual const void* target(type_info const& __ti) const _NOEXCEPT { + if (__ti == typeid(__func::__block_type)) + return &__f_; + return (const void*)nullptr; + } + + virtual const std::type_info& target_type() const _NOEXCEPT { return typeid(__func::__block_type); } +# endif // _LIBCPP_HAS_NO_RTTI }; -#endif // _LIBCPP_HAS_EXTENSION_BLOCKS +# endif // _LIBCPP_HAS_EXTENSION_BLOCKS } // namespace __function -template +template class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)> : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, - public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> -{ -#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION - typedef __function::__value_func<_Rp(_ArgTypes...)> __func; -#else - typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; -#endif - - __func __f_; - - template , function>, - __invokable<_Fp, _ArgTypes...> - >::value> - struct __callable; - template - struct __callable<_Fp, true> - { - static const bool value = is_void<_Rp>::value || - __is_core_convertible::type, - _Rp>::value; - }; - template - struct __callable<_Fp, false> - { - static const bool value = false; - }; + public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { +# ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION + typedef __function::__value_func<_Rp(_ArgTypes...)> __func; +# else + typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; +# endif + + __func __f_; + + template , function>, __invokable<_Fp, _ArgTypes...> >::value> + struct __callable; + template + struct __callable<_Fp, true> { + static const bool value = + is_void<_Rp>::value || __is_core_convertible::type, _Rp>::value; + }; + template + struct __callable<_Fp, false> { + static const bool value = false; + }; template using _EnableIfLValueCallable = __enable_if_t<__callable<_Fp&>::value>; -public: - typedef _Rp result_type; - - // construct/copy/destroy: - _LIBCPP_HIDE_FROM_ABI - function() _NOEXCEPT { } - _LIBCPP_HIDE_FROM_ABI - _LIBCPP_HIDE_FROM_ABI function(nullptr_t) _NOEXCEPT {} - _LIBCPP_HIDE_FROM_ABI function(const function&); - _LIBCPP_HIDE_FROM_ABI function(function&&) _NOEXCEPT; - template> - _LIBCPP_HIDE_FROM_ABI function(_Fp); - -#if _LIBCPP_STD_VER <= 14 - template - _LIBCPP_HIDE_FROM_ABI - function(allocator_arg_t, const _Alloc&) _NOEXCEPT {} - template - _LIBCPP_HIDE_FROM_ABI - function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {} - template - _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, const function&); - template - _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, function&&); - template> - _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc& __a, _Fp __f); -#endif - _LIBCPP_HIDE_FROM_ABI function& operator=(const function&); - _LIBCPP_HIDE_FROM_ABI function& operator=(function&&) _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI function& operator=(nullptr_t) _NOEXCEPT; - template>> - _LIBCPP_HIDE_FROM_ABI function& operator=(_Fp&&); - - _LIBCPP_HIDE_FROM_ABI ~function(); - - // function modifiers: - _LIBCPP_HIDE_FROM_ABI void swap(function&) _NOEXCEPT; +public: + typedef _Rp result_type; + + // construct/copy/destroy: + _LIBCPP_HIDE_FROM_ABI function() _NOEXCEPT {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI function(nullptr_t) _NOEXCEPT {} + _LIBCPP_HIDE_FROM_ABI function(const function&); + _LIBCPP_HIDE_FROM_ABI function(function&&) _NOEXCEPT; + template > + _LIBCPP_HIDE_FROM_ABI function(_Fp); + +# if _LIBCPP_STD_VER <= 14 + template + _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&) _NOEXCEPT {} + template + _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {} + template + _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, const function&); + template + _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc&, function&&); + template > + _LIBCPP_HIDE_FROM_ABI function(allocator_arg_t, const _Alloc& __a, _Fp __f); +# endif + + _LIBCPP_HIDE_FROM_ABI function& operator=(const function&); + _LIBCPP_HIDE_FROM_ABI function& operator=(function&&) _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI function& operator=(nullptr_t) _NOEXCEPT; + template >> + _LIBCPP_HIDE_FROM_ABI function& operator=(_Fp&&); + + _LIBCPP_HIDE_FROM_ABI ~function(); + + // function modifiers: + _LIBCPP_HIDE_FROM_ABI void swap(function&) _NOEXCEPT; + +# if _LIBCPP_STD_VER <= 14 + template + _LIBCPP_HIDE_FROM_ABI void assign(_Fp&& __f, const _Alloc& __a) { + function(allocator_arg, __a, std::forward<_Fp>(__f)).swap(*this); + } +# endif -#if _LIBCPP_STD_VER <= 14 - template - _LIBCPP_HIDE_FROM_ABI - void assign(_Fp&& __f, const _Alloc& __a) - {function(allocator_arg, __a, std::forward<_Fp>(__f)).swap(*this);} -#endif + // function capacity: + _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return static_cast(__f_); } - // function capacity: - _LIBCPP_HIDE_FROM_ABI - explicit operator bool() const _NOEXCEPT { - return static_cast(__f_); - } + // deleted overloads close possible hole in the type system + template + bool operator==(const function<_R2(_ArgTypes2...)>&) const = delete; +# if _LIBCPP_STD_VER <= 17 + template + bool operator!=(const function<_R2(_ArgTypes2...)>&) const = delete; +# endif - // deleted overloads close possible hole in the type system - template - bool operator==(const function<_R2(_ArgTypes2...)>&) const = delete; -#if _LIBCPP_STD_VER <= 17 - template - bool operator!=(const function<_R2(_ArgTypes2...)>&) const = delete; -#endif public: - // function invocation: - _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes...) const; - -#ifndef _LIBCPP_HAS_NO_RTTI - // function target access: - _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT; - template - _LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT; - template - _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT; -#endif // _LIBCPP_HAS_NO_RTTI + // function invocation: + _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes...) const; + +# ifndef _LIBCPP_HAS_NO_RTTI + // function target access: + _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT; + template + _LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT; + template + _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT; +# endif // _LIBCPP_HAS_NO_RTTI }; -#if _LIBCPP_STD_VER >= 17 -template -function(_Rp(*)(_Ap...)) -> function<_Rp(_Ap...)>; +# if _LIBCPP_STD_VER >= 17 +template +function(_Rp (*)(_Ap...)) -> function<_Rp(_Ap...)>; -template::type> +template ::type> function(_Fp) -> function<_Stripped>; -#endif // _LIBCPP_STD_VER >= 17 +# endif // _LIBCPP_STD_VER >= 17 -template +template function<_Rp(_ArgTypes...)>::function(const function& __f) : __f_(__f.__f_) {} -#if _LIBCPP_STD_VER <= 14 -template +# if _LIBCPP_STD_VER <= 14 +template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - const function& __f) : __f_(__f.__f_) {} -#endif +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, const function& __f) : __f_(__f.__f_) {} +# endif template -function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT - : __f_(std::move(__f.__f_)) {} +function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT : __f_(std::move(__f.__f_)) {} -#if _LIBCPP_STD_VER <= 14 -template +# if _LIBCPP_STD_VER <= 14 +template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - function&& __f) - : __f_(std::move(__f.__f_)) {} -#endif +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, function&& __f) : __f_(std::move(__f.__f_)) {} +# endif template template function<_Rp(_ArgTypes...)>::function(_Fp __f) : __f_(std::move(__f)) {} -#if _LIBCPP_STD_VER <= 14 +# if _LIBCPP_STD_VER <= 14 template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, - _Fp __f) - : __f_(std::move(__f), __a) {} -#endif +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, _Fp __f) : __f_(std::move(__f), __a) {} +# endif -template -function<_Rp(_ArgTypes...)>& -function<_Rp(_ArgTypes...)>::operator=(const function& __f) -{ - function(__f).swap(*this); - return *this; +template +function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(const function& __f) { + function(__f).swap(*this); + return *this; } -template -function<_Rp(_ArgTypes...)>& -function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT -{ - __f_ = std::move(__f.__f_); - return *this; +template +function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT { + __f_ = std::move(__f.__f_); + return *this; } -template -function<_Rp(_ArgTypes...)>& -function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT -{ - __f_ = nullptr; - return *this; +template +function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT { + __f_ = nullptr; + return *this; } -template +template template -function<_Rp(_ArgTypes...)>& -function<_Rp(_ArgTypes...)>::operator=(_Fp&& __f) -{ - function(std::forward<_Fp>(__f)).swap(*this); - return *this; +function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(_Fp&& __f) { + function(std::forward<_Fp>(__f)).swap(*this); + return *this; } -template +template function<_Rp(_ArgTypes...)>::~function() {} -template -void -function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT -{ - __f_.swap(__f.__f_); +template +void function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT { + __f_.swap(__f.__f_); } -template -_Rp -function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const -{ - return __f_(std::forward<_ArgTypes>(__arg)...); +template +_Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const { + return __f_(std::forward<_ArgTypes>(__arg)...); } -#ifndef _LIBCPP_HAS_NO_RTTI +# ifndef _LIBCPP_HAS_NO_RTTI -template -const std::type_info& -function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT -{ - return __f_.target_type(); +template +const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { + return __f_.target_type(); } -template +template template -_Tp* -function<_Rp(_ArgTypes...)>::target() _NOEXCEPT -{ - return (_Tp*)(__f_.template target<_Tp>()); +_Tp* function<_Rp(_ArgTypes...)>::target() _NOEXCEPT { + return (_Tp*)(__f_.template target<_Tp>()); } -template +template template -const _Tp* -function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT -{ - return __f_.template target<_Tp>(); +const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT { + return __f_.template target<_Tp>(); } -#endif // _LIBCPP_HAS_NO_RTTI +# endif // _LIBCPP_HAS_NO_RTTI template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator==(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT {return !__f;} +inline _LIBCPP_HIDE_FROM_ABI bool operator==(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT { + return !__f; +} -#if _LIBCPP_STD_VER <= 17 +# if _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator==(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT {return !__f;} +inline _LIBCPP_HIDE_FROM_ABI bool operator==(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT { + return !__f; +} template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator!=(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT {return (bool)__f;} +inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT { + return (bool)__f; +} template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator!=(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT {return (bool)__f;} +inline _LIBCPP_HIDE_FROM_ABI bool operator!=(nullptr_t, const function<_Rp(_ArgTypes...)>& __f) _NOEXCEPT { + return (bool)__f; +} -#endif // _LIBCPP_STD_VER <= 17 +# endif // _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI -void -swap(function<_Rp(_ArgTypes...)>& __x, function<_Rp(_ArgTypes...)>& __y) _NOEXCEPT -{return __x.swap(__y);} +inline _LIBCPP_HIDE_FROM_ABI void swap(function<_Rp(_ArgTypes...)>& __x, function<_Rp(_ArgTypes...)>& __y) _NOEXCEPT { + return __x.swap(__y); +} _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__functional/hash.h b/libcxx/include/__functional/hash.h index c369e39ce003f..ff22055d6915e 100644 --- a/libcxx/include/__functional/hash.h +++ b/libcxx/include/__functional/hash.h @@ -35,133 +35,117 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI -_Size -__loadword(const void* __p) -{ - _Size __r; - std::memcpy(&__r, __p, sizeof(__r)); - return __r; +inline _LIBCPP_HIDE_FROM_ABI _Size __loadword(const void* __p) { + _Size __r; + std::memcpy(&__r, __p, sizeof(__r)); + return __r; } // We use murmur2 when size_t is 32 bits, and cityhash64 when size_t // is 64 bits. This is because cityhash64 uses 64bit x 64bit // multiplication, which can be very slow on 32-bit systems. -template +template struct __murmur2_or_cityhash; template -struct __murmur2_or_cityhash<_Size, 32> -{ - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - _Size operator()(const void* __key, _Size __len) const { - // murmur2 - const _Size __m = 0x5bd1e995; - const _Size __r = 24; - _Size __h = __len; - const unsigned char* __data = static_cast(__key); - for (; __len >= 4; __data += 4, __len -= 4) - { - _Size __k = std::__loadword<_Size>(__data); - __k *= __m; - __k ^= __k >> __r; - __k *= __m; - __h *= __m; - __h ^= __k; - } - switch (__len) - { - case 3: - __h ^= static_cast<_Size>(__data[2] << 16); - _LIBCPP_FALLTHROUGH(); - case 2: - __h ^= static_cast<_Size>(__data[1] << 8); - _LIBCPP_FALLTHROUGH(); - case 1: - __h ^= __data[0]; - __h *= __m; - } - __h ^= __h >> 13; +struct __murmur2_or_cityhash<_Size, 32> { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK _Size + operator()(const void* __key, _Size __len) const { + // murmur2 + const _Size __m = 0x5bd1e995; + const _Size __r = 24; + _Size __h = __len; + const unsigned char* __data = static_cast(__key); + for (; __len >= 4; __data += 4, __len -= 4) { + _Size __k = std::__loadword<_Size>(__data); + __k *= __m; + __k ^= __k >> __r; + __k *= __m; __h *= __m; - __h ^= __h >> 15; - return __h; + __h ^= __k; } + switch (__len) { + case 3: + __h ^= static_cast<_Size>(__data[2] << 16); + _LIBCPP_FALLTHROUGH(); + case 2: + __h ^= static_cast<_Size>(__data[1] << 8); + _LIBCPP_FALLTHROUGH(); + case 1: + __h ^= __data[0]; + __h *= __m; + } + __h ^= __h >> 13; + __h *= __m; + __h ^= __h >> 15; + return __h; + } }; template -struct __murmur2_or_cityhash<_Size, 64> -{ +struct __murmur2_or_cityhash<_Size, 64> { // cityhash64 - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - _Size operator()(const void* __key, _Size __len) const { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK _Size + operator()(const void* __key, _Size __len) const { const char* __s = static_cast(__key); if (__len <= 32) { - if (__len <= 16) { + if (__len <= 16) { return __hash_len_0_to_16(__s, __len); - } else { + } else { return __hash_len_17_to_32(__s, __len); - } + } } else if (__len <= 64) { - return __hash_len_33_to_64(__s, __len); + return __hash_len_33_to_64(__s, __len); } // For strings over 64 bytes we hash the end first, and then as we // loop we keep 56 bytes of state: v, w, x, y, and z. _Size __x = std::__loadword<_Size>(__s + __len - 40); - _Size __y = std::__loadword<_Size>(__s + __len - 16) + - std::__loadword<_Size>(__s + __len - 56); - _Size __z = __hash_len_16(std::__loadword<_Size>(__s + __len - 48) + __len, - std::__loadword<_Size>(__s + __len - 24)); + _Size __y = std::__loadword<_Size>(__s + __len - 16) + std::__loadword<_Size>(__s + __len - 56); + _Size __z = + __hash_len_16(std::__loadword<_Size>(__s + __len - 48) + __len, std::__loadword<_Size>(__s + __len - 24)); pair<_Size, _Size> __v = __weak_hash_len_32_with_seeds(__s + __len - 64, __len, __z); pair<_Size, _Size> __w = __weak_hash_len_32_with_seeds(__s + __len - 32, __y + __k1, __x); - __x = __x * __k1 + std::__loadword<_Size>(__s); + __x = __x * __k1 + std::__loadword<_Size>(__s); // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks. __len = (__len - 1) & ~static_cast<_Size>(63); do { - __x = __rotate(__x + __y + __v.first + std::__loadword<_Size>(__s + 8), 37) * __k1; - __y = __rotate(__y + __v.second + std::__loadword<_Size>(__s + 48), 42) * __k1; - __x ^= __w.second; - __y += __v.first + std::__loadword<_Size>(__s + 40); - __z = __rotate(__z + __w.first, 33) * __k1; - __v = __weak_hash_len_32_with_seeds(__s, __v.second * __k1, __x + __w.first); - __w = __weak_hash_len_32_with_seeds(__s + 32, __z + __w.second, - __y + std::__loadword<_Size>(__s + 16)); - std::swap(__z, __x); - __s += 64; - __len -= 64; + __x = __rotate(__x + __y + __v.first + std::__loadword<_Size>(__s + 8), 37) * __k1; + __y = __rotate(__y + __v.second + std::__loadword<_Size>(__s + 48), 42) * __k1; + __x ^= __w.second; + __y += __v.first + std::__loadword<_Size>(__s + 40); + __z = __rotate(__z + __w.first, 33) * __k1; + __v = __weak_hash_len_32_with_seeds(__s, __v.second * __k1, __x + __w.first); + __w = __weak_hash_len_32_with_seeds(__s + 32, __z + __w.second, __y + std::__loadword<_Size>(__s + 16)); + std::swap(__z, __x); + __s += 64; + __len -= 64; } while (__len != 0); - return __hash_len_16( - __hash_len_16(__v.first, __w.first) + __shift_mix(__y) * __k1 + __z, - __hash_len_16(__v.second, __w.second) + __x); + return __hash_len_16(__hash_len_16(__v.first, __w.first) + __shift_mix(__y) * __k1 + __z, + __hash_len_16(__v.second, __w.second) + __x); } - private: - // Some primes between 2^63 and 2^64. - static const _Size __k0 = 0xc3a5c85c97cb3127ULL; - static const _Size __k1 = 0xb492b66fbe98f273ULL; - static const _Size __k2 = 0x9ae16a3b2f90404fULL; - static const _Size __k3 = 0xc949d7c7509e6557ULL; +private: + // Some primes between 2^63 and 2^64. + static const _Size __k0 = 0xc3a5c85c97cb3127ULL; + static const _Size __k1 = 0xb492b66fbe98f273ULL; + static const _Size __k2 = 0x9ae16a3b2f90404fULL; + static const _Size __k3 = 0xc949d7c7509e6557ULL; - _LIBCPP_HIDE_FROM_ABI - static _Size __rotate(_Size __val, int __shift) { + _LIBCPP_HIDE_FROM_ABI static _Size __rotate(_Size __val, int __shift) { return __shift == 0 ? __val : ((__val >> __shift) | (__val << (64 - __shift))); } - _LIBCPP_HIDE_FROM_ABI - static _Size __rotate_by_at_least_1(_Size __val, int __shift) { + _LIBCPP_HIDE_FROM_ABI static _Size __rotate_by_at_least_1(_Size __val, int __shift) { return (__val >> __shift) | (__val << (64 - __shift)); } - _LIBCPP_HIDE_FROM_ABI - static _Size __shift_mix(_Size __val) { - return __val ^ (__val >> 47); - } + _LIBCPP_HIDE_FROM_ABI static _Size __shift_mix(_Size __val) { return __val ^ (__val >> 47); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static _Size __hash_len_16(_Size __u, _Size __v) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size __hash_len_16(_Size __u, _Size __v) { const _Size __mul = 0x9ddfea08eb382d69ULL; - _Size __a = (__u ^ __v) * __mul; + _Size __a = (__u ^ __v) * __mul; __a ^= (__a >> 47); _Size __b = (__v ^ __a) * __mul; __b ^= (__b >> 47); @@ -169,8 +153,8 @@ struct __murmur2_or_cityhash<_Size, 64> return __b; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static _Size __hash_len_0_to_16(const char* __s, _Size __len) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size + __hash_len_0_to_16(const char* __s, _Size __len) { if (__len > 8) { const _Size __a = std::__loadword<_Size>(__s); const _Size __b = std::__loadword<_Size>(__s + __len - 8); @@ -189,32 +173,29 @@ struct __murmur2_or_cityhash<_Size, 64> const unsigned char __a = static_cast(__s[0]); const unsigned char __b = static_cast(__s[__len >> 1]); const unsigned char __c = static_cast(__s[__len - 1]); - const uint32_t __y = static_cast(__a) + - (static_cast(__b) << 8); - const uint32_t __z = __len + (static_cast(__c) << 2); + const uint32_t __y = static_cast(__a) + (static_cast(__b) << 8); + const uint32_t __z = __len + (static_cast(__c) << 2); return __shift_mix(__y * __k2 ^ __z * __k3) * __k2; } return __k2; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static _Size __hash_len_17_to_32(const char *__s, _Size __len) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size + __hash_len_17_to_32(const char* __s, _Size __len) { const _Size __a = std::__loadword<_Size>(__s) * __k1; const _Size __b = std::__loadword<_Size>(__s + 8); const _Size __c = std::__loadword<_Size>(__s + __len - 8) * __k2; const _Size __d = std::__loadword<_Size>(__s + __len - 16) * __k0; - return __hash_len_16(__rotate(__a - __b, 43) + __rotate(__c, 30) + __d, - __a + __rotate(__b ^ __k3, 20) - __c + __len); + return __hash_len_16( + __rotate(__a - __b, 43) + __rotate(__c, 30) + __d, __a + __rotate(__b ^ __k3, 20) - __c + __len); } // Return a 16-byte hash for 48 bytes. Quick and dirty. // Callers do best to use "random-looking" values for a and b. - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static pair<_Size, _Size> __weak_hash_len_32_with_seeds( - _Size __w, _Size __x, _Size __y, _Size __z, _Size __a, _Size __b) - { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static pair<_Size, _Size> + __weak_hash_len_32_with_seeds(_Size __w, _Size __x, _Size __y, _Size __z, _Size __a, _Size __b) { __a += __w; - __b = __rotate(__b + __a + __z, 21); + __b = __rotate(__b + __a + __z, 21); const _Size __c = __a; __a += __x; __a += __y; @@ -223,24 +204,22 @@ struct __murmur2_or_cityhash<_Size, 64> } // Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static pair<_Size, _Size> __weak_hash_len_32_with_seeds( - const char* __s, _Size __a, _Size __b) - { - return __weak_hash_len_32_with_seeds(std::__loadword<_Size>(__s), - std::__loadword<_Size>(__s + 8), - std::__loadword<_Size>(__s + 16), - std::__loadword<_Size>(__s + 24), - __a, - __b); + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static pair<_Size, _Size> + __weak_hash_len_32_with_seeds(const char* __s, _Size __a, _Size __b) { + return __weak_hash_len_32_with_seeds( + std::__loadword<_Size>(__s), + std::__loadword<_Size>(__s + 8), + std::__loadword<_Size>(__s + 16), + std::__loadword<_Size>(__s + 24), + __a, + __b); } // Return an 8-byte hash for 33 to 64 bytes. - _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK - static _Size __hash_len_33_to_64(const char *__s, size_t __len) { + _LIBCPP_HIDE_FROM_ABI _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK static _Size + __hash_len_33_to_64(const char* __s, size_t __len) { _Size __z = std::__loadword<_Size>(__s + 24); - _Size __a = std::__loadword<_Size>(__s) + - (__len + std::__loadword<_Size>(__s + __len - 16)) * __k0; + _Size __a = std::__loadword<_Size>(__s) + (__len + std::__loadword<_Size>(__s + __len - 16)) * __k0; _Size __b = __rotate(__a + __z, 52); _Size __c = __rotate(__a, 37); __a += std::__loadword<_Size>(__s + 8); @@ -248,7 +227,7 @@ struct __murmur2_or_cityhash<_Size, 64> __a += std::__loadword<_Size>(__s + 16); _Size __vf = __a + __z; _Size __vs = __b + __rotate(__a, 31) + __c; - __a = std::__loadword<_Size>(__s + 16) + std::__loadword<_Size>(__s + __len - 32); + __a = std::__loadword<_Size>(__s + 16) + std::__loadword<_Size>(__s + __len - 32); __z += std::__loadword<_Size>(__s + __len - 8); __b = __rotate(__a + __z, 52); __c = __rotate(__a, 37); @@ -257,7 +236,7 @@ struct __murmur2_or_cityhash<_Size, 64> __a += std::__loadword<_Size>(__s + __len - 16); _Size __wf = __a + __z; _Size __ws = __b + __rotate(__a, 31) + __c; - _Size __r = __shift_mix((__vf + __ws) * __k2 + (__wf + __vs) * __k0); + _Size __r = __shift_mix((__vf + __ws) * __k2 + (__wf + __vs) * __k0); return __shift_mix(__r * __k0 + __vs) * __k2; } }; @@ -266,104 +245,76 @@ template struct __scalar_hash; template -struct __scalar_hash<_Tp, 0> - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - union - { - _Tp __t; - size_t __a; - } __u; - __u.__a = 0; - __u.__t = __v; - return __u.__a; - } +struct __scalar_hash<_Tp, 0> : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + union { + _Tp __t; + size_t __a; + } __u; + __u.__a = 0; + __u.__t = __v; + return __u.__a; + } }; template -struct __scalar_hash<_Tp, 1> - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - union - { - _Tp __t; - size_t __a; - } __u; - __u.__t = __v; - return __u.__a; - } +struct __scalar_hash<_Tp, 1> : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + union { + _Tp __t; + size_t __a; + } __u; + __u.__t = __v; + return __u.__a; + } }; template -struct __scalar_hash<_Tp, 2> - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - union - { - _Tp __t; - struct - { - size_t __a; - size_t __b; - } __s; - } __u; - __u.__t = __v; - return __murmur2_or_cityhash()(&__u, sizeof(__u)); - } +struct __scalar_hash<_Tp, 2> : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + union { + _Tp __t; + struct { + size_t __a; + size_t __b; + } __s; + } __u; + __u.__t = __v; + return __murmur2_or_cityhash()(&__u, sizeof(__u)); + } }; template -struct __scalar_hash<_Tp, 3> - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - union - { - _Tp __t; - struct - { - size_t __a; - size_t __b; - size_t __c; - } __s; - } __u; - __u.__t = __v; - return __murmur2_or_cityhash()(&__u, sizeof(__u)); - } +struct __scalar_hash<_Tp, 3> : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + union { + _Tp __t; + struct { + size_t __a; + size_t __b; + size_t __c; + } __s; + } __u; + __u.__t = __v; + return __murmur2_or_cityhash()(&__u, sizeof(__u)); + } }; template -struct __scalar_hash<_Tp, 4> - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - union - { - _Tp __t; - struct - { - size_t __a; - size_t __b; - size_t __c; - size_t __d; - } __s; - } __u; - __u.__t = __v; - return __murmur2_or_cityhash()(&__u, sizeof(__u)); - } +struct __scalar_hash<_Tp, 4> : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + union { + _Tp __t; + struct { + size_t __a; + size_t __b; + size_t __c; + size_t __d; + } __s; + } __u; + __u.__t = __v; + return __murmur2_or_cityhash()(&__u, sizeof(__u)); + } }; struct _PairT { @@ -371,314 +322,223 @@ struct _PairT { size_t second; }; -_LIBCPP_HIDE_FROM_ABI -inline size_t __hash_combine(size_t __lhs, size_t __rhs) _NOEXCEPT { - typedef __scalar_hash<_PairT> _HashT; - const _PairT __p = {__lhs, __rhs}; - return _HashT()(__p); +_LIBCPP_HIDE_FROM_ABI inline size_t __hash_combine(size_t __lhs, size_t __rhs) _NOEXCEPT { + typedef __scalar_hash<_PairT> _HashT; + const _PairT __p = {__lhs, __rhs}; + return _HashT()(__p); } -template -struct _LIBCPP_TEMPLATE_VIS hash<_Tp*> - : public __unary_function<_Tp*, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp* __v) const _NOEXCEPT - { - union - { - _Tp* __t; - size_t __a; - } __u; - __u.__t = __v; - return __murmur2_or_cityhash()(&__u, sizeof(__u)); - } +template +struct _LIBCPP_TEMPLATE_VIS hash<_Tp*> : public __unary_function<_Tp*, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp* __v) const _NOEXCEPT { + union { + _Tp* __t; + size_t __a; + } __u; + __u.__t = __v; + return __murmur2_or_cityhash()(&__u, sizeof(__u)); + } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(bool __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(bool __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(char __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(char __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(signed char __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(signed char __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(unsigned char __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned char __v) const _NOEXCEPT { return static_cast(__v); } }; #ifndef _LIBCPP_HAS_NO_CHAR8_T template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(char8_t __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(char8_t __v) const _NOEXCEPT { return static_cast(__v); } }; #endif // !_LIBCPP_HAS_NO_CHAR8_T template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(char16_t __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(char16_t __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(char32_t __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(char32_t __v) const _NOEXCEPT { return static_cast(__v); } }; #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(wchar_t __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(wchar_t __v) const _NOEXCEPT { return static_cast(__v); } }; #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(short __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(short __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(unsigned short __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned short __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(int __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(int __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(unsigned int __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned int __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(long __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(long __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(unsigned long __v) const _NOEXCEPT {return static_cast(__v);} +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned long __v) const _NOEXCEPT { return static_cast(__v); } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __scalar_hash -{ -}; +struct _LIBCPP_TEMPLATE_VIS hash : public __scalar_hash {}; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __scalar_hash -{ -}; +struct _LIBCPP_TEMPLATE_VIS hash : public __scalar_hash {}; #ifndef _LIBCPP_HAS_NO_INT128 template <> -struct _LIBCPP_TEMPLATE_VIS hash<__int128_t> - : public __scalar_hash<__int128_t> -{ -}; +struct _LIBCPP_TEMPLATE_VIS hash<__int128_t> : public __scalar_hash<__int128_t> {}; template <> -struct _LIBCPP_TEMPLATE_VIS hash<__uint128_t> - : public __scalar_hash<__uint128_t> -{ -}; +struct _LIBCPP_TEMPLATE_VIS hash<__uint128_t> : public __scalar_hash<__uint128_t> {}; #endif template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __scalar_hash -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(float __v) const _NOEXCEPT - { - // -0.0 and 0.0 should return same hash - if (__v == 0.0f) - return 0; - return __scalar_hash::operator()(__v); - } +struct _LIBCPP_TEMPLATE_VIS hash : public __scalar_hash { + _LIBCPP_HIDE_FROM_ABI size_t operator()(float __v) const _NOEXCEPT { + // -0.0 and 0.0 should return same hash + if (__v == 0.0f) + return 0; + return __scalar_hash::operator()(__v); + } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __scalar_hash -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(double __v) const _NOEXCEPT - { - // -0.0 and 0.0 should return same hash - if (__v == 0.0) - return 0; - return __scalar_hash::operator()(__v); - } +struct _LIBCPP_TEMPLATE_VIS hash : public __scalar_hash { + _LIBCPP_HIDE_FROM_ABI size_t operator()(double __v) const _NOEXCEPT { + // -0.0 and 0.0 should return same hash + if (__v == 0.0) + return 0; + return __scalar_hash::operator()(__v); + } }; template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __scalar_hash -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(long double __v) const _NOEXCEPT - { - // -0.0 and 0.0 should return same hash - if (__v == 0.0L) - return 0; +struct _LIBCPP_TEMPLATE_VIS hash : public __scalar_hash { + _LIBCPP_HIDE_FROM_ABI size_t operator()(long double __v) const _NOEXCEPT { + // -0.0 and 0.0 should return same hash + if (__v == 0.0L) + return 0; #if defined(__i386__) || (defined(__x86_64__) && defined(__ILP32__)) - // Zero out padding bits - union - { - long double __t; - struct - { - size_t __a; - size_t __b; - size_t __c; - size_t __d; - } __s; - } __u; - __u.__s.__a = 0; - __u.__s.__b = 0; - __u.__s.__c = 0; - __u.__s.__d = 0; - __u.__t = __v; - return __u.__s.__a ^ __u.__s.__b ^ __u.__s.__c ^ __u.__s.__d; + // Zero out padding bits + union { + long double __t; + struct { + size_t __a; + size_t __b; + size_t __c; + size_t __d; + } __s; + } __u; + __u.__s.__a = 0; + __u.__s.__b = 0; + __u.__s.__c = 0; + __u.__s.__d = 0; + __u.__t = __v; + return __u.__s.__a ^ __u.__s.__b ^ __u.__s.__c ^ __u.__s.__d; #elif defined(__x86_64__) - // Zero out padding bits - union - { - long double __t; - struct - { - size_t __a; - size_t __b; - } __s; - } __u; - __u.__s.__a = 0; - __u.__s.__b = 0; - __u.__t = __v; - return __u.__s.__a ^ __u.__s.__b; + // Zero out padding bits + union { + long double __t; + struct { + size_t __a; + size_t __b; + } __s; + } __u; + __u.__s.__a = 0; + __u.__s.__b = 0; + __u.__t = __v; + return __u.__s.__a ^ __u.__s.__b; #else - return __scalar_hash::operator()(__v); + return __scalar_hash::operator()(__v); #endif - } + } }; template ::value> -struct _LIBCPP_TEMPLATE_VIS __enum_hash - : public __unary_function<_Tp, size_t> -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(_Tp __v) const _NOEXCEPT - { - typedef typename underlying_type<_Tp>::type type; - return hash()(static_cast(__v)); - } +struct _LIBCPP_TEMPLATE_VIS __enum_hash : public __unary_function<_Tp, size_t> { + _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { + typedef typename underlying_type<_Tp>::type type; + return hash()(static_cast(__v)); + } }; template struct _LIBCPP_TEMPLATE_VIS __enum_hash<_Tp, false> { - __enum_hash() = delete; - __enum_hash(__enum_hash const&) = delete; - __enum_hash& operator=(__enum_hash const&) = delete; + __enum_hash() = delete; + __enum_hash(__enum_hash const&) = delete; + __enum_hash& operator=(__enum_hash const&) = delete; }; template -struct _LIBCPP_TEMPLATE_VIS hash : public __enum_hash<_Tp> -{ -}; +struct _LIBCPP_TEMPLATE_VIS hash : public __enum_hash<_Tp> {}; #if _LIBCPP_STD_VER >= 17 template <> -struct _LIBCPP_TEMPLATE_VIS hash - : public __unary_function -{ - _LIBCPP_HIDE_FROM_ABI - size_t operator()(nullptr_t) const _NOEXCEPT { - return 662607004ull; - } +struct _LIBCPP_TEMPLATE_VIS hash : public __unary_function { + _LIBCPP_HIDE_FROM_ABI size_t operator()(nullptr_t) const _NOEXCEPT { return 662607004ull; } }; #endif #ifndef _LIBCPP_CXX03_LANG template -using __check_hash_requirements _LIBCPP_NODEBUG = integral_constant::value && - is_move_constructible<_Hash>::value && - __invokable_r::value ->; +using __check_hash_requirements _LIBCPP_NODEBUG = + integral_constant::value && is_move_constructible<_Hash>::value && + __invokable_r::value >; template > -using __has_enabled_hash _LIBCPP_NODEBUG = integral_constant::value && - is_default_constructible<_Hash>::value ->; +using __has_enabled_hash _LIBCPP_NODEBUG = + integral_constant::value && is_default_constructible<_Hash>::value >; -#if _LIBCPP_STD_VER >= 17 +# if _LIBCPP_STD_VER >= 17 template using __enable_hash_helper_imp _LIBCPP_NODEBUG = _Type; -template -using __enable_hash_helper _LIBCPP_NODEBUG = __enable_hash_helper_imp<_Type, - __enable_if_t<__all<__has_enabled_hash<_Keys>::value...>::value> ->; -#else -template +template +using __enable_hash_helper _LIBCPP_NODEBUG = + __enable_hash_helper_imp<_Type, __enable_if_t<__all<__has_enabled_hash<_Keys>::value...>::value> >; +# else +template using __enable_hash_helper _LIBCPP_NODEBUG = _Type; -#endif +# endif #endif // !_LIBCPP_CXX03_LANG diff --git a/libcxx/include/__functional/identity.h b/libcxx/include/__functional/identity.h index ee92c4130e716..d0e7a7d0769a9 100644 --- a/libcxx/include/__functional/identity.h +++ b/libcxx/include/__functional/identity.h @@ -38,13 +38,12 @@ struct __is_identity<__identity> : true_type {}; #if _LIBCPP_STD_VER >= 20 struct identity { - template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator()(_Tp&& __t) const noexcept - { - return std::forward<_Tp>(__t); - } + template + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator()(_Tp&& __t) const noexcept { + return std::forward<_Tp>(__t); + } - using is_transparent = void; + using is_transparent = void; }; template <> diff --git a/libcxx/include/__functional/invoke.h b/libcxx/include/__functional/invoke.h index df2f99fe27d6f..ef4bf25f07759 100644 --- a/libcxx/include/__functional/invoke.h +++ b/libcxx/include/__functional/invoke.h @@ -22,12 +22,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 -template +template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 invoke_result_t<_Fn, _Args...> -invoke(_Fn&& __f, _Args&&... __args) - noexcept(is_nothrow_invocable_v<_Fn, _Args...>) -{ - return std::__invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); +invoke(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_v<_Fn, _Args...>) { + return std::__invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); } #endif // _LIBCPP_STD_VER >= 17 @@ -37,17 +35,17 @@ template requires is_invocable_r_v<_Result, _Fn, _Args...> _LIBCPP_HIDE_FROM_ABI constexpr _Result invoke_r(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_r_v<_Result, _Fn, _Args...>) { - if constexpr (is_void_v<_Result>) { - static_cast(std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...)); - } else { - // TODO: Use reference_converts_from_temporary_v once implemented - // using _ImplicitInvokeResult = invoke_result_t<_Fn, _Args...>; - // static_assert(!reference_converts_from_temporary_v<_Result, _ImplicitInvokeResult>, - static_assert(true, - "Returning from invoke_r would bind a temporary object to the reference return type, " - "which would result in a dangling reference."); - return std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); - } + if constexpr (is_void_v<_Result>) { + static_cast(std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...)); + } else { + // TODO: Use reference_converts_from_temporary_v once implemented + // using _ImplicitInvokeResult = invoke_result_t<_Fn, _Args...>; + // static_assert(!reference_converts_from_temporary_v<_Result, _ImplicitInvokeResult>, + static_assert(true, + "Returning from invoke_r would bind a temporary object to the reference return type, " + "which would result in a dangling reference."); + return std::invoke(std::forward<_Fn>(__f), std::forward<_Args>(__args)...); + } } #endif diff --git a/libcxx/include/__functional/is_transparent.h b/libcxx/include/__functional/is_transparent.h index c539a07d696dd..13fc94f71c6ba 100644 --- a/libcxx/include/__functional/is_transparent.h +++ b/libcxx/include/__functional/is_transparent.h @@ -26,8 +26,7 @@ template struct __is_transparent : false_type {}; template -struct __is_transparent<_Tp, _Up, __void_t > - : true_type {}; +struct __is_transparent<_Tp, _Up, __void_t > : true_type {}; #endif diff --git a/libcxx/include/__functional/mem_fn.h b/libcxx/include/__functional/mem_fn.h index 77c8136d8785f..349a6ce3a7572 100644 --- a/libcxx/include/__functional/mem_fn.h +++ b/libcxx/include/__functional/mem_fn.h @@ -23,34 +23,30 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -class __mem_fn : public __weak_result_type<_Tp> -{ +class __mem_fn : public __weak_result_type<_Tp> { public: - // types - typedef _Tp type; + // types + typedef _Tp type; + private: - type __f_; + type __f_; public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - __mem_fn(type __f) _NOEXCEPT : __f_(__f) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn(type __f) _NOEXCEPT : __f_(__f) {} - // invoke - template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + // invoke + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - typename __invoke_return::type - operator() (_ArgTypes&&... __args) const { - return std::__invoke(__f_, std::forward<_ArgTypes>(__args)...); - } + typename __invoke_return::type + operator()(_ArgTypes&&... __args) const { + return std::__invoke(__f_, std::forward<_ArgTypes>(__args)...); + } }; -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -__mem_fn<_Rp _Tp::*> -mem_fn(_Rp _Tp::* __pm) _NOEXCEPT -{ - return __mem_fn<_Rp _Tp::*>(__pm); +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn<_Rp _Tp::*> mem_fn(_Rp _Tp::*__pm) _NOEXCEPT { + return __mem_fn<_Rp _Tp::*>(__pm); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__functional/mem_fun_ref.h b/libcxx/include/__functional/mem_fun_ref.h index 620d479fdd61a..fe43c46560924 100644 --- a/libcxx/include/__functional/mem_fun_ref.h +++ b/libcxx/include/__functional/mem_fun_ref.h @@ -22,149 +22,122 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) -template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_t - : public __unary_function<_Tp*, _Sp> -{ - _Sp (_Tp::*__p_)(); +template +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_t : public __unary_function<_Tp*, _Sp> { + _Sp (_Tp::*__p_)(); + public: - _LIBCPP_HIDE_FROM_ABI explicit mem_fun_t(_Sp (_Tp::*__p)()) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p) const - {return (__p->*__p_)();} + _LIBCPP_HIDE_FROM_ABI explicit mem_fun_t(_Sp (_Tp::*__p)()) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p) const { return (__p->*__p_)(); } }; -template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_t - : public __binary_function<_Tp*, _Ap, _Sp> -{ - _Sp (_Tp::*__p_)(_Ap); +template +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_t : public __binary_function<_Tp*, _Ap, _Sp> { + _Sp (_Tp::*__p_)(_Ap); + public: - _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_t(_Sp (_Tp::*__p)(_Ap)) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p, _Ap __x) const - {return (__p->*__p_)(__x);} + _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_t(_Sp (_Tp::*__p)(_Ap)) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp* __p, _Ap __x) const { return (__p->*__p_)(__x); } }; -template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -mem_fun_t<_Sp,_Tp> -mem_fun(_Sp (_Tp::*__f)()) - {return mem_fun_t<_Sp,_Tp>(__f);} - -template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -mem_fun1_t<_Sp,_Tp,_Ap> -mem_fun(_Sp (_Tp::*__f)(_Ap)) - {return mem_fun1_t<_Sp,_Tp,_Ap>(__f);} - -template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_ref_t - : public __unary_function<_Tp, _Sp> -{ - _Sp (_Tp::*__p_)(); +template +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun_t<_Sp, _Tp> mem_fun(_Sp (_Tp::*__f)()) { + return mem_fun_t<_Sp, _Tp>(__f); +} + +template +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun1_t<_Sp, _Tp, _Ap> mem_fun(_Sp (_Tp::*__f)(_Ap)) { + return mem_fun1_t<_Sp, _Tp, _Ap>(__f); +} + +template +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun_ref_t : public __unary_function<_Tp, _Sp> { + _Sp (_Tp::*__p_)(); + public: - _LIBCPP_HIDE_FROM_ABI explicit mem_fun_ref_t(_Sp (_Tp::*__p)()) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p) const - {return (__p.*__p_)();} + _LIBCPP_HIDE_FROM_ABI explicit mem_fun_ref_t(_Sp (_Tp::*__p)()) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p) const { return (__p.*__p_)(); } }; -template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_ref_t - : public __binary_function<_Tp, _Ap, _Sp> -{ - _Sp (_Tp::*__p_)(_Ap); +template +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 mem_fun1_ref_t : public __binary_function<_Tp, _Ap, _Sp> { + _Sp (_Tp::*__p_)(_Ap); + public: - _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap)) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p, _Ap __x) const - {return (__p.*__p_)(__x);} + _LIBCPP_HIDE_FROM_ABI explicit mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap)) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(_Tp& __p, _Ap __x) const { return (__p.*__p_)(__x); } }; -template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -mem_fun_ref_t<_Sp,_Tp> -mem_fun_ref(_Sp (_Tp::*__f)()) - {return mem_fun_ref_t<_Sp,_Tp>(__f);} +template +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun_ref_t<_Sp, _Tp> mem_fun_ref(_Sp (_Tp::*__f)()) { + return mem_fun_ref_t<_Sp, _Tp>(__f); +} -template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -mem_fun1_ref_t<_Sp,_Tp,_Ap> -mem_fun_ref(_Sp (_Tp::*__f)(_Ap)) - {return mem_fun1_ref_t<_Sp,_Tp,_Ap>(__f);} +template +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI mem_fun1_ref_t<_Sp, _Tp, _Ap> +mem_fun_ref(_Sp (_Tp::*__f)(_Ap)) { + return mem_fun1_ref_t<_Sp, _Tp, _Ap>(__f); +} template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_t - : public __unary_function -{ - _Sp (_Tp::*__p_)() const; +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_t : public __unary_function { + _Sp (_Tp::*__p_)() const; + public: - _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_t(_Sp (_Tp::*__p)() const) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p) const - {return (__p->*__p_)();} + _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_t(_Sp (_Tp::*__p)() const) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p) const { return (__p->*__p_)(); } }; template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_t - : public __binary_function -{ - _Sp (_Tp::*__p_)(_Ap) const; + : public __binary_function { + _Sp (_Tp::*__p_)(_Ap) const; + public: - _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_t(_Sp (_Tp::*__p)(_Ap) const) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p, _Ap __x) const - {return (__p->*__p_)(__x);} + _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_t(_Sp (_Tp::*__p)(_Ap) const) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp* __p, _Ap __x) const { return (__p->*__p_)(__x); } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -const_mem_fun_t<_Sp,_Tp> -mem_fun(_Sp (_Tp::*__f)() const) - {return const_mem_fun_t<_Sp,_Tp>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun_t<_Sp, _Tp> mem_fun(_Sp (_Tp::*__f)() const) { + return const_mem_fun_t<_Sp, _Tp>(__f); +} template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -const_mem_fun1_t<_Sp,_Tp,_Ap> -mem_fun(_Sp (_Tp::*__f)(_Ap) const) - {return const_mem_fun1_t<_Sp,_Tp,_Ap>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun1_t<_Sp, _Tp, _Ap> +mem_fun(_Sp (_Tp::*__f)(_Ap) const) { + return const_mem_fun1_t<_Sp, _Tp, _Ap>(__f); +} template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_ref_t - : public __unary_function<_Tp, _Sp> -{ - _Sp (_Tp::*__p_)() const; +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun_ref_t : public __unary_function<_Tp, _Sp> { + _Sp (_Tp::*__p_)() const; + public: - _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_ref_t(_Sp (_Tp::*__p)() const) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p) const - {return (__p.*__p_)();} + _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun_ref_t(_Sp (_Tp::*__p)() const) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p) const { return (__p.*__p_)(); } }; template -class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_ref_t - : public __binary_function<_Tp, _Ap, _Sp> -{ - _Sp (_Tp::*__p_)(_Ap) const; +class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 const_mem_fun1_ref_t : public __binary_function<_Tp, _Ap, _Sp> { + _Sp (_Tp::*__p_)(_Ap) const; + public: - _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap) const) - : __p_(__p) {} - _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p, _Ap __x) const - {return (__p.*__p_)(__x);} + _LIBCPP_HIDE_FROM_ABI explicit const_mem_fun1_ref_t(_Sp (_Tp::*__p)(_Ap) const) : __p_(__p) {} + _LIBCPP_HIDE_FROM_ABI _Sp operator()(const _Tp& __p, _Ap __x) const { return (__p.*__p_)(__x); } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -const_mem_fun_ref_t<_Sp,_Tp> -mem_fun_ref(_Sp (_Tp::*__f)() const) - {return const_mem_fun_ref_t<_Sp,_Tp>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun_ref_t<_Sp, _Tp> +mem_fun_ref(_Sp (_Tp::*__f)() const) { + return const_mem_fun_ref_t<_Sp, _Tp>(__f); +} template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -const_mem_fun1_ref_t<_Sp,_Tp,_Ap> -mem_fun_ref(_Sp (_Tp::*__f)(_Ap) const) - {return const_mem_fun1_ref_t<_Sp,_Tp,_Ap>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI const_mem_fun1_ref_t<_Sp, _Tp, _Ap> +mem_fun_ref(_Sp (_Tp::*__f)(_Ap) const) { + return const_mem_fun1_ref_t<_Sp, _Tp, _Ap>(__f); +} #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) diff --git a/libcxx/include/__functional/not_fn.h b/libcxx/include/__functional/not_fn.h index bbd1a8bc35af5..23a491c135d79 100644 --- a/libcxx/include/__functional/not_fn.h +++ b/libcxx/include/__functional/not_fn.h @@ -28,26 +28,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 struct __not_fn_op { - template - _LIBCPP_HIDE_FROM_ABI - _LIBCPP_CONSTEXPR_SINCE_CXX20 auto operator()(_Args&&... __args) const - noexcept(noexcept(!std::invoke(std::forward<_Args>(__args)...))) - -> decltype( !std::invoke(std::forward<_Args>(__args)...)) - { return !std::invoke(std::forward<_Args>(__args)...); } + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto operator()(_Args&&... __args) const + noexcept(noexcept(!std::invoke(std::forward<_Args>(__args)...))) + -> decltype(!std::invoke(std::forward<_Args>(__args)...)) { + return !std::invoke(std::forward<_Args>(__args)...); + } }; template struct __not_fn_t : __perfect_forward<__not_fn_op, _Fn> { - using __perfect_forward<__not_fn_op, _Fn>::__perfect_forward; + using __perfect_forward<__not_fn_op, _Fn>::__perfect_forward; }; -template , _Fn> && - is_move_constructible_v> ->> -_LIBCPP_HIDE_FROM_ABI -_LIBCPP_CONSTEXPR_SINCE_CXX20 auto not_fn(_Fn&& __f) { - return __not_fn_t>(std::forward<_Fn>(__f)); +template , _Fn> && is_move_constructible_v> >> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto not_fn(_Fn&& __f) { + return __not_fn_t>(std::forward<_Fn>(__f)); } #endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/include/__functional/operations.h b/libcxx/include/__functional/operations.h index 23928e9fc22d1..7ddc00650f162 100644 --- a/libcxx/include/__functional/operations.h +++ b/libcxx/include/__functional/operations.h @@ -30,13 +30,11 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS plus - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x + __y;} +struct _LIBCPP_TEMPLATE_VIS plus : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x + __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(plus); @@ -50,15 +48,14 @@ struct __desugars_to<__plus_tag, plus, _Tp, _Up> : true_type {}; #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS plus -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) + std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) + std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) + std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS plus { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) + std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) + std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) + std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -67,27 +64,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS minus - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x - __y;} +struct _LIBCPP_TEMPLATE_VIS minus : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x - __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(minus); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS minus -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) - std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) - std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) - std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS minus { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) - std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) - std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) - std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -96,27 +90,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS multiplies - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x * __y;} +struct _LIBCPP_TEMPLATE_VIS multiplies : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x * __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(multiplies); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS multiplies -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) * std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) * std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) * std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS multiplies { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) * std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) * std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) * std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -125,27 +116,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS divides - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x / __y;} +struct _LIBCPP_TEMPLATE_VIS divides : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x / __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(divides); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS divides -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) / std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) / std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) / std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS divides { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) / std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) / std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) / std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -154,27 +142,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS modulus - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x % __y;} +struct _LIBCPP_TEMPLATE_VIS modulus : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x % __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(modulus); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS modulus -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) % std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) % std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) % std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS modulus { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) % std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) % std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) % std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -183,27 +168,21 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS negate - : __unary_function<_Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x) const - {return -__x;} +struct _LIBCPP_TEMPLATE_VIS negate : __unary_function<_Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return -__x; } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(negate); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS negate -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_Tp&& __x) const - noexcept(noexcept(- std::forward<_Tp>(__x))) - -> decltype( - std::forward<_Tp>(__x)) - { return - std::forward<_Tp>(__x); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS negate { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const + noexcept(noexcept(-std::forward<_Tp>(__x))) -> decltype(-std::forward<_Tp>(__x)) { + return -std::forward<_Tp>(__x); + } + typedef void is_transparent; }; #endif @@ -214,51 +193,42 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS bit_and - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x & __y;} +struct _LIBCPP_TEMPLATE_VIS bit_and : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x & __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_and); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS bit_and -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) & std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) & std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) & std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS bit_and { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) & std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) & std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) & std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif #if _LIBCPP_STD_VER >= 14 template -struct _LIBCPP_TEMPLATE_VIS bit_not - : __unary_function<_Tp, _Tp> -{ - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x) const - {return ~__x;} +struct _LIBCPP_TEMPLATE_VIS bit_not : __unary_function<_Tp, _Tp> { + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x) const { return ~__x; } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_not); template <> -struct _LIBCPP_TEMPLATE_VIS bit_not -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_Tp&& __x) const - noexcept(noexcept(~std::forward<_Tp>(__x))) - -> decltype( ~std::forward<_Tp>(__x)) - { return ~std::forward<_Tp>(__x); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS bit_not { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const + noexcept(noexcept(~std::forward<_Tp>(__x))) -> decltype(~std::forward<_Tp>(__x)) { + return ~std::forward<_Tp>(__x); + } + typedef void is_transparent; }; #endif @@ -267,27 +237,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS bit_or - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x | __y;} +struct _LIBCPP_TEMPLATE_VIS bit_or : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x | __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_or); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS bit_or -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) | std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) | std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) | std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS bit_or { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) | std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) | std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) | std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -296,27 +263,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS bit_xor - : __binary_function<_Tp, _Tp, _Tp> -{ - typedef _Tp __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - _Tp operator()(const _Tp& __x, const _Tp& __y) const - {return __x ^ __y;} +struct _LIBCPP_TEMPLATE_VIS bit_xor : __binary_function<_Tp, _Tp, _Tp> { + typedef _Tp __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _Tp operator()(const _Tp& __x, const _Tp& __y) const { + return __x ^ __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(bit_xor); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS bit_xor -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) ^ std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) ^ std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) ^ std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS bit_xor { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) ^ std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) ^ std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) ^ std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -327,27 +291,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS equal_to - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x == __y;} +struct _LIBCPP_TEMPLATE_VIS equal_to : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x == __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(equal_to); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS equal_to -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) == std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) == std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) == std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS equal_to { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) == std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) == std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) == std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -365,27 +326,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS not_equal_to - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x != __y;} +struct _LIBCPP_TEMPLATE_VIS not_equal_to : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x != __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(not_equal_to); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS not_equal_to -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) != std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) != std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) != std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS not_equal_to { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) != std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) != std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) != std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -394,27 +352,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS less - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x < __y;} +struct _LIBCPP_TEMPLATE_VIS less : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x < __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(less); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS less -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) < std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) < std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) < std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS less { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) < std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) < std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) < std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -423,27 +378,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS less_equal - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x <= __y;} +struct _LIBCPP_TEMPLATE_VIS less_equal : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x <= __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(less_equal); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS less_equal -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) <= std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) <= std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) <= std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS less_equal { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) <= std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) <= std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) <= std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -452,27 +404,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS greater_equal - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x >= __y;} +struct _LIBCPP_TEMPLATE_VIS greater_equal : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x >= __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(greater_equal); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS greater_equal -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) >= std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) >= std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) >= std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS greater_equal { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) >= std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) >= std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) >= std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -481,27 +430,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS greater - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x > __y;} +struct _LIBCPP_TEMPLATE_VIS greater : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x > __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(greater); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS greater -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) > std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) > std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) > std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS greater { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) > std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) > std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) > std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -512,27 +458,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS logical_and - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x && __y;} +struct _LIBCPP_TEMPLATE_VIS logical_and : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x && __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_and); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS logical_and -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) && std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) && std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) && std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS logical_and { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) && std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) && std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) && std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif @@ -541,27 +484,21 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS logical_not - : __unary_function<_Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x) const - {return !__x;} +struct _LIBCPP_TEMPLATE_VIS logical_not : __unary_function<_Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x) const { return !__x; } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_not); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS logical_not -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_Tp&& __x) const - noexcept(noexcept(!std::forward<_Tp>(__x))) - -> decltype( !std::forward<_Tp>(__x)) - { return !std::forward<_Tp>(__x); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS logical_not { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_Tp&& __x) const + noexcept(noexcept(!std::forward<_Tp>(__x))) -> decltype(!std::forward<_Tp>(__x)) { + return !std::forward<_Tp>(__x); + } + typedef void is_transparent; }; #endif @@ -570,27 +507,24 @@ template #else template #endif -struct _LIBCPP_TEMPLATE_VIS logical_or - : __binary_function<_Tp, _Tp, bool> -{ - typedef bool __result_type; // used by valarray - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const _Tp& __x, const _Tp& __y) const - {return __x || __y;} +struct _LIBCPP_TEMPLATE_VIS logical_or : __binary_function<_Tp, _Tp, bool> { + typedef bool __result_type; // used by valarray + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const { + return __x || __y; + } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(logical_or); #if _LIBCPP_STD_VER >= 14 template <> -struct _LIBCPP_TEMPLATE_VIS logical_or -{ - template - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - auto operator()(_T1&& __t, _T2&& __u) const - noexcept(noexcept(std::forward<_T1>(__t) || std::forward<_T2>(__u))) - -> decltype( std::forward<_T1>(__t) || std::forward<_T2>(__u)) - { return std::forward<_T1>(__t) || std::forward<_T2>(__u); } - typedef void is_transparent; +struct _LIBCPP_TEMPLATE_VIS logical_or { + template + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI auto operator()(_T1&& __t, _T2&& __u) const + noexcept(noexcept(std::forward<_T1>(__t) || std::forward<_T2>(__u))) + -> decltype(std::forward<_T1>(__t) || std::forward<_T2>(__u)) { + return std::forward<_T1>(__t) || std::forward<_T2>(__u); + } + typedef void is_transparent; }; #endif diff --git a/libcxx/include/__functional/perfect_forward.h b/libcxx/include/__functional/perfect_forward.h index 2fbb623a7d892..74177c789b4ad 100644 --- a/libcxx/include/__functional/perfect_forward.h +++ b/libcxx/include/__functional/perfect_forward.h @@ -40,57 +40,59 @@ struct __perfect_forward_impl<_Op, index_sequence<_Idx...>, _BoundArgs...> { tuple<_BoundArgs...> __bound_args_; public: - template , _Args&&...> - >> + template , _Args&&...> >> _LIBCPP_HIDE_FROM_ABI explicit constexpr __perfect_forward_impl(_Args&&... __bound_args) - : __bound_args_(std::forward<_Args>(__bound_args)...) {} + : __bound_args_(std::forward<_Args>(__bound_args)...) {} _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl(__perfect_forward_impl const&) = default; - _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl(__perfect_forward_impl&&) = default; + _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl(__perfect_forward_impl&&) = default; _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl& operator=(__perfect_forward_impl const&) = default; - _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl& operator=(__perfect_forward_impl&&) = default; + _LIBCPP_HIDE_FROM_ABI __perfect_forward_impl& operator=(__perfect_forward_impl&&) = default; template >> - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) & - noexcept(noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...))) - -> decltype( _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) - { return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...); } + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) & noexcept( + noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...))) + -> decltype(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) { + return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...); + } template >> auto operator()(_Args&&...) & = delete; template >> - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const& - noexcept(noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...))) - -> decltype( _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) - { return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...); } + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const& noexcept( + noexcept(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...))) + -> decltype(_Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...)) { + return _Op()(std::get<_Idx>(__bound_args_)..., std::forward<_Args>(__args)...); + } template >> auto operator()(_Args&&...) const& = delete; template >> - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) && - noexcept(noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...))) - -> decltype( _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) - { return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...); } + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) && noexcept( + noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...))) + -> decltype(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) { + return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...); + } template >> auto operator()(_Args&&...) && = delete; template >> - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const&& - noexcept(noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...))) - -> decltype( _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) - { return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...); } + _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Args&&... __args) const&& noexcept( + noexcept(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...))) + -> decltype(_Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...)) { + return _Op()(std::get<_Idx>(std::move(__bound_args_))..., std::forward<_Args>(__args)...); + } template >> auto operator()(_Args&&...) const&& = delete; }; // __perfect_forward implements a perfect-forwarding call wrapper as explained in [func.require]. -template +template using __perfect_forward = __perfect_forward_impl<_Op, index_sequence_for<_Args...>, _Args...>; #endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/include/__functional/pointer_to_binary_function.h b/libcxx/include/__functional/pointer_to_binary_function.h index 88e6db222e070..51a7c3fe0fc0f 100644 --- a/libcxx/include/__functional/pointer_to_binary_function.h +++ b/libcxx/include/__functional/pointer_to_binary_function.h @@ -23,21 +23,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 pointer_to_binary_function - : public __binary_function<_Arg1, _Arg2, _Result> -{ - _Result (*__f_)(_Arg1, _Arg2); + : public __binary_function<_Arg1, _Arg2, _Result> { + _Result (*__f_)(_Arg1, _Arg2); + public: - _LIBCPP_HIDE_FROM_ABI explicit pointer_to_binary_function(_Result (*__f)(_Arg1, _Arg2)) - : __f_(__f) {} - _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg1 __x, _Arg2 __y) const - {return __f_(__x, __y);} + _LIBCPP_HIDE_FROM_ABI explicit pointer_to_binary_function(_Result (*__f)(_Arg1, _Arg2)) : __f_(__f) {} + _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg1 __x, _Arg2 __y) const { return __f_(__x, __y); } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -pointer_to_binary_function<_Arg1,_Arg2,_Result> -ptr_fun(_Result (*__f)(_Arg1,_Arg2)) - {return pointer_to_binary_function<_Arg1,_Arg2,_Result>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI pointer_to_binary_function<_Arg1, _Arg2, _Result> +ptr_fun(_Result (*__f)(_Arg1, _Arg2)) { + return pointer_to_binary_function<_Arg1, _Arg2, _Result>(__f); +} #endif diff --git a/libcxx/include/__functional/pointer_to_unary_function.h b/libcxx/include/__functional/pointer_to_unary_function.h index ee13acfff757a..0338e76717894 100644 --- a/libcxx/include/__functional/pointer_to_unary_function.h +++ b/libcxx/include/__functional/pointer_to_unary_function.h @@ -23,21 +23,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 pointer_to_unary_function - : public __unary_function<_Arg, _Result> -{ - _Result (*__f_)(_Arg); + : public __unary_function<_Arg, _Result> { + _Result (*__f_)(_Arg); + public: - _LIBCPP_HIDE_FROM_ABI explicit pointer_to_unary_function(_Result (*__f)(_Arg)) - : __f_(__f) {} - _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg __x) const - {return __f_(__x);} + _LIBCPP_HIDE_FROM_ABI explicit pointer_to_unary_function(_Result (*__f)(_Arg)) : __f_(__f) {} + _LIBCPP_HIDE_FROM_ABI _Result operator()(_Arg __x) const { return __f_(__x); } }; template -_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI -pointer_to_unary_function<_Arg,_Result> -ptr_fun(_Result (*__f)(_Arg)) - {return pointer_to_unary_function<_Arg,_Result>(__f);} +_LIBCPP_DEPRECATED_IN_CXX11 inline _LIBCPP_HIDE_FROM_ABI pointer_to_unary_function<_Arg, _Result> +ptr_fun(_Result (*__f)(_Arg)) { + return pointer_to_unary_function<_Arg, _Result>(__f); +} #endif // _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_BINDERS) diff --git a/libcxx/include/__functional/ranges_operations.h b/libcxx/include/__functional/ranges_operations.h index 0ed631d8a74bd..38b28018049eb 100644 --- a/libcxx/include/__functional/ranges_operations.h +++ b/libcxx/include/__functional/ranges_operations.h @@ -29,8 +29,8 @@ namespace ranges { struct equal_to { template - requires equality_comparable_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires equality_comparable_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(std::forward<_Tp>(__t) == std::forward<_Up>(__u)))) { return std::forward<_Tp>(__t) == std::forward<_Up>(__u); } @@ -40,8 +40,8 @@ struct equal_to { struct not_equal_to { template - requires equality_comparable_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires equality_comparable_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(!(std::forward<_Tp>(__t) == std::forward<_Up>(__u))))) { return !(std::forward<_Tp>(__t) == std::forward<_Up>(__u)); } @@ -51,8 +51,8 @@ struct not_equal_to { struct less { template - requires totally_ordered_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires totally_ordered_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(std::forward<_Tp>(__t) < std::forward<_Up>(__u)))) { return std::forward<_Tp>(__t) < std::forward<_Up>(__u); } @@ -62,8 +62,8 @@ struct less { struct less_equal { template - requires totally_ordered_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires totally_ordered_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(!(std::forward<_Up>(__u) < std::forward<_Tp>(__t))))) { return !(std::forward<_Up>(__u) < std::forward<_Tp>(__t)); } @@ -73,8 +73,8 @@ struct less_equal { struct greater { template - requires totally_ordered_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires totally_ordered_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(std::forward<_Up>(__u) < std::forward<_Tp>(__t)))) { return std::forward<_Up>(__u) < std::forward<_Tp>(__t); } @@ -84,8 +84,8 @@ struct greater { struct greater_equal { template - requires totally_ordered_with<_Tp, _Up> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp &&__t, _Up &&__u) const + requires totally_ordered_with<_Tp, _Up> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t, _Up&& __u) const noexcept(noexcept(bool(!(std::forward<_Tp>(__t) < std::forward<_Up>(__u))))) { return !(std::forward<_Tp>(__t) < std::forward<_Up>(__u)); } diff --git a/libcxx/include/__functional/reference_wrapper.h b/libcxx/include/__functional/reference_wrapper.h index 3688590d10d18..54de06a8879c6 100644 --- a/libcxx/include/__functional/reference_wrapper.h +++ b/libcxx/include/__functional/reference_wrapper.h @@ -26,44 +26,43 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> -{ +class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> { public: - // types - typedef _Tp type; + // types + typedef _Tp type; + private: - type* __f_; + type* __f_; - static void __fun(_Tp&) _NOEXCEPT; - static void __fun(_Tp&&) = delete; + static void __fun(_Tp&) _NOEXCEPT; + static void __fun(_Tp&&) = delete; public: - template ::value, decltype(__fun(std::declval<_Up>())) > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - reference_wrapper(_Up&& __u) _NOEXCEPT_(noexcept(__fun(std::declval<_Up>()))) { - type& __f = static_cast<_Up&&>(__u); - __f_ = std::addressof(__f); - } - - // access - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - operator type&() const _NOEXCEPT {return *__f_;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - type& get() const _NOEXCEPT {return *__f_;} - - // invoke - template - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 - typename __invoke_of::type - operator() (_ArgTypes&&... __args) const + template < + class _Up, + class = __enable_if_t::value, decltype(__fun(std::declval<_Up>())) > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper(_Up&& __u) + _NOEXCEPT_(noexcept(__fun(std::declval<_Up>()))) { + type& __f = static_cast<_Up&&>(__u); + __f_ = std::addressof(__f); + } + + // access + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator type&() const _NOEXCEPT { return *__f_; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 type& get() const _NOEXCEPT { return *__f_; } + + // invoke + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __invoke_of::type + operator()(_ArgTypes&&... __args) const #if _LIBCPP_STD_VER >= 17 - // Since is_nothrow_invocable requires C++17 LWG3764 is not backported - // to earlier versions. - noexcept(is_nothrow_invocable_v<_Tp&, _ArgTypes...>) + // Since is_nothrow_invocable requires C++17 LWG3764 is not backported + // to earlier versions. + noexcept(is_nothrow_invocable_v<_Tp&, _ArgTypes...>) #endif - { - return std::__invoke(get(), std::forward<_ArgTypes>(__args)...); - } + { + return std::__invoke(get(), std::forward<_ArgTypes>(__args)...); + } }; #if _LIBCPP_STD_VER >= 17 @@ -72,39 +71,31 @@ reference_wrapper(_Tp&) -> reference_wrapper<_Tp>; #endif template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -reference_wrapper<_Tp> -ref(_Tp& __t) _NOEXCEPT -{ - return reference_wrapper<_Tp>(__t); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT { + return reference_wrapper<_Tp>(__t); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -reference_wrapper<_Tp> -ref(reference_wrapper<_Tp> __t) _NOEXCEPT -{ - return __t; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> +ref(reference_wrapper<_Tp> __t) _NOEXCEPT { + return __t; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -reference_wrapper -cref(const _Tp& __t) _NOEXCEPT -{ - return reference_wrapper(__t); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper cref(const _Tp& __t) _NOEXCEPT { + return reference_wrapper(__t); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -reference_wrapper -cref(reference_wrapper<_Tp> __t) _NOEXCEPT -{ - return __t; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper +cref(reference_wrapper<_Tp> __t) _NOEXCEPT { + return __t; } -template void ref(const _Tp&&) = delete; -template void cref(const _Tp&&) = delete; +template +void ref(const _Tp&&) = delete; +template +void cref(const _Tp&&) = delete; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__functional/unary_function.h b/libcxx/include/__functional/unary_function.h index f07cac175a99f..69b1bc94220ae 100644 --- a/libcxx/include/__functional/unary_function.h +++ b/libcxx/include/__functional/unary_function.h @@ -20,18 +20,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) template -struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 unary_function -{ - typedef _Arg argument_type; - typedef _Result result_type; +struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 unary_function { + typedef _Arg argument_type; + typedef _Result result_type; }; #endif // _LIBCPP_STD_VER <= 14 -template struct __unary_function_keep_layout_base { +template +struct __unary_function_keep_layout_base { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) using argument_type _LIBCPP_DEPRECATED_IN_CXX17 = _Arg; - using result_type _LIBCPP_DEPRECATED_IN_CXX17 = _Result; + using result_type _LIBCPP_DEPRECATED_IN_CXX17 = _Result; #endif }; diff --git a/libcxx/include/__functional/unary_negate.h b/libcxx/include/__functional/unary_negate.h index b2eed8e2db357..d130b7d728a27 100644 --- a/libcxx/include/__functional/unary_negate.h +++ b/libcxx/include/__functional/unary_negate.h @@ -23,22 +23,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 unary_negate - : public __unary_function -{ - _Predicate __pred_; + : public __unary_function { + _Predicate __pred_; + public: - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - explicit unary_negate(const _Predicate& __pred) - : __pred_(__pred) {} - _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI - bool operator()(const typename _Predicate::argument_type& __x) const - {return !__pred_(__x);} + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI explicit unary_negate(const _Predicate& __pred) + : __pred_(__pred) {} + _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI bool + operator()(const typename _Predicate::argument_type& __x) const { + return !__pred_(__x); + } }; template -_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI -unary_negate<_Predicate> -not1(const _Predicate& __pred) {return unary_negate<_Predicate>(__pred);} +_LIBCPP_DEPRECATED_IN_CXX17 inline _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI unary_negate<_Predicate> +not1(const _Predicate& __pred) { + return unary_negate<_Predicate>(__pred); +} #endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS) diff --git a/libcxx/include/__functional/weak_result_type.h b/libcxx/include/__functional/weak_result_type.h index da6fe54bfae72..ad7a8395186cd 100644 --- a/libcxx/include/__functional/weak_result_type.h +++ b/libcxx/include/__functional/weak_result_type.h @@ -25,268 +25,205 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -struct __has_result_type -{ +struct __has_result_type { private: - template static false_type __test(...); - template static true_type __test(typename _Up::result_type* = 0); + template + static false_type __test(...); + template + static true_type __test(typename _Up::result_type* = 0); + public: - static const bool value = decltype(__test<_Tp>(0))::value; + static const bool value = decltype(__test<_Tp>(0))::value; }; // __weak_result_type template -struct __derives_from_unary_function -{ +struct __derives_from_unary_function { private: - struct __two {char __lx; char __lxx;}; - static __two __test(...); - template - static __unary_function<_Ap, _Rp> - __test(const volatile __unary_function<_Ap, _Rp>*); + struct __two { + char __lx; + char __lxx; + }; + static __two __test(...); + template + static __unary_function<_Ap, _Rp> __test(const volatile __unary_function<_Ap, _Rp>*); public: - static const bool value = !is_same::value; - typedef decltype(__test((_Tp*)0)) type; + static const bool value = !is_same::value; + typedef decltype(__test((_Tp*)0)) type; }; template -struct __derives_from_binary_function -{ +struct __derives_from_binary_function { private: - struct __two {char __lx; char __lxx;}; - static __two __test(...); - template - static __binary_function<_A1, _A2, _Rp> - __test(const volatile __binary_function<_A1, _A2, _Rp>*); + struct __two { + char __lx; + char __lxx; + }; + static __two __test(...); + template + static __binary_function<_A1, _A2, _Rp> __test(const volatile __binary_function<_A1, _A2, _Rp>*); public: - static const bool value = !is_same::value; - typedef decltype(__test((_Tp*)0)) type; + static const bool value = !is_same::value; + typedef decltype(__test((_Tp*)0)) type; }; template ::value> -struct __maybe_derive_from_unary_function // bool is true - : public __derives_from_unary_function<_Tp>::type -{ -}; +struct __maybe_derive_from_unary_function // bool is true + : public __derives_from_unary_function<_Tp>::type {}; template -struct __maybe_derive_from_unary_function<_Tp, false> -{ -}; +struct __maybe_derive_from_unary_function<_Tp, false> {}; template ::value> -struct __maybe_derive_from_binary_function // bool is true - : public __derives_from_binary_function<_Tp>::type -{ -}; +struct __maybe_derive_from_binary_function // bool is true + : public __derives_from_binary_function<_Tp>::type {}; template -struct __maybe_derive_from_binary_function<_Tp, false> -{ -}; +struct __maybe_derive_from_binary_function<_Tp, false> {}; template ::value> struct __weak_result_type_imp // bool is true : public __maybe_derive_from_unary_function<_Tp>, - public __maybe_derive_from_binary_function<_Tp> -{ + public __maybe_derive_from_binary_function<_Tp> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = typename _Tp::result_type; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = typename _Tp::result_type; #endif }; template struct __weak_result_type_imp<_Tp, false> - : public __maybe_derive_from_unary_function<_Tp>, - public __maybe_derive_from_binary_function<_Tp> -{ -}; + : public __maybe_derive_from_unary_function<_Tp>, public __maybe_derive_from_binary_function<_Tp> {}; template -struct __weak_result_type - : public __weak_result_type_imp<_Tp> -{ -}; +struct __weak_result_type : public __weak_result_type_imp<_Tp> {}; // 0 argument case template -struct __weak_result_type<_Rp ()> -{ +struct __weak_result_type<_Rp()> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; template -struct __weak_result_type<_Rp (&)()> -{ +struct __weak_result_type<_Rp (&)()> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; template -struct __weak_result_type<_Rp (*)()> -{ +struct __weak_result_type<_Rp (*)()> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; // 1 argument case template -struct __weak_result_type<_Rp (_A1)> - : public __unary_function<_A1, _Rp> -{ -}; +struct __weak_result_type<_Rp(_A1)> : public __unary_function<_A1, _Rp> {}; template -struct __weak_result_type<_Rp (&)(_A1)> - : public __unary_function<_A1, _Rp> -{ -}; +struct __weak_result_type<_Rp (&)(_A1)> : public __unary_function<_A1, _Rp> {}; template -struct __weak_result_type<_Rp (*)(_A1)> - : public __unary_function<_A1, _Rp> -{ -}; +struct __weak_result_type<_Rp (*)(_A1)> : public __unary_function<_A1, _Rp> {}; template -struct __weak_result_type<_Rp (_Cp::*)()> - : public __unary_function<_Cp*, _Rp> -{ -}; +struct __weak_result_type<_Rp (_Cp::*)()> : public __unary_function<_Cp*, _Rp> {}; template -struct __weak_result_type<_Rp (_Cp::*)() const> - : public __unary_function -{ -}; +struct __weak_result_type<_Rp (_Cp::*)() const> : public __unary_function {}; template -struct __weak_result_type<_Rp (_Cp::*)() volatile> - : public __unary_function -{ -}; +struct __weak_result_type<_Rp (_Cp::*)() volatile> : public __unary_function {}; template -struct __weak_result_type<_Rp (_Cp::*)() const volatile> - : public __unary_function -{ -}; +struct __weak_result_type<_Rp (_Cp::*)() const volatile> : public __unary_function {}; // 2 argument case template -struct __weak_result_type<_Rp (_A1, _A2)> - : public __binary_function<_A1, _A2, _Rp> -{ -}; +struct __weak_result_type<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {}; template -struct __weak_result_type<_Rp (*)(_A1, _A2)> - : public __binary_function<_A1, _A2, _Rp> -{ -}; +struct __weak_result_type<_Rp (*)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {}; template -struct __weak_result_type<_Rp (&)(_A1, _A2)> - : public __binary_function<_A1, _A2, _Rp> -{ -}; +struct __weak_result_type<_Rp (&)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {}; template -struct __weak_result_type<_Rp (_Cp::*)(_A1)> - : public __binary_function<_Cp*, _A1, _Rp> -{ -}; +struct __weak_result_type<_Rp (_Cp::*)(_A1)> : public __binary_function<_Cp*, _A1, _Rp> {}; template -struct __weak_result_type<_Rp (_Cp::*)(_A1) const> - : public __binary_function -{ -}; +struct __weak_result_type<_Rp (_Cp::*)(_A1) const> : public __binary_function {}; template -struct __weak_result_type<_Rp (_Cp::*)(_A1) volatile> - : public __binary_function -{ -}; +struct __weak_result_type<_Rp (_Cp::*)(_A1) volatile> : public __binary_function {}; template -struct __weak_result_type<_Rp (_Cp::*)(_A1) const volatile> - : public __binary_function -{ +struct __weak_result_type<_Rp (_Cp::*)(_A1) const volatile> : public __binary_function { }; // 3 or more arguments -template -struct __weak_result_type<_Rp (_A1, _A2, _A3, _A4...)> -{ +template +struct __weak_result_type<_Rp(_A1, _A2, _A3, _A4...)> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (&)(_A1, _A2, _A3, _A4...)> -{ +template +struct __weak_result_type<_Rp (&)(_A1, _A2, _A3, _A4...)> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (*)(_A1, _A2, _A3, _A4...)> -{ +template +struct __weak_result_type<_Rp (*)(_A1, _A2, _A3, _A4...)> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...)> -{ +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...)> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const> -{ +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) volatile> -{ +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) volatile> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const volatile> -{ +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const volatile> { #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) - using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; + using result_type _LIBCPP_NODEBUG _LIBCPP_DEPRECATED_IN_CXX17 = _Rp; #endif }; -template -struct __invoke_return -{ - typedef decltype(std::__invoke(std::declval<_Tp>(), std::declval<_Args>()...)) type; +template +struct __invoke_return { + typedef decltype(std::__invoke(std::declval<_Tp>(), std::declval<_Args>()...)) type; }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__fwd/get.h b/libcxx/include/__fwd/get.h index 1ca7148e3fa2e..e7261b826953d 100644 --- a/libcxx/include/__fwd/get.h +++ b/libcxx/include/__fwd/get.h @@ -26,70 +26,54 @@ _LIBCPP_BEGIN_NAMESPACE_STD #ifndef _LIBCPP_CXX03_LANG -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -typename tuple_element<_Ip, tuple<_Tp...> >::type& +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type& get(tuple<_Tp...>&) _NOEXCEPT; -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const typename tuple_element<_Ip, tuple<_Tp...> >::type& +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type& get(const tuple<_Tp...>&) _NOEXCEPT; -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -typename tuple_element<_Ip, tuple<_Tp...> >::type&& +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&& get(tuple<_Tp...>&&) _NOEXCEPT; -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const typename tuple_element<_Ip, tuple<_Tp...> >::type&& +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&& get(const tuple<_Tp...>&&) _NOEXCEPT; #endif //_LIBCPP_CXX03_LANG template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -typename tuple_element<_Ip, pair<_T1, _T2> >::type& +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type& get(pair<_T1, _T2>&) _NOEXCEPT; template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const typename tuple_element<_Ip, pair<_T1, _T2> >::type& +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type& get(const pair<_T1, _T2>&) _NOEXCEPT; #ifndef _LIBCPP_CXX03_LANG template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -typename tuple_element<_Ip, pair<_T1, _T2> >::type&& +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&& get(pair<_T1, _T2>&&) _NOEXCEPT; template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const typename tuple_element<_Ip, pair<_T1, _T2> >::type&& +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&& get(const pair<_T1, _T2>&&) _NOEXCEPT; #endif template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp& -get(array<_Tp, _Size>&) _NOEXCEPT; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& get(array<_Tp, _Size>&) _NOEXCEPT; template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp& -get(const array<_Tp, _Size>&) _NOEXCEPT; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& get(const array<_Tp, _Size>&) _NOEXCEPT; #ifndef _LIBCPP_CXX03_LANG template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -_Tp&& -get(array<_Tp, _Size>&&) _NOEXCEPT; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&& get(array<_Tp, _Size>&&) _NOEXCEPT; template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -const _Tp&& -get(const array<_Tp, _Size>&&) _NOEXCEPT; +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&& get(const array<_Tp, _Size>&&) _NOEXCEPT; #endif #if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__fwd/span.h b/libcxx/include/__fwd/span.h index e9fa70382f590..8dafa742c19df 100644 --- a/libcxx/include/__fwd/span.h +++ b/libcxx/include/__fwd/span.h @@ -26,7 +26,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 inline constexpr size_t dynamic_extent = numeric_limits::max(); -template class span; +template +class span; #endif diff --git a/libcxx/include/__fwd/string_view.h b/libcxx/include/__fwd/string_view.h index 786765ca6a2ed..72a64be5b00b5 100644 --- a/libcxx/include/__fwd/string_view.h +++ b/libcxx/include/__fwd/string_view.h @@ -19,17 +19,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template > +template > class _LIBCPP_TEMPLATE_VIS basic_string_view; -typedef basic_string_view string_view; +typedef basic_string_view string_view; #ifndef _LIBCPP_HAS_NO_CHAR8_T -typedef basic_string_view u8string_view; +typedef basic_string_view u8string_view; #endif typedef basic_string_view u16string_view; typedef basic_string_view u32string_view; #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS -typedef basic_string_view wstring_view; +typedef basic_string_view wstring_view; #endif // clang-format off diff --git a/libcxx/include/__fwd/subrange.h b/libcxx/include/__fwd/subrange.h index 24db670575f4e..d09b9b1c5b97b 100644 --- a/libcxx/include/__fwd/subrange.h +++ b/libcxx/include/__fwd/subrange.h @@ -17,7 +17,7 @@ #if _LIBCPP_STD_VER >= 20 -#include <__iterator/concepts.h> +# include <__iterator/concepts.h> _LIBCPP_BEGIN_NAMESPACE_STD diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table index fa950ac7e9b78..3cee48ef8538c 100644 --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -55,7 +55,6 @@ _LIBCPP_PUSH_MACROS #include <__undef_macros> - _LIBCPP_BEGIN_NAMESPACE_STD template @@ -67,7 +66,7 @@ struct __is_hash_value_type_imp : false_type {}; template struct __is_hash_value_type_imp<__hash_value_type<_Key, _Value> > : true_type {}; -template +template struct __is_hash_value_type : false_type {}; template @@ -76,110 +75,91 @@ struct __is_hash_value_type<_One> : __is_hash_value_type_imp<__remove_cvref_t<_O _LIBCPP_EXPORTED_FROM_ABI size_t __next_prime(size_t __n); template -struct __hash_node_base -{ - typedef typename pointer_traits<_NodePtr>::element_type __node_type; - typedef __hash_node_base __first_node; - typedef __rebind_pointer_t<_NodePtr, __first_node> __node_base_pointer; - typedef _NodePtr __node_pointer; +struct __hash_node_base { + typedef typename pointer_traits<_NodePtr>::element_type __node_type; + typedef __hash_node_base __first_node; + typedef __rebind_pointer_t<_NodePtr, __first_node> __node_base_pointer; + typedef _NodePtr __node_pointer; #if defined(_LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB) typedef __node_base_pointer __next_pointer; #else - typedef __conditional_t::value, __node_base_pointer, __node_pointer> __next_pointer; + typedef __conditional_t::value, __node_base_pointer, __node_pointer> __next_pointer; #endif - __next_pointer __next_; + __next_pointer __next_; - _LIBCPP_HIDE_FROM_ABI - __next_pointer __ptr() _NOEXCEPT { - return static_cast<__next_pointer>( - pointer_traits<__node_base_pointer>::pointer_to(*this)); - } + _LIBCPP_HIDE_FROM_ABI __next_pointer __ptr() _NOEXCEPT { + return static_cast<__next_pointer>(pointer_traits<__node_base_pointer>::pointer_to(*this)); + } - _LIBCPP_HIDE_FROM_ABI - __node_pointer __upcast() _NOEXCEPT { - return static_cast<__node_pointer>( - pointer_traits<__node_base_pointer>::pointer_to(*this)); - } + _LIBCPP_HIDE_FROM_ABI __node_pointer __upcast() _NOEXCEPT { + return static_cast<__node_pointer>(pointer_traits<__node_base_pointer>::pointer_to(*this)); + } - _LIBCPP_HIDE_FROM_ABI - size_t __hash() const _NOEXCEPT { - return static_cast<__node_type const&>(*this).__hash_; - } + _LIBCPP_HIDE_FROM_ABI size_t __hash() const _NOEXCEPT { return static_cast<__node_type const&>(*this).__hash_; } - _LIBCPP_HIDE_FROM_ABI __hash_node_base() _NOEXCEPT : __next_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {} + _LIBCPP_HIDE_FROM_ABI __hash_node_base() _NOEXCEPT : __next_(nullptr) {} + _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {} }; template -struct __hash_node - : public __hash_node_base - < - __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > - > -{ - typedef _Tp __node_value_type; - using _Base = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >; - using __next_pointer = typename _Base::__next_pointer; +struct __hash_node : public __hash_node_base< __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > > { + typedef _Tp __node_value_type; + using _Base = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >; + using __next_pointer = typename _Base::__next_pointer; - size_t __hash_; + size_t __hash_; - // We allow starting the lifetime of nodes without initializing the value held by the node, - // since that is handled by the hash table itself in order to be allocator-aware. + // We allow starting the lifetime of nodes without initializing the value held by the node, + // since that is handled by the hash table itself in order to be allocator-aware. #ifndef _LIBCPP_CXX03_LANG + private: - union { - _Tp __value_; - }; + union { + _Tp __value_; + }; public: - _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; } + _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; } #else + private: - _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)]; + _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)]; public: - _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { - return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); - } + _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); } #endif - _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {} - _LIBCPP_HIDE_FROM_ABI ~__hash_node() {} + _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {} + _LIBCPP_HIDE_FROM_ABI ~__hash_node() {} }; -inline _LIBCPP_HIDE_FROM_ABI -bool -__is_hash_power2(size_t __bc) -{ - return __bc > 2 && !(__bc & (__bc - 1)); -} +inline _LIBCPP_HIDE_FROM_ABI bool __is_hash_power2(size_t __bc) { return __bc > 2 && !(__bc & (__bc - 1)); } -inline _LIBCPP_HIDE_FROM_ABI -size_t -__constrain_hash(size_t __h, size_t __bc) -{ - return !(__bc & (__bc - 1)) ? __h & (__bc - 1) : - (__h < __bc ? __h : __h % __bc); +inline _LIBCPP_HIDE_FROM_ABI size_t __constrain_hash(size_t __h, size_t __bc) { + return !(__bc & (__bc - 1)) ? __h & (__bc - 1) : (__h < __bc ? __h : __h % __bc); } -inline _LIBCPP_HIDE_FROM_ABI -size_t -__next_hash_pow2(size_t __n) -{ - return __n < 2 ? __n : (size_t(1) << (numeric_limits::digits - __libcpp_clz(__n-1))); +inline _LIBCPP_HIDE_FROM_ABI size_t __next_hash_pow2(size_t __n) { + return __n < 2 ? __n : (size_t(1) << (numeric_limits::digits - __libcpp_clz(__n - 1))); } +template +class __hash_table; -template class __hash_table; - -template class _LIBCPP_TEMPLATE_VIS __hash_iterator; -template class _LIBCPP_TEMPLATE_VIS __hash_const_iterator; -template class _LIBCPP_TEMPLATE_VIS __hash_local_iterator; -template class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator; -template class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; -template class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_const_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_local_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; +template +class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; template struct __hash_key_value_types { @@ -189,73 +169,47 @@ struct __hash_key_value_types { typedef _Tp __container_value_type; static const bool __is_map = false; - _LIBCPP_HIDE_FROM_ABI - static key_type const& __get_key(_Tp const& __v) { - return __v; - } - _LIBCPP_HIDE_FROM_ABI - static __container_value_type const& __get_value(__node_value_type const& __v) { - return __v; - } - _LIBCPP_HIDE_FROM_ABI - static __container_value_type* __get_ptr(__node_value_type& __n) { - return std::addressof(__n); - } - _LIBCPP_HIDE_FROM_ABI - static __container_value_type&& __move(__node_value_type& __v) { - return std::move(__v); - } + _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(_Tp const& __v) { return __v; } + _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(__node_value_type const& __v) { return __v; } + _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) { return std::addressof(__n); } + _LIBCPP_HIDE_FROM_ABI static __container_value_type&& __move(__node_value_type& __v) { return std::move(__v); } }; template struct __hash_key_value_types<__hash_value_type<_Key, _Tp> > { - typedef _Key key_type; - typedef _Tp mapped_type; - typedef __hash_value_type<_Key, _Tp> __node_value_type; - typedef pair __container_value_type; - typedef __container_value_type __map_value_type; + typedef _Key key_type; + typedef _Tp mapped_type; + typedef __hash_value_type<_Key, _Tp> __node_value_type; + typedef pair __container_value_type; + typedef __container_value_type __map_value_type; static const bool __is_map = true; - _LIBCPP_HIDE_FROM_ABI - static key_type const& __get_key(__container_value_type const& __v) { - return __v.first; - } + _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__container_value_type const& __v) { return __v.first; } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static __container_value_type const& - __get_value(_Up& __t) { + _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) { return __t.__get_value(); } template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - static __container_value_type const& - __get_value(_Up& __t) { + _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) { return __t; } - _LIBCPP_HIDE_FROM_ABI - static __container_value_type* __get_ptr(__node_value_type& __n) { + _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) { return std::addressof(__n.__get_value()); } - _LIBCPP_HIDE_FROM_ABI - static pair __move(__node_value_type& __v) { - return __v.__move(); - } + _LIBCPP_HIDE_FROM_ABI static pair __move(__node_value_type& __v) { return __v.__move(); } }; -template , - bool = _KVTypes::__is_map> +template , bool = _KVTypes::__is_map> struct __hash_map_pointer_types {}; template struct __hash_map_pointer_types<_Tp, _AllocPtr, _KVTypes, true> { - typedef typename _KVTypes::__map_value_type _Mv; - typedef __rebind_pointer_t<_AllocPtr, _Mv> - __map_value_type_pointer; - typedef __rebind_pointer_t<_AllocPtr, const _Mv> - __const_map_value_type_pointer; + typedef typename _KVTypes::__map_value_type _Mv; + typedef __rebind_pointer_t<_AllocPtr, _Mv> __map_value_type_pointer; + typedef __rebind_pointer_t<_AllocPtr, const _Mv> __const_map_value_type_pointer; }; template ::element_type> @@ -263,39 +217,36 @@ struct __hash_node_types; template struct __hash_node_types<_NodePtr, __hash_node<_Tp, _VoidPtr> > - : public __hash_key_value_types<_Tp>, __hash_map_pointer_types<_Tp, _VoidPtr> + : public __hash_key_value_types<_Tp>, + __hash_map_pointer_types<_Tp, _VoidPtr> { - typedef __hash_key_value_types<_Tp> __base; + typedef __hash_key_value_types<_Tp> __base; public: typedef ptrdiff_t difference_type; typedef size_t size_type; - typedef __rebind_pointer_t<_NodePtr, void> __void_pointer; + typedef __rebind_pointer_t<_NodePtr, void> __void_pointer; - typedef typename pointer_traits<_NodePtr>::element_type __node_type; - typedef _NodePtr __node_pointer; + typedef typename pointer_traits<_NodePtr>::element_type __node_type; + typedef _NodePtr __node_pointer; - typedef __hash_node_base<__node_pointer> __node_base_type; - typedef __rebind_pointer_t<_NodePtr, __node_base_type> - __node_base_pointer; + typedef __hash_node_base<__node_pointer> __node_base_type; + typedef __rebind_pointer_t<_NodePtr, __node_base_type> __node_base_pointer; - typedef typename __node_base_type::__next_pointer __next_pointer; + typedef typename __node_base_type::__next_pointer __next_pointer; - typedef _Tp __node_value_type; - typedef __rebind_pointer_t<_VoidPtr, __node_value_type> - __node_value_type_pointer; - typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> - __const_node_value_type_pointer; + typedef _Tp __node_value_type; + typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer; + typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> __const_node_value_type_pointer; private: - static_assert(!is_const<__node_type>::value, - "_NodePtr should never be a pointer to const"); - static_assert((is_same::element_type, void>::value), - "_VoidPtr does not point to unqualified void type"); - static_assert((is_same<__rebind_pointer_t<_VoidPtr, __node_type>, - _NodePtr>::value), "_VoidPtr does not rebind to _NodePtr."); + static_assert(!is_const<__node_type>::value, "_NodePtr should never be a pointer to const"); + static_assert((is_same::element_type, void>::value), + "_VoidPtr does not point to unqualified void type"); + static_assert((is_same<__rebind_pointer_t<_VoidPtr, __node_type>, _NodePtr>::value), + "_VoidPtr does not rebind to _NodePtr."); }; template @@ -309,7 +260,6 @@ struct __hash_node_types_from_iterator<__hash_local_iterator<_NodePtr> > : __has template struct __hash_node_types_from_iterator<__hash_const_local_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {}; - template struct __make_hash_node_types { typedef __hash_node<_NodeValueTp, _VoidPtr> _NodeTp; @@ -318,394 +268,327 @@ struct __make_hash_node_types { }; template -class _LIBCPP_TEMPLATE_VIS __hash_iterator -{ - typedef __hash_node_types<_NodePtr> _NodeTypes; - typedef _NodePtr __node_pointer; - typedef typename _NodeTypes::__next_pointer __next_pointer; +class _LIBCPP_TEMPLATE_VIS __hash_iterator { + typedef __hash_node_types<_NodePtr> _NodeTypes; + typedef _NodePtr __node_pointer; + typedef typename _NodeTypes::__next_pointer __next_pointer; - __next_pointer __node_; + __next_pointer __node_; public: - typedef forward_iterator_tag iterator_category; - typedef typename _NodeTypes::__node_value_type value_type; - typedef typename _NodeTypes::difference_type difference_type; - typedef value_type& reference; - typedef typename _NodeTypes::__node_value_type_pointer pointer; + typedef forward_iterator_tag iterator_category; + typedef typename _NodeTypes::__node_value_type value_type; + typedef typename _NodeTypes::difference_type difference_type; + typedef value_type& reference; + typedef typename _NodeTypes::__node_value_type_pointer pointer; - _LIBCPP_HIDE_FROM_ABI __hash_iterator() _NOEXCEPT : __node_(nullptr) { - } + _LIBCPP_HIDE_FROM_ABI __hash_iterator() _NOEXCEPT : __node_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI - reference operator*() const { - return __node_->__upcast()->__get_value(); - } + _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __node_->__upcast()->__get_value(); } - _LIBCPP_HIDE_FROM_ABI - pointer operator->() const { - return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); - } + _LIBCPP_HIDE_FROM_ABI pointer operator->() const { + return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); + } - _LIBCPP_HIDE_FROM_ABI - __hash_iterator& operator++() { - __node_ = __node_->__next_; - return *this; - } + _LIBCPP_HIDE_FROM_ABI __hash_iterator& operator++() { + __node_ = __node_->__next_; + return *this; + } - _LIBCPP_HIDE_FROM_ABI - __hash_iterator operator++(int) - { - __hash_iterator __t(*this); - ++(*this); - return __t; - } + _LIBCPP_HIDE_FROM_ABI __hash_iterator operator++(int) { + __hash_iterator __t(*this); + ++(*this); + return __t; + } - friend _LIBCPP_HIDE_FROM_ABI - bool operator==(const __hash_iterator& __x, const __hash_iterator& __y) - { - return __x.__node_ == __y.__node_; - } - friend _LIBCPP_HIDE_FROM_ABI - bool operator!=(const __hash_iterator& __x, const __hash_iterator& __y) - {return !(__x == __y);} + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_iterator& __x, const __hash_iterator& __y) { + return __x.__node_ == __y.__node_; + } + friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_iterator& __x, const __hash_iterator& __y) { + return !(__x == __y); + } private: - _LIBCPP_HIDE_FROM_ABI - explicit __hash_iterator(__next_pointer __node) _NOEXCEPT - : __node_(__node) - { - } - - template friend class __hash_table; - template friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator; - template friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; - template friend class _LIBCPP_TEMPLATE_VIS unordered_map; - template friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; + _LIBCPP_HIDE_FROM_ABI explicit __hash_iterator(__next_pointer __node) _NOEXCEPT : __node_(__node) {} + + template + friend class __hash_table; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_const_iterator; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_map; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; }; template -class _LIBCPP_TEMPLATE_VIS __hash_const_iterator -{ - static_assert(!is_const::element_type>::value, ""); - typedef __hash_node_types<_NodePtr> _NodeTypes; - typedef _NodePtr __node_pointer; - typedef typename _NodeTypes::__next_pointer __next_pointer; +class _LIBCPP_TEMPLATE_VIS __hash_const_iterator { + static_assert(!is_const::element_type>::value, ""); + typedef __hash_node_types<_NodePtr> _NodeTypes; + typedef _NodePtr __node_pointer; + typedef typename _NodeTypes::__next_pointer __next_pointer; - __next_pointer __node_; + __next_pointer __node_; public: - typedef __hash_iterator<_NodePtr> __non_const_iterator; - - typedef forward_iterator_tag iterator_category; - typedef typename _NodeTypes::__node_value_type value_type; - typedef typename _NodeTypes::difference_type difference_type; - typedef const value_type& reference; - typedef typename _NodeTypes::__const_node_value_type_pointer pointer; + typedef __hash_iterator<_NodePtr> __non_const_iterator; + typedef forward_iterator_tag iterator_category; + typedef typename _NodeTypes::__node_value_type value_type; + typedef typename _NodeTypes::difference_type difference_type; + typedef const value_type& reference; + typedef typename _NodeTypes::__const_node_value_type_pointer pointer; - _LIBCPP_HIDE_FROM_ABI __hash_const_iterator() _NOEXCEPT : __node_(nullptr) { - } + _LIBCPP_HIDE_FROM_ABI __hash_const_iterator() _NOEXCEPT : __node_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI - __hash_const_iterator(const __non_const_iterator& __x) _NOEXCEPT - : __node_(__x.__node_) - { - } + _LIBCPP_HIDE_FROM_ABI __hash_const_iterator(const __non_const_iterator& __x) _NOEXCEPT : __node_(__x.__node_) {} - _LIBCPP_HIDE_FROM_ABI - reference operator*() const { - return __node_->__upcast()->__get_value(); - } - _LIBCPP_HIDE_FROM_ABI - pointer operator->() const { - return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); - } + _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __node_->__upcast()->__get_value(); } + _LIBCPP_HIDE_FROM_ABI pointer operator->() const { + return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); + } - _LIBCPP_HIDE_FROM_ABI - __hash_const_iterator& operator++() { - __node_ = __node_->__next_; - return *this; - } + _LIBCPP_HIDE_FROM_ABI __hash_const_iterator& operator++() { + __node_ = __node_->__next_; + return *this; + } - _LIBCPP_HIDE_FROM_ABI - __hash_const_iterator operator++(int) - { - __hash_const_iterator __t(*this); - ++(*this); - return __t; - } + _LIBCPP_HIDE_FROM_ABI __hash_const_iterator operator++(int) { + __hash_const_iterator __t(*this); + ++(*this); + return __t; + } - friend _LIBCPP_HIDE_FROM_ABI - bool operator==(const __hash_const_iterator& __x, const __hash_const_iterator& __y) - { - return __x.__node_ == __y.__node_; - } - friend _LIBCPP_HIDE_FROM_ABI - bool operator!=(const __hash_const_iterator& __x, const __hash_const_iterator& __y) - {return !(__x == __y);} + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_const_iterator& __x, const __hash_const_iterator& __y) { + return __x.__node_ == __y.__node_; + } + friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_const_iterator& __x, const __hash_const_iterator& __y) { + return !(__x == __y); + } private: - _LIBCPP_HIDE_FROM_ABI - explicit __hash_const_iterator(__next_pointer __node) _NOEXCEPT - : __node_(__node) - { - } - - template friend class __hash_table; - template friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; - template friend class _LIBCPP_TEMPLATE_VIS unordered_map; - template friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; + _LIBCPP_HIDE_FROM_ABI explicit __hash_const_iterator(__next_pointer __node) _NOEXCEPT : __node_(__node) {} + + template + friend class __hash_table; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_map; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; }; template -class _LIBCPP_TEMPLATE_VIS __hash_local_iterator -{ - typedef __hash_node_types<_NodePtr> _NodeTypes; - typedef _NodePtr __node_pointer; - typedef typename _NodeTypes::__next_pointer __next_pointer; +class _LIBCPP_TEMPLATE_VIS __hash_local_iterator { + typedef __hash_node_types<_NodePtr> _NodeTypes; + typedef _NodePtr __node_pointer; + typedef typename _NodeTypes::__next_pointer __next_pointer; - __next_pointer __node_; - size_t __bucket_; - size_t __bucket_count_; + __next_pointer __node_; + size_t __bucket_; + size_t __bucket_count_; public: - typedef forward_iterator_tag iterator_category; - typedef typename _NodeTypes::__node_value_type value_type; - typedef typename _NodeTypes::difference_type difference_type; - typedef value_type& reference; - typedef typename _NodeTypes::__node_value_type_pointer pointer; + typedef forward_iterator_tag iterator_category; + typedef typename _NodeTypes::__node_value_type value_type; + typedef typename _NodeTypes::difference_type difference_type; + typedef value_type& reference; + typedef typename _NodeTypes::__node_value_type_pointer pointer; - _LIBCPP_HIDE_FROM_ABI __hash_local_iterator() _NOEXCEPT : __node_(nullptr) { - } + _LIBCPP_HIDE_FROM_ABI __hash_local_iterator() _NOEXCEPT : __node_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI - reference operator*() const { - return __node_->__upcast()->__get_value(); - } + _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __node_->__upcast()->__get_value(); } - _LIBCPP_HIDE_FROM_ABI - pointer operator->() const { - return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); - } + _LIBCPP_HIDE_FROM_ABI pointer operator->() const { + return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); + } - _LIBCPP_HIDE_FROM_ABI - __hash_local_iterator& operator++() { - __node_ = __node_->__next_; - if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_) - __node_ = nullptr; - return *this; - } + _LIBCPP_HIDE_FROM_ABI __hash_local_iterator& operator++() { + __node_ = __node_->__next_; + if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_) + __node_ = nullptr; + return *this; + } - _LIBCPP_HIDE_FROM_ABI - __hash_local_iterator operator++(int) - { - __hash_local_iterator __t(*this); - ++(*this); - return __t; - } + _LIBCPP_HIDE_FROM_ABI __hash_local_iterator operator++(int) { + __hash_local_iterator __t(*this); + ++(*this); + return __t; + } - friend _LIBCPP_HIDE_FROM_ABI - bool operator==(const __hash_local_iterator& __x, const __hash_local_iterator& __y) - { - return __x.__node_ == __y.__node_; - } - friend _LIBCPP_HIDE_FROM_ABI - bool operator!=(const __hash_local_iterator& __x, const __hash_local_iterator& __y) - {return !(__x == __y);} + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __hash_local_iterator& __x, const __hash_local_iterator& __y) { + return __x.__node_ == __y.__node_; + } + friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __hash_local_iterator& __x, const __hash_local_iterator& __y) { + return !(__x == __y); + } private: - _LIBCPP_HIDE_FROM_ABI - explicit __hash_local_iterator(__next_pointer __node, size_t __bucket, - size_t __bucket_count) _NOEXCEPT - : __node_(__node), - __bucket_(__bucket), - __bucket_count_(__bucket_count) - { - if (__node_ != nullptr) - __node_ = __node_->__next_; - } + _LIBCPP_HIDE_FROM_ABI explicit __hash_local_iterator( + __next_pointer __node, size_t __bucket, size_t __bucket_count) _NOEXCEPT + : __node_(__node), + __bucket_(__bucket), + __bucket_count_(__bucket_count) { + if (__node_ != nullptr) + __node_ = __node_->__next_; + } - template friend class __hash_table; - template friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator; - template friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; + template + friend class __hash_table; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_map_iterator; }; template -class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator -{ - typedef __hash_node_types<_ConstNodePtr> _NodeTypes; - typedef _ConstNodePtr __node_pointer; - typedef typename _NodeTypes::__next_pointer __next_pointer; - - __next_pointer __node_; - size_t __bucket_; - size_t __bucket_count_; - - typedef pointer_traits<__node_pointer> __pointer_traits; - typedef typename __pointer_traits::element_type __node; - typedef __remove_const_t<__node> __non_const_node; - typedef __rebind_pointer_t<__node_pointer, __non_const_node> - __non_const_node_pointer; -public: - typedef __hash_local_iterator<__non_const_node_pointer> - __non_const_iterator; +class _LIBCPP_TEMPLATE_VIS __hash_const_local_iterator { + typedef __hash_node_types<_ConstNodePtr> _NodeTypes; + typedef _ConstNodePtr __node_pointer; + typedef typename _NodeTypes::__next_pointer __next_pointer; - typedef forward_iterator_tag iterator_category; - typedef typename _NodeTypes::__node_value_type value_type; - typedef typename _NodeTypes::difference_type difference_type; - typedef const value_type& reference; - typedef typename _NodeTypes::__const_node_value_type_pointer pointer; + __next_pointer __node_; + size_t __bucket_; + size_t __bucket_count_; + typedef pointer_traits<__node_pointer> __pointer_traits; + typedef typename __pointer_traits::element_type __node; + typedef __remove_const_t<__node> __non_const_node; + typedef __rebind_pointer_t<__node_pointer, __non_const_node> __non_const_node_pointer; - _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator() _NOEXCEPT : __node_(nullptr) { - } +public: + typedef __hash_local_iterator<__non_const_node_pointer> __non_const_iterator; - _LIBCPP_HIDE_FROM_ABI - __hash_const_local_iterator(const __non_const_iterator& __x) _NOEXCEPT - : __node_(__x.__node_), - __bucket_(__x.__bucket_), - __bucket_count_(__x.__bucket_count_) - { - } + typedef forward_iterator_tag iterator_category; + typedef typename _NodeTypes::__node_value_type value_type; + typedef typename _NodeTypes::difference_type difference_type; + typedef const value_type& reference; + typedef typename _NodeTypes::__const_node_value_type_pointer pointer; - _LIBCPP_HIDE_FROM_ABI - reference operator*() const { - return __node_->__upcast()->__get_value(); - } + _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator() _NOEXCEPT : __node_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI - pointer operator->() const { - return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); - } + _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator(const __non_const_iterator& __x) _NOEXCEPT + : __node_(__x.__node_), + __bucket_(__x.__bucket_), + __bucket_count_(__x.__bucket_count_) {} - _LIBCPP_HIDE_FROM_ABI - __hash_const_local_iterator& operator++() { - __node_ = __node_->__next_; - if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_) - __node_ = nullptr; - return *this; - } + _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __node_->__upcast()->__get_value(); } - _LIBCPP_HIDE_FROM_ABI - __hash_const_local_iterator operator++(int) - { - __hash_const_local_iterator __t(*this); - ++(*this); - return __t; - } + _LIBCPP_HIDE_FROM_ABI pointer operator->() const { + return pointer_traits::pointer_to(__node_->__upcast()->__get_value()); + } - friend _LIBCPP_HIDE_FROM_ABI - bool operator==(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) - { - return __x.__node_ == __y.__node_; - } - friend _LIBCPP_HIDE_FROM_ABI - bool operator!=(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) - {return !(__x == __y);} + _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator& operator++() { + __node_ = __node_->__next_; + if (__node_ != nullptr && std::__constrain_hash(__node_->__hash(), __bucket_count_) != __bucket_) + __node_ = nullptr; + return *this; + } + + _LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator operator++(int) { + __hash_const_local_iterator __t(*this); + ++(*this); + return __t; + } + + friend _LIBCPP_HIDE_FROM_ABI bool + operator==(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) { + return __x.__node_ == __y.__node_; + } + friend _LIBCPP_HIDE_FROM_ABI bool + operator!=(const __hash_const_local_iterator& __x, const __hash_const_local_iterator& __y) { + return !(__x == __y); + } private: - _LIBCPP_HIDE_FROM_ABI - explicit __hash_const_local_iterator(__next_pointer __node_ptr, size_t __bucket, - size_t __bucket_count) _NOEXCEPT - : __node_(__node_ptr), - __bucket_(__bucket), - __bucket_count_(__bucket_count) - { - if (__node_ != nullptr) - __node_ = __node_->__next_; - } + _LIBCPP_HIDE_FROM_ABI explicit __hash_const_local_iterator( + __next_pointer __node_ptr, size_t __bucket, size_t __bucket_count) _NOEXCEPT + : __node_(__node_ptr), + __bucket_(__bucket), + __bucket_count_(__bucket_count) { + if (__node_ != nullptr) + __node_ = __node_->__next_; + } - template friend class __hash_table; - template friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; + template + friend class __hash_table; + template + friend class _LIBCPP_TEMPLATE_VIS __hash_map_const_iterator; }; template -class __bucket_list_deallocator -{ - typedef _Alloc allocator_type; - typedef allocator_traits __alloc_traits; - typedef typename __alloc_traits::size_type size_type; +class __bucket_list_deallocator { + typedef _Alloc allocator_type; + typedef allocator_traits __alloc_traits; + typedef typename __alloc_traits::size_type size_type; + + __compressed_pair __data_; - __compressed_pair __data_; public: - typedef typename __alloc_traits::pointer pointer; - - _LIBCPP_HIDE_FROM_ABI - __bucket_list_deallocator() - _NOEXCEPT_(is_nothrow_default_constructible::value) - : __data_(0, __default_init_tag()) {} - - _LIBCPP_HIDE_FROM_ABI - __bucket_list_deallocator(const allocator_type& __a, size_type __size) - _NOEXCEPT_(is_nothrow_copy_constructible::value) - : __data_(__size, __a) {} - - _LIBCPP_HIDE_FROM_ABI - __bucket_list_deallocator(__bucket_list_deallocator&& __x) - _NOEXCEPT_(is_nothrow_move_constructible::value) - : __data_(std::move(__x.__data_)) - { - __x.size() = 0; - } + typedef typename __alloc_traits::pointer pointer; - _LIBCPP_HIDE_FROM_ABI - size_type& size() _NOEXCEPT {return __data_.first();} - _LIBCPP_HIDE_FROM_ABI - size_type size() const _NOEXCEPT {return __data_.first();} + _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator() _NOEXCEPT_(is_nothrow_default_constructible::value) + : __data_(0, __default_init_tag()) {} - _LIBCPP_HIDE_FROM_ABI - allocator_type& __alloc() _NOEXCEPT {return __data_.second();} - _LIBCPP_HIDE_FROM_ABI - const allocator_type& __alloc() const _NOEXCEPT {return __data_.second();} + _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator(const allocator_type& __a, size_type __size) + _NOEXCEPT_(is_nothrow_copy_constructible::value) + : __data_(__size, __a) {} - _LIBCPP_HIDE_FROM_ABI - void operator()(pointer __p) _NOEXCEPT - { - __alloc_traits::deallocate(__alloc(), __p, size()); - } + _LIBCPP_HIDE_FROM_ABI __bucket_list_deallocator(__bucket_list_deallocator&& __x) + _NOEXCEPT_(is_nothrow_move_constructible::value) + : __data_(std::move(__x.__data_)) { + __x.size() = 0; + } + + _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __data_.first(); } + _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __data_.first(); } + + _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { return __data_.second(); } + _LIBCPP_HIDE_FROM_ABI const allocator_type& __alloc() const _NOEXCEPT { return __data_.second(); } + + _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT { __alloc_traits::deallocate(__alloc(), __p, size()); } }; -template class __hash_map_node_destructor; +template +class __hash_map_node_destructor; template -class __hash_node_destructor -{ - typedef _Alloc allocator_type; - typedef allocator_traits __alloc_traits; +class __hash_node_destructor { + typedef _Alloc allocator_type; + typedef allocator_traits __alloc_traits; public: - typedef typename __alloc_traits::pointer pointer; + typedef typename __alloc_traits::pointer pointer; + private: - typedef __hash_node_types _NodeTypes; + typedef __hash_node_types _NodeTypes; - allocator_type& __na_; + allocator_type& __na_; public: - bool __value_constructed; - - _LIBCPP_HIDE_FROM_ABI __hash_node_destructor(__hash_node_destructor const&) = default; - _LIBCPP_HIDE_FROM_ABI __hash_node_destructor& operator=(const __hash_node_destructor&) = delete; + bool __value_constructed; + _LIBCPP_HIDE_FROM_ABI __hash_node_destructor(__hash_node_destructor const&) = default; + _LIBCPP_HIDE_FROM_ABI __hash_node_destructor& operator=(const __hash_node_destructor&) = delete; - _LIBCPP_HIDE_FROM_ABI - explicit __hash_node_destructor(allocator_type& __na, - bool __constructed = false) _NOEXCEPT - : __na_(__na), - __value_constructed(__constructed) - {} + _LIBCPP_HIDE_FROM_ABI explicit __hash_node_destructor(allocator_type& __na, bool __constructed = false) _NOEXCEPT + : __na_(__na), + __value_constructed(__constructed) {} - _LIBCPP_HIDE_FROM_ABI - void operator()(pointer __p) _NOEXCEPT - { - if (__value_constructed) { - __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__get_value())); - std::__destroy_at(std::addressof(*__p)); - } - if (__p) - __alloc_traits::deallocate(__na_, __p, 1); + _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT { + if (__value_constructed) { + __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__get_value())); + std::__destroy_at(std::addressof(*__p)); } + if (__p) + __alloc_traits::deallocate(__na_, __p, 1); + } - template friend class __hash_map_node_destructor; + template + friend class __hash_map_node_destructor; }; #if _LIBCPP_STD_VER >= 17 @@ -713,33 +596,30 @@ template struct __generic_container_node_destructor; template -struct __generic_container_node_destructor<__hash_node<_Tp, _VoidPtr>, _Alloc> - : __hash_node_destructor<_Alloc> -{ - using __hash_node_destructor<_Alloc>::__hash_node_destructor; +struct __generic_container_node_destructor<__hash_node<_Tp, _VoidPtr>, _Alloc> : __hash_node_destructor<_Alloc> { + using __hash_node_destructor<_Alloc>::__hash_node_destructor; }; #endif template struct __enforce_unordered_container_requirements { #ifndef _LIBCPP_CXX03_LANG - static_assert(__check_hash_requirements<_Key, _Hash>::value, - "the specified hash does not meet the Hash requirements"); - static_assert(is_copy_constructible<_Equal>::value, - "the specified comparator is required to be copy constructible"); + static_assert(__check_hash_requirements<_Key, _Hash>::value, + "the specified hash does not meet the Hash requirements"); + static_assert(is_copy_constructible<_Equal>::value, "the specified comparator is required to be copy constructible"); #endif - typedef int type; + typedef int type; }; template #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Equal const&, _Key const&, _Key const&>::value, - "the specified comparator type does not provide a viable const call operator") - _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Hash const&, _Key const&>::value, - "the specified hash functor does not provide a viable const call operator") +_LIBCPP_DIAGNOSE_WARNING(!__invokable<_Equal const&, _Key const&, _Key const&>::value, + "the specified comparator type does not provide a viable const call operator") +_LIBCPP_DIAGNOSE_WARNING(!__invokable<_Hash const&, _Key const&>::value, + "the specified hash functor does not provide a viable const call operator") #endif -typename __enforce_unordered_container_requirements<_Key, _Hash, _Equal>::type -__diagnose_unordered_container_requirements(int); + typename __enforce_unordered_container_requirements<_Key, _Hash, _Equal>::type + __diagnose_unordered_container_requirements(int); // This dummy overload is used so that the compiler won't emit a spurious // "no matching function for call to __diagnose_unordered_xxx" diagnostic @@ -748,890 +628,668 @@ template int __diagnose_unordered_container_requirements(void*); template -class __hash_table -{ +class __hash_table { public: - typedef _Tp value_type; - typedef _Hash hasher; - typedef _Equal key_equal; - typedef _Alloc allocator_type; + typedef _Tp value_type; + typedef _Hash hasher; + typedef _Equal key_equal; + typedef _Alloc allocator_type; private: - typedef allocator_traits __alloc_traits; - typedef typename - __make_hash_node_types::type - _NodeTypes; -public: + typedef allocator_traits __alloc_traits; + typedef typename __make_hash_node_types::type _NodeTypes; - typedef typename _NodeTypes::__node_value_type __node_value_type; - typedef typename _NodeTypes::__container_value_type __container_value_type; - typedef typename _NodeTypes::key_type key_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef typename __alloc_traits::pointer pointer; - typedef typename __alloc_traits::const_pointer const_pointer; +public: + typedef typename _NodeTypes::__node_value_type __node_value_type; + typedef typename _NodeTypes::__container_value_type __container_value_type; + typedef typename _NodeTypes::key_type key_type; + typedef value_type& reference; + typedef const value_type& const_reference; + typedef typename __alloc_traits::pointer pointer; + typedef typename __alloc_traits::const_pointer const_pointer; #ifndef _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE - typedef typename __alloc_traits::size_type size_type; + typedef typename __alloc_traits::size_type size_type; #else - typedef typename _NodeTypes::size_type size_type; + typedef typename _NodeTypes::size_type size_type; #endif - typedef typename _NodeTypes::difference_type difference_type; + typedef typename _NodeTypes::difference_type difference_type; + public: - // Create __node - - typedef typename _NodeTypes::__node_type __node; - typedef __rebind_alloc<__alloc_traits, __node> __node_allocator; - typedef allocator_traits<__node_allocator> __node_traits; - typedef typename _NodeTypes::__void_pointer __void_pointer; - typedef typename _NodeTypes::__node_pointer __node_pointer; - typedef typename _NodeTypes::__node_pointer __node_const_pointer; - typedef typename _NodeTypes::__node_base_type __first_node; - typedef typename _NodeTypes::__node_base_pointer __node_base_pointer; - typedef typename _NodeTypes::__next_pointer __next_pointer; + // Create __node + + typedef typename _NodeTypes::__node_type __node; + typedef __rebind_alloc<__alloc_traits, __node> __node_allocator; + typedef allocator_traits<__node_allocator> __node_traits; + typedef typename _NodeTypes::__void_pointer __void_pointer; + typedef typename _NodeTypes::__node_pointer __node_pointer; + typedef typename _NodeTypes::__node_pointer __node_const_pointer; + typedef typename _NodeTypes::__node_base_type __first_node; + typedef typename _NodeTypes::__node_base_pointer __node_base_pointer; + typedef typename _NodeTypes::__next_pointer __next_pointer; private: - // check for sane allocator pointer rebinding semantics. Rebinding the - // allocator for a new pointer type should be exactly the same as rebinding - // the pointer using 'pointer_traits'. - static_assert((is_same<__node_pointer, typename __node_traits::pointer>::value), - "Allocator does not rebind pointers in a sane manner."); - typedef __rebind_alloc<__node_traits, __first_node> __node_base_allocator; - typedef allocator_traits<__node_base_allocator> __node_base_traits; - static_assert((is_same<__node_base_pointer, typename __node_base_traits::pointer>::value), - "Allocator does not rebind pointers in a sane manner."); + // check for sane allocator pointer rebinding semantics. Rebinding the + // allocator for a new pointer type should be exactly the same as rebinding + // the pointer using 'pointer_traits'. + static_assert((is_same<__node_pointer, typename __node_traits::pointer>::value), + "Allocator does not rebind pointers in a sane manner."); + typedef __rebind_alloc<__node_traits, __first_node> __node_base_allocator; + typedef allocator_traits<__node_base_allocator> __node_base_traits; + static_assert((is_same<__node_base_pointer, typename __node_base_traits::pointer>::value), + "Allocator does not rebind pointers in a sane manner."); private: + typedef __rebind_alloc<__node_traits, __next_pointer> __pointer_allocator; + typedef __bucket_list_deallocator<__pointer_allocator> __bucket_list_deleter; + typedef unique_ptr<__next_pointer[], __bucket_list_deleter> __bucket_list; + typedef allocator_traits<__pointer_allocator> __pointer_alloc_traits; + typedef typename __bucket_list_deleter::pointer __node_pointer_pointer; + + // --- Member data begin --- + __bucket_list __bucket_list_; + __compressed_pair<__first_node, __node_allocator> __p1_; + __compressed_pair __p2_; + __compressed_pair __p3_; + // --- Member data end --- + + _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __p2_.first(); } - typedef __rebind_alloc<__node_traits, __next_pointer> __pointer_allocator; - typedef __bucket_list_deallocator<__pointer_allocator> __bucket_list_deleter; - typedef unique_ptr<__next_pointer[], __bucket_list_deleter> __bucket_list; - typedef allocator_traits<__pointer_allocator> __pointer_alloc_traits; - typedef typename __bucket_list_deleter::pointer __node_pointer_pointer; - - // --- Member data begin --- - __bucket_list __bucket_list_; - __compressed_pair<__first_node, __node_allocator> __p1_; - __compressed_pair __p2_; - __compressed_pair __p3_; - // --- Member data end --- - - _LIBCPP_HIDE_FROM_ABI - size_type& size() _NOEXCEPT {return __p2_.first();} public: - _LIBCPP_HIDE_FROM_ABI - size_type size() const _NOEXCEPT {return __p2_.first();} - - _LIBCPP_HIDE_FROM_ABI - hasher& hash_function() _NOEXCEPT {return __p2_.second();} - _LIBCPP_HIDE_FROM_ABI - const hasher& hash_function() const _NOEXCEPT {return __p2_.second();} - - _LIBCPP_HIDE_FROM_ABI - float& max_load_factor() _NOEXCEPT {return __p3_.first();} - _LIBCPP_HIDE_FROM_ABI - float max_load_factor() const _NOEXCEPT {return __p3_.first();} - - _LIBCPP_HIDE_FROM_ABI - key_equal& key_eq() _NOEXCEPT {return __p3_.second();} - _LIBCPP_HIDE_FROM_ABI - const key_equal& key_eq() const _NOEXCEPT {return __p3_.second();} - - _LIBCPP_HIDE_FROM_ABI - __node_allocator& __node_alloc() _NOEXCEPT {return __p1_.second();} - _LIBCPP_HIDE_FROM_ABI - const __node_allocator& __node_alloc() const _NOEXCEPT - {return __p1_.second();} + _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __p2_.first(); } + + _LIBCPP_HIDE_FROM_ABI hasher& hash_function() _NOEXCEPT { return __p2_.second(); } + _LIBCPP_HIDE_FROM_ABI const hasher& hash_function() const _NOEXCEPT { return __p2_.second(); } + + _LIBCPP_HIDE_FROM_ABI float& max_load_factor() _NOEXCEPT { return __p3_.first(); } + _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __p3_.first(); } + + _LIBCPP_HIDE_FROM_ABI key_equal& key_eq() _NOEXCEPT { return __p3_.second(); } + _LIBCPP_HIDE_FROM_ABI const key_equal& key_eq() const _NOEXCEPT { return __p3_.second(); } + + _LIBCPP_HIDE_FROM_ABI __node_allocator& __node_alloc() _NOEXCEPT { return __p1_.second(); } + _LIBCPP_HIDE_FROM_ABI const __node_allocator& __node_alloc() const _NOEXCEPT { return __p1_.second(); } public: - typedef __hash_iterator<__node_pointer> iterator; - typedef __hash_const_iterator<__node_pointer> const_iterator; - typedef __hash_local_iterator<__node_pointer> local_iterator; - typedef __hash_const_local_iterator<__node_pointer> const_local_iterator; - - _LIBCPP_HIDE_FROM_ABI - __hash_table() - _NOEXCEPT_( - is_nothrow_default_constructible<__bucket_list>::value && - is_nothrow_default_constructible<__first_node>::value && - is_nothrow_default_constructible<__node_allocator>::value && - is_nothrow_default_constructible::value && - is_nothrow_default_constructible::value); - _LIBCPP_HIDE_FROM_ABI - __hash_table(const hasher& __hf, const key_equal& __eql); - _LIBCPP_HIDE_FROM_ABI __hash_table(const hasher& __hf, const key_equal& __eql, - const allocator_type& __a); - _LIBCPP_HIDE_FROM_ABI explicit __hash_table(const allocator_type& __a); - _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u); - _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u, const allocator_type& __a); - _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u) - _NOEXCEPT_( - is_nothrow_move_constructible<__bucket_list>::value && - is_nothrow_move_constructible<__first_node>::value && - is_nothrow_move_constructible<__node_allocator>::value && - is_nothrow_move_constructible::value && - is_nothrow_move_constructible::value); - _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u, const allocator_type& __a); - _LIBCPP_HIDE_FROM_ABI ~__hash_table(); - - _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(const __hash_table& __u); - _LIBCPP_HIDE_FROM_ABI - __hash_table& operator=(__hash_table&& __u) - _NOEXCEPT_( - __node_traits::propagate_on_container_move_assignment::value && - is_nothrow_move_assignable<__node_allocator>::value && - is_nothrow_move_assignable::value && - is_nothrow_move_assignable::value); - template - _LIBCPP_HIDE_FROM_ABI void __assign_unique(_InputIterator __first, _InputIterator __last); - template - _LIBCPP_HIDE_FROM_ABI void __assign_multi(_InputIterator __first, _InputIterator __last); - - _LIBCPP_HIDE_FROM_ABI - size_type max_size() const _NOEXCEPT - { - return std::min( - __node_traits::max_size(__node_alloc()), - numeric_limits::max() - ); - } + typedef __hash_iterator<__node_pointer> iterator; + typedef __hash_const_iterator<__node_pointer> const_iterator; + typedef __hash_local_iterator<__node_pointer> local_iterator; + typedef __hash_const_local_iterator<__node_pointer> const_local_iterator; + + _LIBCPP_HIDE_FROM_ABI __hash_table() _NOEXCEPT_( + is_nothrow_default_constructible<__bucket_list>::value&& is_nothrow_default_constructible<__first_node>::value&& + is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_default_constructible::value&& + is_nothrow_default_constructible::value); + _LIBCPP_HIDE_FROM_ABI __hash_table(const hasher& __hf, const key_equal& __eql); + _LIBCPP_HIDE_FROM_ABI __hash_table(const hasher& __hf, const key_equal& __eql, const allocator_type& __a); + _LIBCPP_HIDE_FROM_ABI explicit __hash_table(const allocator_type& __a); + _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u); + _LIBCPP_HIDE_FROM_ABI __hash_table(const __hash_table& __u, const allocator_type& __a); + _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u) _NOEXCEPT_( + is_nothrow_move_constructible<__bucket_list>::value&& is_nothrow_move_constructible<__first_node>::value&& + is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible::value&& + is_nothrow_move_constructible::value); + _LIBCPP_HIDE_FROM_ABI __hash_table(__hash_table&& __u, const allocator_type& __a); + _LIBCPP_HIDE_FROM_ABI ~__hash_table(); + + _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(const __hash_table& __u); + _LIBCPP_HIDE_FROM_ABI __hash_table& operator=(__hash_table&& __u) + _NOEXCEPT_(__node_traits::propagate_on_container_move_assignment::value&& + is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable::value&& + is_nothrow_move_assignable::value); + template + _LIBCPP_HIDE_FROM_ABI void __assign_unique(_InputIterator __first, _InputIterator __last); + template + _LIBCPP_HIDE_FROM_ABI void __assign_multi(_InputIterator __first, _InputIterator __last); + + _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { + return std::min(__node_traits::max_size(__node_alloc()), numeric_limits::max()); + } private: - _LIBCPP_HIDE_FROM_ABI - __next_pointer __node_insert_multi_prepare(size_t __cp_hash, - value_type& __cp_val); - _LIBCPP_HIDE_FROM_ABI - void __node_insert_multi_perform(__node_pointer __cp, - __next_pointer __pn) _NOEXCEPT; - - _LIBCPP_HIDE_FROM_ABI - __next_pointer __node_insert_unique_prepare(size_t __nd_hash, - value_type& __nd_val); - _LIBCPP_HIDE_FROM_ABI - void __node_insert_unique_perform(__node_pointer __ptr) _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI __next_pointer __node_insert_multi_prepare(size_t __cp_hash, value_type& __cp_val); + _LIBCPP_HIDE_FROM_ABI void __node_insert_multi_perform(__node_pointer __cp, __next_pointer __pn) _NOEXCEPT; + + _LIBCPP_HIDE_FROM_ABI __next_pointer __node_insert_unique_prepare(size_t __nd_hash, value_type& __nd_val); + _LIBCPP_HIDE_FROM_ABI void __node_insert_unique_perform(__node_pointer __ptr) _NOEXCEPT; public: - _LIBCPP_HIDE_FROM_ABI - pair __node_insert_unique(__node_pointer __nd); - _LIBCPP_HIDE_FROM_ABI - iterator __node_insert_multi(__node_pointer __nd); - _LIBCPP_HIDE_FROM_ABI - iterator __node_insert_multi(const_iterator __p, - __node_pointer __nd); - - template - _LIBCPP_HIDE_FROM_ABI - pair __emplace_unique_key_args(_Key const& __k, _Args&&... __args); - - template - _LIBCPP_HIDE_FROM_ABI - pair __emplace_unique_impl(_Args&&... __args); - - template - _LIBCPP_HIDE_FROM_ABI - pair __emplace_unique(_Pp&& __x) { - return __emplace_unique_extract_key(std::forward<_Pp>(__x), - __can_extract_key<_Pp, key_type>()); - } + _LIBCPP_HIDE_FROM_ABI pair __node_insert_unique(__node_pointer __nd); + _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd); + _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(const_iterator __p, __node_pointer __nd); - template ::value, int> = 0> - _LIBCPP_HIDE_FROM_ABI - pair - __emplace_unique(_First&& __f, _Second&& __s) { - return __emplace_unique_key_args(__f, std::forward<_First>(__f), - std::forward<_Second>(__s)); - } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique_key_args(_Key const& __k, _Args&&... __args); - template - _LIBCPP_HIDE_FROM_ABI - pair __emplace_unique(_Args&&... __args) { - return __emplace_unique_impl(std::forward<_Args>(__args)...); - } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique_impl(_Args&&... __args); - template - _LIBCPP_HIDE_FROM_ABI - pair - __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) { - return __emplace_unique_impl(std::forward<_Pp>(__x)); - } - template - _LIBCPP_HIDE_FROM_ABI - pair - __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) { - return __emplace_unique_key_args(__x, std::forward<_Pp>(__x)); - } - template - _LIBCPP_HIDE_FROM_ABI - pair - __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) { - return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x)); - } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique(_Pp&& __x) { + return __emplace_unique_extract_key(std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>()); + } - template - _LIBCPP_HIDE_FROM_ABI - iterator __emplace_multi(_Args&&... __args); - template - _LIBCPP_HIDE_FROM_ABI - iterator __emplace_hint_multi(const_iterator __p, _Args&&... __args); + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique(_First&& __f, _Second&& __s) { + return __emplace_unique_key_args(__f, std::forward<_First>(__f), std::forward<_Second>(__s)); + } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique(_Args&&... __args) { + return __emplace_unique_impl(std::forward<_Args>(__args)...); + } - _LIBCPP_HIDE_FROM_ABI - pair - __insert_unique(__container_value_type&& __x) { - return __emplace_unique_key_args(_NodeTypes::__get_key(__x), std::move(__x)); - } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) { + return __emplace_unique_impl(std::forward<_Pp>(__x)); + } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) { + return __emplace_unique_key_args(__x, std::forward<_Pp>(__x)); + } + template + _LIBCPP_HIDE_FROM_ABI pair __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) { + return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x)); + } - template ::value> > - _LIBCPP_HIDE_FROM_ABI - pair __insert_unique(_Pp&& __x) { - return __emplace_unique(std::forward<_Pp>(__x)); - } + template + _LIBCPP_HIDE_FROM_ABI iterator __emplace_multi(_Args&&... __args); + template + _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_multi(const_iterator __p, _Args&&... __args); - template - _LIBCPP_HIDE_FROM_ABI - iterator __insert_multi(_Pp&& __x) { - return __emplace_multi(std::forward<_Pp>(__x)); - } + _LIBCPP_HIDE_FROM_ABI pair __insert_unique(__container_value_type&& __x) { + return __emplace_unique_key_args(_NodeTypes::__get_key(__x), std::move(__x)); + } - template - _LIBCPP_HIDE_FROM_ABI - iterator __insert_multi(const_iterator __p, _Pp&& __x) { - return __emplace_hint_multi(__p, std::forward<_Pp>(__x)); - } + template ::value> > + _LIBCPP_HIDE_FROM_ABI pair __insert_unique(_Pp&& __x) { + return __emplace_unique(std::forward<_Pp>(__x)); + } - _LIBCPP_HIDE_FROM_ABI - pair __insert_unique(const __container_value_type& __x) { - return __emplace_unique_key_args(_NodeTypes::__get_key(__x), __x); - } + template + _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(_Pp&& __x) { + return __emplace_multi(std::forward<_Pp>(__x)); + } + + template + _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(const_iterator __p, _Pp&& __x) { + return __emplace_hint_multi(__p, std::forward<_Pp>(__x)); + } + + _LIBCPP_HIDE_FROM_ABI pair __insert_unique(const __container_value_type& __x) { + return __emplace_unique_key_args(_NodeTypes::__get_key(__x), __x); + } #if _LIBCPP_STD_VER >= 17 - template - _LIBCPP_HIDE_FROM_ABI - _InsertReturnType __node_handle_insert_unique(_NodeHandle&& __nh); - template - _LIBCPP_HIDE_FROM_ABI - iterator __node_handle_insert_unique(const_iterator __hint, - _NodeHandle&& __nh); - template - _LIBCPP_HIDE_FROM_ABI - void __node_handle_merge_unique(_Table& __source); - - template - _LIBCPP_HIDE_FROM_ABI - iterator __node_handle_insert_multi(_NodeHandle&& __nh); - template - _LIBCPP_HIDE_FROM_ABI - iterator __node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh); - template - _LIBCPP_HIDE_FROM_ABI - void __node_handle_merge_multi(_Table& __source); - - template - _LIBCPP_HIDE_FROM_ABI - _NodeHandle __node_handle_extract(key_type const& __key); - template - _LIBCPP_HIDE_FROM_ABI - _NodeHandle __node_handle_extract(const_iterator __it); + template + _LIBCPP_HIDE_FROM_ABI _InsertReturnType __node_handle_insert_unique(_NodeHandle&& __nh); + template + _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_unique(const_iterator __hint, _NodeHandle&& __nh); + template + _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_unique(_Table& __source); + + template + _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(_NodeHandle&& __nh); + template + _LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh); + template + _LIBCPP_HIDE_FROM_ABI void __node_handle_merge_multi(_Table& __source); + + template + _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(key_type const& __key); + template + _LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(const_iterator __it); #endif - _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI void __rehash_unique(size_type __n) { __rehash(__n); } - _LIBCPP_HIDE_FROM_ABI void __rehash_multi(size_type __n) { __rehash(__n); } - _LIBCPP_HIDE_FROM_ABI void __reserve_unique(size_type __n) - { - __rehash_unique(static_cast(std::ceil(__n / max_load_factor()))); - } - _LIBCPP_HIDE_FROM_ABI void __reserve_multi(size_type __n) - { - __rehash_multi(static_cast(std::ceil(__n / max_load_factor()))); - } + _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI void __rehash_unique(size_type __n) { __rehash(__n); } + _LIBCPP_HIDE_FROM_ABI void __rehash_multi(size_type __n) { __rehash(__n); } + _LIBCPP_HIDE_FROM_ABI void __reserve_unique(size_type __n) { + __rehash_unique(static_cast(std::ceil(__n / max_load_factor()))); + } + _LIBCPP_HIDE_FROM_ABI void __reserve_multi(size_type __n) { + __rehash_multi(static_cast(std::ceil(__n / max_load_factor()))); + } - _LIBCPP_HIDE_FROM_ABI - size_type bucket_count() const _NOEXCEPT - { - return __bucket_list_.get_deleter().size(); - } + _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __bucket_list_.get_deleter().size(); } - _LIBCPP_HIDE_FROM_ABI - iterator begin() _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI - iterator end() _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI - const_iterator begin() const _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI - const_iterator end() const _NOEXCEPT; - - template - _LIBCPP_HIDE_FROM_ABI - size_type bucket(const _Key& __k) const - { - _LIBCPP_ASSERT_UNCATEGORIZED(bucket_count() > 0, - "unordered container::bucket(key) called when bucket_count() == 0"); - return std::__constrain_hash(hash_function()(__k), bucket_count()); - } + _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT; + + template + _LIBCPP_HIDE_FROM_ABI size_type bucket(const _Key& __k) const { + _LIBCPP_ASSERT_UNCATEGORIZED( + bucket_count() > 0, "unordered container::bucket(key) called when bucket_count() == 0"); + return std::__constrain_hash(hash_function()(__k), bucket_count()); + } - template - _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __x); - template - _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __x) const; - - typedef __hash_node_destructor<__node_allocator> _Dp; - typedef unique_ptr<__node, _Dp> __node_holder; - - _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p); - _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last); - template - _LIBCPP_HIDE_FROM_ABI size_type __erase_unique(const _Key& __k); - template - _LIBCPP_HIDE_FROM_ABI size_type __erase_multi(const _Key& __k); - _LIBCPP_HIDE_FROM_ABI __node_holder remove(const_iterator __p) _NOEXCEPT; - - template - _LIBCPP_HIDE_FROM_ABI - size_type __count_unique(const _Key& __k) const; - template - _LIBCPP_HIDE_FROM_ABI size_type __count_multi(const _Key& __k) const; - - template - _LIBCPP_HIDE_FROM_ABI pair - __equal_range_unique(const _Key& __k); - template - _LIBCPP_HIDE_FROM_ABI pair - __equal_range_unique(const _Key& __k) const; - - template - _LIBCPP_HIDE_FROM_ABI pair - __equal_range_multi(const _Key& __k); - template - _LIBCPP_HIDE_FROM_ABI pair - __equal_range_multi(const _Key& __k) const; - - _LIBCPP_HIDE_FROM_ABI void swap(__hash_table& __u) + template + _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __x); + template + _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __x) const; + + typedef __hash_node_destructor<__node_allocator> _Dp; + typedef unique_ptr<__node, _Dp> __node_holder; + + _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p); + _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last); + template + _LIBCPP_HIDE_FROM_ABI size_type __erase_unique(const _Key& __k); + template + _LIBCPP_HIDE_FROM_ABI size_type __erase_multi(const _Key& __k); + _LIBCPP_HIDE_FROM_ABI __node_holder remove(const_iterator __p) _NOEXCEPT; + + template + _LIBCPP_HIDE_FROM_ABI size_type __count_unique(const _Key& __k) const; + template + _LIBCPP_HIDE_FROM_ABI size_type __count_multi(const _Key& __k) const; + + template + _LIBCPP_HIDE_FROM_ABI pair __equal_range_unique(const _Key& __k); + template + _LIBCPP_HIDE_FROM_ABI pair __equal_range_unique(const _Key& __k) const; + + template + _LIBCPP_HIDE_FROM_ABI pair __equal_range_multi(const _Key& __k); + template + _LIBCPP_HIDE_FROM_ABI pair __equal_range_multi(const _Key& __k) const; + + _LIBCPP_HIDE_FROM_ABI void swap(__hash_table& __u) #if _LIBCPP_STD_VER <= 11 - _NOEXCEPT_( - __is_nothrow_swappable::value && __is_nothrow_swappable::value - && (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value - || __is_nothrow_swappable<__pointer_allocator>::value) - && (!__node_traits::propagate_on_container_swap::value - || __is_nothrow_swappable<__node_allocator>::value) - ); + _NOEXCEPT_(__is_nothrow_swappable::value&& __is_nothrow_swappable::value && + (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value || + __is_nothrow_swappable<__pointer_allocator>::value) && + (!__node_traits::propagate_on_container_swap::value || + __is_nothrow_swappable<__node_allocator>::value)); #else - _NOEXCEPT_(__is_nothrow_swappable::value && __is_nothrow_swappable::value); + _NOEXCEPT_(__is_nothrow_swappable::value&& __is_nothrow_swappable::value); #endif - _LIBCPP_HIDE_FROM_ABI - size_type max_bucket_count() const _NOEXCEPT - {return max_size(); } - _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const; - _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT - { - size_type __bc = bucket_count(); - return __bc != 0 ? (float)size() / __bc : 0.f; - } - _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) _NOEXCEPT - { - _LIBCPP_ASSERT_UNCATEGORIZED(__mlf > 0, - "unordered container::max_load_factor(lf) called with lf <= 0"); - max_load_factor() = std::max(__mlf, load_factor()); - } + _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return max_size(); } + _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const; + _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { + size_type __bc = bucket_count(); + return __bc != 0 ? (float)size() / __bc : 0.f; + } + _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) _NOEXCEPT { + _LIBCPP_ASSERT_UNCATEGORIZED(__mlf > 0, "unordered container::max_load_factor(lf) called with lf <= 0"); + max_load_factor() = std::max(__mlf, load_factor()); + } - _LIBCPP_HIDE_FROM_ABI - local_iterator - begin(size_type __n) - { - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < bucket_count(), - "unordered container::begin(n) called with n >= bucket_count()"); - return local_iterator(__bucket_list_[__n], __n, bucket_count()); - } + _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __n < bucket_count(), "unordered container::begin(n) called with n >= bucket_count()"); + return local_iterator(__bucket_list_[__n], __n, bucket_count()); + } - _LIBCPP_HIDE_FROM_ABI - local_iterator - end(size_type __n) - { - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < bucket_count(), - "unordered container::end(n) called with n >= bucket_count()"); - return local_iterator(nullptr, __n, bucket_count()); - } + _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __n < bucket_count(), "unordered container::end(n) called with n >= bucket_count()"); + return local_iterator(nullptr, __n, bucket_count()); + } - _LIBCPP_HIDE_FROM_ABI - const_local_iterator - cbegin(size_type __n) const - { - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < bucket_count(), - "unordered container::cbegin(n) called with n >= bucket_count()"); - return const_local_iterator(__bucket_list_[__n], __n, bucket_count()); - } + _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __n < bucket_count(), "unordered container::cbegin(n) called with n >= bucket_count()"); + return const_local_iterator(__bucket_list_[__n], __n, bucket_count()); + } - _LIBCPP_HIDE_FROM_ABI - const_local_iterator - cend(size_type __n) const - { - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < bucket_count(), - "unordered container::cend(n) called with n >= bucket_count()"); - return const_local_iterator(nullptr, __n, bucket_count()); - } + _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __n < bucket_count(), "unordered container::cend(n) called with n >= bucket_count()"); + return const_local_iterator(nullptr, __n, bucket_count()); + } private: - template - _LIBCPP_HIDE_FROM_ABI void __rehash(size_type __n); - template - _LIBCPP_HIDE_FROM_ABI void __do_rehash(size_type __n); - - template - _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&& ...__args); - - template - _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest); - - - _LIBCPP_HIDE_FROM_ABI - void __copy_assign_alloc(const __hash_table& __u) - {__copy_assign_alloc(__u, integral_constant());} - _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u, true_type); - _LIBCPP_HIDE_FROM_ABI - void __copy_assign_alloc(const __hash_table&, false_type) {} - - _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, false_type); - _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, true_type) - _NOEXCEPT_( - is_nothrow_move_assignable<__node_allocator>::value && - is_nothrow_move_assignable::value && - is_nothrow_move_assignable::value); - _LIBCPP_HIDE_FROM_ABI - void __move_assign_alloc(__hash_table& __u) - _NOEXCEPT_( - !__node_traits::propagate_on_container_move_assignment::value || - (is_nothrow_move_assignable<__pointer_allocator>::value && - is_nothrow_move_assignable<__node_allocator>::value)) - {__move_assign_alloc(__u, integral_constant());} - _LIBCPP_HIDE_FROM_ABI - void __move_assign_alloc(__hash_table& __u, true_type) - _NOEXCEPT_( - is_nothrow_move_assignable<__pointer_allocator>::value && - is_nothrow_move_assignable<__node_allocator>::value) - { - __bucket_list_.get_deleter().__alloc() = - std::move(__u.__bucket_list_.get_deleter().__alloc()); - __node_alloc() = std::move(__u.__node_alloc()); - } - _LIBCPP_HIDE_FROM_ABI - void __move_assign_alloc(__hash_table&, false_type) _NOEXCEPT {} + template + _LIBCPP_HIDE_FROM_ABI void __rehash(size_type __n); + template + _LIBCPP_HIDE_FROM_ABI void __do_rehash(size_type __n); - _LIBCPP_HIDE_FROM_ABI void __deallocate_node(__next_pointer __np) _NOEXCEPT; - _LIBCPP_HIDE_FROM_ABI __next_pointer __detach() _NOEXCEPT; + template + _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args); - template friend class _LIBCPP_TEMPLATE_VIS unordered_map; - template friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; + template + _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest); + + _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u) { + __copy_assign_alloc(__u, integral_constant()); + } + _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u, true_type); + _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table&, false_type) {} + + _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, false_type); + _LIBCPP_HIDE_FROM_ABI void __move_assign(__hash_table& __u, true_type) + _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable::value&& + is_nothrow_move_assignable::value); + _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table& __u) _NOEXCEPT_( + !__node_traits::propagate_on_container_move_assignment::value || + (is_nothrow_move_assignable<__pointer_allocator>::value && is_nothrow_move_assignable<__node_allocator>::value)) { + __move_assign_alloc(__u, integral_constant()); + } + _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table& __u, true_type) _NOEXCEPT_( + is_nothrow_move_assignable<__pointer_allocator>::value&& is_nothrow_move_assignable<__node_allocator>::value) { + __bucket_list_.get_deleter().__alloc() = std::move(__u.__bucket_list_.get_deleter().__alloc()); + __node_alloc() = std::move(__u.__node_alloc()); + } + _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table&, false_type) _NOEXCEPT {} + + _LIBCPP_HIDE_FROM_ABI void __deallocate_node(__next_pointer __np) _NOEXCEPT; + _LIBCPP_HIDE_FROM_ABI __next_pointer __detach() _NOEXCEPT; + + template + friend class _LIBCPP_TEMPLATE_VIS unordered_map; + template + friend class _LIBCPP_TEMPLATE_VIS unordered_multimap; }; template -inline -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table() - _NOEXCEPT_( - is_nothrow_default_constructible<__bucket_list>::value && - is_nothrow_default_constructible<__first_node>::value && - is_nothrow_default_constructible<__node_allocator>::value && - is_nothrow_default_constructible::value && - is_nothrow_default_constructible::value) - : __p2_(0, __default_init_tag()), - __p3_(1.0f, __default_init_tag()) -{ -} +inline __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table() _NOEXCEPT_( + is_nothrow_default_constructible<__bucket_list>::value&& is_nothrow_default_constructible<__first_node>::value&& + is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_default_constructible::value&& + is_nothrow_default_constructible::value) + : __p2_(0, __default_init_tag()), __p3_(1.0f, __default_init_tag()) {} template -inline -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const hasher& __hf, - const key_equal& __eql) - : __bucket_list_(nullptr, __bucket_list_deleter()), - __p1_(), - __p2_(0, __hf), - __p3_(1.0f, __eql) -{ -} +inline __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const hasher& __hf, const key_equal& __eql) + : __bucket_list_(nullptr, __bucket_list_deleter()), __p1_(), __p2_(0, __hf), __p3_(1.0f, __eql) {} template -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const hasher& __hf, - const key_equal& __eql, - const allocator_type& __a) +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table( + const hasher& __hf, const key_equal& __eql, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), __p1_(__default_init_tag(), __node_allocator(__a)), __p2_(0, __hf), - __p3_(1.0f, __eql) -{ -} + __p3_(1.0f, __eql) {} template __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), __p1_(__default_init_tag(), __node_allocator(__a)), __p2_(0, __default_init_tag()), - __p3_(1.0f, __default_init_tag()) -{ -} + __p3_(1.0f, __default_init_tag()) {} template __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u) : __bucket_list_(nullptr, - __bucket_list_deleter(allocator_traits<__pointer_allocator>:: - select_on_container_copy_construction( - __u.__bucket_list_.get_deleter().__alloc()), 0)), - __p1_(__default_init_tag(), allocator_traits<__node_allocator>:: - select_on_container_copy_construction(__u.__node_alloc())), + __bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction( + __u.__bucket_list_.get_deleter().__alloc()), + 0)), + __p1_(__default_init_tag(), + allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())), __p2_(0, __u.hash_function()), - __p3_(__u.__p3_) -{ -} + __p3_(__u.__p3_) {} template -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, - const allocator_type& __a) +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), __p1_(__default_init_tag(), __node_allocator(__a)), __p2_(0, __u.hash_function()), - __p3_(__u.__p3_) -{ -} + __p3_(__u.__p3_) {} template -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u) - _NOEXCEPT_( - is_nothrow_move_constructible<__bucket_list>::value && - is_nothrow_move_constructible<__first_node>::value && - is_nothrow_move_constructible<__node_allocator>::value && - is_nothrow_move_constructible::value && +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u) _NOEXCEPT_( + is_nothrow_move_constructible<__bucket_list>::value&& is_nothrow_move_constructible<__first_node>::value&& + is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible::value&& is_nothrow_move_constructible::value) : __bucket_list_(std::move(__u.__bucket_list_)), __p1_(std::move(__u.__p1_)), __p2_(std::move(__u.__p2_)), - __p3_(std::move(__u.__p3_)) -{ - if (size() > 0) - { - __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - __u.__p1_.first().__next_ = nullptr; - __u.size() = 0; - } + __p3_(std::move(__u.__p3_)) { + if (size() > 0) { + __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr(); + __u.__p1_.first().__next_ = nullptr; + __u.size() = 0; + } } template -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u, - const allocator_type& __a) +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), __p1_(__default_init_tag(), __node_allocator(__a)), __p2_(0, std::move(__u.hash_function())), - __p3_(std::move(__u.__p3_)) -{ - if (__a == allocator_type(__u.__node_alloc())) - { - __bucket_list_.reset(__u.__bucket_list_.release()); - __bucket_list_.get_deleter().size() = __u.__bucket_list_.get_deleter().size(); - __u.__bucket_list_.get_deleter().size() = 0; - if (__u.size() > 0) - { - __p1_.first().__next_ = __u.__p1_.first().__next_; - __u.__p1_.first().__next_ = nullptr; - __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - size() = __u.size(); - __u.size() = 0; - } + __p3_(std::move(__u.__p3_)) { + if (__a == allocator_type(__u.__node_alloc())) { + __bucket_list_.reset(__u.__bucket_list_.release()); + __bucket_list_.get_deleter().size() = __u.__bucket_list_.get_deleter().size(); + __u.__bucket_list_.get_deleter().size() = 0; + if (__u.size() > 0) { + __p1_.first().__next_ = __u.__p1_.first().__next_; + __u.__p1_.first().__next_ = nullptr; + __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr(); + size() = __u.size(); + __u.size() = 0; } + } } template -__hash_table<_Tp, _Hash, _Equal, _Alloc>::~__hash_table() -{ +__hash_table<_Tp, _Hash, _Equal, _Alloc>::~__hash_table() { #if defined(_LIBCPP_CXX03_LANG) - static_assert((is_copy_constructible::value), - "Predicate must be copy-constructible."); - static_assert((is_copy_constructible::value), - "Hasher must be copy-constructible."); + static_assert((is_copy_constructible::value), "Predicate must be copy-constructible."); + static_assert((is_copy_constructible::value), "Hasher must be copy-constructible."); #endif - __deallocate_node(__p1_.first().__next_); + __deallocate_node(__p1_.first().__next_); } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc( - const __hash_table& __u, true_type) -{ - if (__node_alloc() != __u.__node_alloc()) - { - clear(); - __bucket_list_.reset(); - __bucket_list_.get_deleter().size() = 0; - } - __bucket_list_.get_deleter().__alloc() = __u.__bucket_list_.get_deleter().__alloc(); - __node_alloc() = __u.__node_alloc(); +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_table& __u, true_type) { + if (__node_alloc() != __u.__node_alloc()) { + clear(); + __bucket_list_.reset(); + __bucket_list_.get_deleter().size() = 0; + } + __bucket_list_.get_deleter().__alloc() = __u.__bucket_list_.get_deleter().__alloc(); + __node_alloc() = __u.__node_alloc(); } template -__hash_table<_Tp, _Hash, _Equal, _Alloc>& -__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) -{ - if (this != std::addressof(__u)) - { - __copy_assign_alloc(__u); - hash_function() = __u.hash_function(); - key_eq() = __u.key_eq(); - max_load_factor() = __u.max_load_factor(); - __assign_multi(__u.begin(), __u.end()); - } - return *this; +__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) { + if (this != std::addressof(__u)) { + __copy_assign_alloc(__u); + hash_function() = __u.hash_function(); + key_eq() = __u.key_eq(); + max_load_factor() = __u.max_load_factor(); + __assign_multi(__u.begin(), __u.end()); + } + return *this; } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np) - _NOEXCEPT -{ - __node_allocator& __na = __node_alloc(); - while (__np != nullptr) - { - __next_pointer __next = __np->__next_; - __node_pointer __real_np = __np->__upcast(); - __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__get_value())); - std::__destroy_at(std::addressof(*__real_np)); - __node_traits::deallocate(__na, __real_np, 1); - __np = __next; - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np) _NOEXCEPT { + __node_allocator& __na = __node_alloc(); + while (__np != nullptr) { + __next_pointer __next = __np->__next_; + __node_pointer __real_np = __np->__upcast(); + __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__get_value())); + std::__destroy_at(std::addressof(*__real_np)); + __node_traits::deallocate(__na, __real_np, 1); + __np = __next; + } } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__detach() _NOEXCEPT -{ - size_type __bc = bucket_count(); - for (size_type __i = 0; __i < __bc; ++__i) - __bucket_list_[__i] = nullptr; - size() = 0; - __next_pointer __cache = __p1_.first().__next_; - __p1_.first().__next_ = nullptr; - return __cache; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__detach() _NOEXCEPT { + size_type __bc = bucket_count(); + for (size_type __i = 0; __i < __bc; ++__i) + __bucket_list_[__i] = nullptr; + size() = 0; + __next_pointer __cache = __p1_.first().__next_; + __p1_.first().__next_ = nullptr; + return __cache; } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign( - __hash_table& __u, true_type) - _NOEXCEPT_( - is_nothrow_move_assignable<__node_allocator>::value && - is_nothrow_move_assignable::value && - is_nothrow_move_assignable::value) -{ - clear(); - __bucket_list_.reset(__u.__bucket_list_.release()); - __bucket_list_.get_deleter().size() = __u.__bucket_list_.get_deleter().size(); - __u.__bucket_list_.get_deleter().size() = 0; - __move_assign_alloc(__u); - size() = __u.size(); - hash_function() = std::move(__u.hash_function()); - max_load_factor() = __u.max_load_factor(); - key_eq() = std::move(__u.key_eq()); - __p1_.first().__next_ = __u.__p1_.first().__next_; - if (size() > 0) - { - __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - __u.__p1_.first().__next_ = nullptr; - __u.size() = 0; - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u, true_type) + _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value&& is_nothrow_move_assignable::value&& + is_nothrow_move_assignable::value) { + clear(); + __bucket_list_.reset(__u.__bucket_list_.release()); + __bucket_list_.get_deleter().size() = __u.__bucket_list_.get_deleter().size(); + __u.__bucket_list_.get_deleter().size() = 0; + __move_assign_alloc(__u); + size() = __u.size(); + hash_function() = std::move(__u.hash_function()); + max_load_factor() = __u.max_load_factor(); + key_eq() = std::move(__u.key_eq()); + __p1_.first().__next_ = __u.__p1_.first().__next_; + if (size() > 0) { + __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr(); + __u.__p1_.first().__next_ = nullptr; + __u.size() = 0; + } } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign( - __hash_table& __u, false_type) -{ - if (__node_alloc() == __u.__node_alloc()) - __move_assign(__u, true_type()); - else - { - hash_function() = std::move(__u.hash_function()); - key_eq() = std::move(__u.key_eq()); - max_load_factor() = __u.max_load_factor(); - if (bucket_count() != 0) - { - __next_pointer __cache = __detach(); -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - const_iterator __i = __u.begin(); - while (__cache != nullptr && __u.size() != 0) - { - __cache->__upcast()->__get_value() = - std::move(__u.remove(__i++)->__get_value()); - __next_pointer __next = __cache->__next_; - __node_insert_multi(__cache->__upcast()); - __cache = __next; - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u, false_type) { + if (__node_alloc() == __u.__node_alloc()) + __move_assign(__u, true_type()); + else { + hash_function() = std::move(__u.hash_function()); + key_eq() = std::move(__u.key_eq()); + max_load_factor() = __u.max_load_factor(); + if (bucket_count() != 0) { + __next_pointer __cache = __detach(); #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - __deallocate_node(__cache); - throw; - } + try { #endif // _LIBCPP_HAS_NO_EXCEPTIONS - __deallocate_node(__cache); - } const_iterator __i = __u.begin(); - while (__u.size() != 0) - { - __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__get_value())); - __node_insert_multi(__h.get()); - __h.release(); + while (__cache != nullptr && __u.size() != 0) { + __cache->__upcast()->__get_value() = std::move(__u.remove(__i++)->__get_value()); + __next_pointer __next = __cache->__next_; + __node_insert_multi(__cache->__upcast()); + __cache = __next; } +#ifndef _LIBCPP_HAS_NO_EXCEPTIONS + } catch (...) { + __deallocate_node(__cache); + throw; + } +#endif // _LIBCPP_HAS_NO_EXCEPTIONS + __deallocate_node(__cache); + } + const_iterator __i = __u.begin(); + while (__u.size() != 0) { + __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__get_value())); + __node_insert_multi(__h.get()); + __h.release(); } + } } template -inline -__hash_table<_Tp, _Hash, _Equal, _Alloc>& -__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(__hash_table&& __u) - _NOEXCEPT_( - __node_traits::propagate_on_container_move_assignment::value && - is_nothrow_move_assignable<__node_allocator>::value && - is_nothrow_move_assignable::value && - is_nothrow_move_assignable::value) -{ - __move_assign(__u, integral_constant()); - return *this; +inline __hash_table<_Tp, _Hash, _Equal, _Alloc>& +__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(__hash_table&& __u) _NOEXCEPT_( + __node_traits::propagate_on_container_move_assignment::value&& is_nothrow_move_assignable<__node_allocator>::value&& + is_nothrow_move_assignable::value&& is_nothrow_move_assignable::value) { + __move_assign(__u, integral_constant()); + return *this; } template template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __first, - _InputIterator __last) -{ - typedef iterator_traits<_InputIterator> _ITraits; - typedef typename _ITraits::value_type _ItValueType; - static_assert((is_same<_ItValueType, __container_value_type>::value), - "__assign_unique may only be called with the containers value type"); - - if (bucket_count() != 0) - { - __next_pointer __cache = __detach(); +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __first, _InputIterator __last) { + typedef iterator_traits<_InputIterator> _ITraits; + typedef typename _ITraits::value_type _ItValueType; + static_assert((is_same<_ItValueType, __container_value_type>::value), + "__assign_unique may only be called with the containers value type"); + + if (bucket_count() != 0) { + __next_pointer __cache = __detach(); #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { + try { #endif // _LIBCPP_HAS_NO_EXCEPTIONS - for (; __cache != nullptr && __first != __last; ++__first) - { - __cache->__upcast()->__get_value() = *__first; - __next_pointer __next = __cache->__next_; - __node_insert_unique(__cache->__upcast()); - __cache = __next; - } + for (; __cache != nullptr && __first != __last; ++__first) { + __cache->__upcast()->__get_value() = *__first; + __next_pointer __next = __cache->__next_; + __node_insert_unique(__cache->__upcast()); + __cache = __next; + } #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - __deallocate_node(__cache); - throw; - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - __deallocate_node(__cache); + } catch (...) { + __deallocate_node(__cache); + throw; } - for (; __first != __last; ++__first) - __insert_unique(*__first); +#endif // _LIBCPP_HAS_NO_EXCEPTIONS + __deallocate_node(__cache); + } + for (; __first != __last; ++__first) + __insert_unique(*__first); } template template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __first, - _InputIterator __last) -{ - typedef iterator_traits<_InputIterator> _ITraits; - typedef typename _ITraits::value_type _ItValueType; - static_assert((is_same<_ItValueType, __container_value_type>::value || - is_same<_ItValueType, __node_value_type>::value), - "__assign_multi may only be called with the containers value type" - " or the nodes value type"); - if (bucket_count() != 0) - { - __next_pointer __cache = __detach(); +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __first, _InputIterator __last) { + typedef iterator_traits<_InputIterator> _ITraits; + typedef typename _ITraits::value_type _ItValueType; + static_assert( + (is_same<_ItValueType, __container_value_type>::value || is_same<_ItValueType, __node_value_type>::value), + "__assign_multi may only be called with the containers value type" + " or the nodes value type"); + if (bucket_count() != 0) { + __next_pointer __cache = __detach(); #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try - { + try { #endif // _LIBCPP_HAS_NO_EXCEPTIONS - for (; __cache != nullptr && __first != __last; ++__first) - { - __cache->__upcast()->__get_value() = *__first; - __next_pointer __next = __cache->__next_; - __node_insert_multi(__cache->__upcast()); - __cache = __next; - } + for (; __cache != nullptr && __first != __last; ++__first) { + __cache->__upcast()->__get_value() = *__first; + __next_pointer __next = __cache->__next_; + __node_insert_multi(__cache->__upcast()); + __cache = __next; + } #ifndef _LIBCPP_HAS_NO_EXCEPTIONS - } - catch (...) - { - __deallocate_node(__cache); - throw; - } -#endif // _LIBCPP_HAS_NO_EXCEPTIONS - __deallocate_node(__cache); + } catch (...) { + __deallocate_node(__cache); + throw; } - for (; __first != __last; ++__first) - __insert_multi(_NodeTypes::__get_value(*__first)); +#endif // _LIBCPP_HAS_NO_EXCEPTIONS + __deallocate_node(__cache); + } + for (; __first != __last; ++__first) + __insert_multi(_NodeTypes::__get_value(*__first)); } template -inline -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() _NOEXCEPT -{ - return iterator(__p1_.first().__next_); +inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() _NOEXCEPT { + return iterator(__p1_.first().__next_); } template -inline -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() _NOEXCEPT -{ - return iterator(nullptr); +inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() _NOEXCEPT { + return iterator(nullptr); } template -inline -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() const _NOEXCEPT -{ - return const_iterator(__p1_.first().__next_); +inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() const _NOEXCEPT { + return const_iterator(__p1_.first().__next_); } template -inline -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() const _NOEXCEPT -{ - return const_iterator(nullptr); +inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::end() const _NOEXCEPT { + return const_iterator(nullptr); } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT -{ - if (size() > 0) - { - __deallocate_node(__p1_.first().__next_); - __p1_.first().__next_ = nullptr; - size_type __bc = bucket_count(); - for (size_type __i = 0; __i < __bc; ++__i) - __bucket_list_[__i] = nullptr; - size() = 0; - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT { + if (size() > 0) { + __deallocate_node(__p1_.first().__next_); + __p1_.first().__next_ = nullptr; + size_type __bc = bucket_count(); + for (size_type __i = 0; __i < __bc; ++__i) + __bucket_list_[__i] = nullptr; + size() = 0; + } } - // Prepare the container for an insertion of the value __value with the hash // __hash. This does a lookup into the container to see if __value is already // present, and performs a rehash if necessary. Returns a pointer to the @@ -1640,36 +1298,28 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT // Note that this function does forward exceptions if key_eq() throws, and never // mutates __value or actually inserts into the map. template -_LIBCPP_HIDE_FROM_ABI -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare( - size_t __hash, value_type& __value) -{ - size_type __bc = bucket_count(); - - if (__bc != 0) - { - size_t __chash = std::__constrain_hash(__hash, __bc); - __next_pointer __ndptr = __bucket_list_[__chash]; - if (__ndptr != nullptr) - { - for (__ndptr = __ndptr->__next_; __ndptr != nullptr && - (__ndptr->__hash() == __hash || - std::__constrain_hash(__ndptr->__hash(), __bc) == __chash); - __ndptr = __ndptr->__next_) - { - if ((__ndptr->__hash() == __hash) && - key_eq()(__ndptr->__upcast()->__get_value(), __value)) - return __ndptr; - } - } - } - if (size()+1 > __bc * max_load_factor() || __bc == 0) - { - __rehash_unique(std::max(2 * __bc + !std::__is_hash_power2(__bc), - size_type(std::ceil(float(size() + 1) / max_load_factor())))); +_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare(size_t __hash, value_type& __value) { + size_type __bc = bucket_count(); + + if (__bc != 0) { + size_t __chash = std::__constrain_hash(__hash, __bc); + __next_pointer __ndptr = __bucket_list_[__chash]; + if (__ndptr != nullptr) { + for (__ndptr = __ndptr->__next_; + __ndptr != nullptr && + (__ndptr->__hash() == __hash || std::__constrain_hash(__ndptr->__hash(), __bc) == __chash); + __ndptr = __ndptr->__next_) { + if ((__ndptr->__hash() == __hash) && key_eq()(__ndptr->__upcast()->__get_value(), __value)) + return __ndptr; + } } - return nullptr; + } + if (size() + 1 > __bc * max_load_factor() || __bc == 0) { + __rehash_unique(std::max( + 2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor())))); + } + return nullptr; } // Insert the node __nd into the container by pushing it into the right bucket, @@ -1677,50 +1327,41 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare( // rehashing has already occurred and that no element with the same key exists // in the map. template -_LIBCPP_HIDE_FROM_ABI -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_perform( - __node_pointer __nd) _NOEXCEPT -{ - size_type __bc = bucket_count(); - size_t __chash = std::__constrain_hash(__nd->__hash(), __bc); - // insert_after __bucket_list_[__chash], or __first_node if bucket is null - __next_pointer __pn = __bucket_list_[__chash]; - if (__pn == nullptr) - { - __pn =__p1_.first().__ptr(); - __nd->__next_ = __pn->__next_; - __pn->__next_ = __nd->__ptr(); - // fix up __bucket_list_ - __bucket_list_[__chash] = __pn; - if (__nd->__next_ != nullptr) - __bucket_list_[std::__constrain_hash(__nd->__next_->__hash(), __bc)] = __nd->__ptr(); - } - else - { - __nd->__next_ = __pn->__next_; - __pn->__next_ = __nd->__ptr(); - } - ++size(); +_LIBCPP_HIDE_FROM_ABI void +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_perform(__node_pointer __nd) _NOEXCEPT { + size_type __bc = bucket_count(); + size_t __chash = std::__constrain_hash(__nd->__hash(), __bc); + // insert_after __bucket_list_[__chash], or __first_node if bucket is null + __next_pointer __pn = __bucket_list_[__chash]; + if (__pn == nullptr) { + __pn = __p1_.first().__ptr(); + __nd->__next_ = __pn->__next_; + __pn->__next_ = __nd->__ptr(); + // fix up __bucket_list_ + __bucket_list_[__chash] = __pn; + if (__nd->__next_ != nullptr) + __bucket_list_[std::__constrain_hash(__nd->__next_->__hash(), __bc)] = __nd->__ptr(); + } else { + __nd->__next_ = __pn->__next_; + __pn->__next_ = __nd->__ptr(); + } + ++size(); } template pair::iterator, bool> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd) -{ - __nd->__hash_ = hash_function()(__nd->__get_value()); - __next_pointer __existing_node = - __node_insert_unique_prepare(__nd->__hash(), __nd->__get_value()); - - // Insert the node, unless it already exists in the container. - bool __inserted = false; - if (__existing_node == nullptr) - { - __node_insert_unique_perform(__nd); - __existing_node = __nd->__ptr(); - __inserted = true; - } - return pair(iterator(__existing_node), __inserted); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd) { + __nd->__hash_ = hash_function()(__nd->__get_value()); + __next_pointer __existing_node = __node_insert_unique_prepare(__nd->__hash(), __nd->__get_value()); + + // Insert the node, unless it already exists in the container. + bool __inserted = false; + if (__existing_node == nullptr) { + __node_insert_unique_perform(__nd); + __existing_node = __nd->__ptr(); + __inserted = true; + } + return pair(iterator(__existing_node), __inserted); } // Prepare the container for an insertion of the value __cp_val with the hash @@ -1732,40 +1373,34 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __ // mutates __value or actually inserts into the map. template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__next_pointer -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare( - size_t __cp_hash, value_type& __cp_val) -{ - size_type __bc = bucket_count(); - if (size()+1 > __bc * max_load_factor() || __bc == 0) - { - __rehash_multi(std::max(2 * __bc + !std::__is_hash_power2(__bc), - size_type(std::ceil(float(size() + 1) / max_load_factor())))); - __bc = bucket_count(); - } - size_t __chash = std::__constrain_hash(__cp_hash, __bc); - __next_pointer __pn = __bucket_list_[__chash]; - if (__pn != nullptr) - { - for (bool __found = false; __pn->__next_ != nullptr && - std::__constrain_hash(__pn->__next_->__hash(), __bc) == __chash; - __pn = __pn->__next_) - { - // __found key_eq() action - // false false loop - // true true loop - // false true set __found to true - // true false break - if (__found != (__pn->__next_->__hash() == __cp_hash && - key_eq()(__pn->__next_->__upcast()->__get_value(), __cp_val))) - { - if (!__found) - __found = true; - else - break; - } - } +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare(size_t __cp_hash, value_type& __cp_val) { + size_type __bc = bucket_count(); + if (size() + 1 > __bc * max_load_factor() || __bc == 0) { + __rehash_multi(std::max( + 2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor())))); + __bc = bucket_count(); + } + size_t __chash = std::__constrain_hash(__cp_hash, __bc); + __next_pointer __pn = __bucket_list_[__chash]; + if (__pn != nullptr) { + for (bool __found = false; + __pn->__next_ != nullptr && std::__constrain_hash(__pn->__next_->__hash(), __bc) == __chash; + __pn = __pn->__next_) { + // __found key_eq() action + // false false loop + // true true loop + // false true set __found to true + // true false break + if (__found != + (__pn->__next_->__hash() == __cp_hash && key_eq()(__pn->__next_->__upcast()->__get_value(), __cp_val))) { + if (!__found) + __found = true; + else + break; + } } - return __pn; + } + return __pn; } // Insert the node __cp into the container after __pn (which is the last node in @@ -1774,746 +1409,601 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare( // all we need to do is update the bucket and size(). Assumes that __cp->__hash // is up-to-date. template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_perform( - __node_pointer __cp, __next_pointer __pn) _NOEXCEPT -{ - size_type __bc = bucket_count(); - size_t __chash = std::__constrain_hash(__cp->__hash_, __bc); - if (__pn == nullptr) - { - __pn =__p1_.first().__ptr(); - __cp->__next_ = __pn->__next_; - __pn->__next_ = __cp->__ptr(); - // fix up __bucket_list_ - __bucket_list_[__chash] = __pn; - if (__cp->__next_ != nullptr) - __bucket_list_[std::__constrain_hash(__cp->__next_->__hash(), __bc)] - = __cp->__ptr(); - } - else - { - __cp->__next_ = __pn->__next_; - __pn->__next_ = __cp->__ptr(); - if (__cp->__next_ != nullptr) - { - size_t __nhash = std::__constrain_hash(__cp->__next_->__hash(), __bc); - if (__nhash != __chash) - __bucket_list_[__nhash] = __cp->__ptr(); - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_perform( + __node_pointer __cp, __next_pointer __pn) _NOEXCEPT { + size_type __bc = bucket_count(); + size_t __chash = std::__constrain_hash(__cp->__hash_, __bc); + if (__pn == nullptr) { + __pn = __p1_.first().__ptr(); + __cp->__next_ = __pn->__next_; + __pn->__next_ = __cp->__ptr(); + // fix up __bucket_list_ + __bucket_list_[__chash] = __pn; + if (__cp->__next_ != nullptr) + __bucket_list_[std::__constrain_hash(__cp->__next_->__hash(), __bc)] = __cp->__ptr(); + } else { + __cp->__next_ = __pn->__next_; + __pn->__next_ = __cp->__ptr(); + if (__cp->__next_ != nullptr) { + size_t __nhash = std::__constrain_hash(__cp->__next_->__hash(), __bc); + if (__nhash != __chash) + __bucket_list_[__nhash] = __cp->__ptr(); } - ++size(); + } + ++size(); } - template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp) -{ - __cp->__hash_ = hash_function()(__cp->__get_value()); - __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__get_value()); - __node_insert_multi_perform(__cp, __pn); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp) { + __cp->__hash_ = hash_function()(__cp->__get_value()); + __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__get_value()); + __node_insert_multi_perform(__cp, __pn); - return iterator(__cp->__ptr()); + return iterator(__cp->__ptr()); } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi( - const_iterator __p, __node_pointer __cp) -{ - if (__p != end() && key_eq()(*__p, __cp->__get_value())) - { - __next_pointer __np = __p.__node_; - __cp->__hash_ = __np->__hash(); - size_type __bc = bucket_count(); - if (size()+1 > __bc * max_load_factor() || __bc == 0) - { - __rehash_multi(std::max(2 * __bc + !std::__is_hash_power2(__bc), - size_type(std::ceil(float(size() + 1) / max_load_factor())))); - __bc = bucket_count(); - } - size_t __chash = std::__constrain_hash(__cp->__hash_, __bc); - __next_pointer __pp = __bucket_list_[__chash]; - while (__pp->__next_ != __np) - __pp = __pp->__next_; - __cp->__next_ = __np; - __pp->__next_ = static_cast<__next_pointer>(__cp); - ++size(); - return iterator(static_cast<__next_pointer>(__cp)); - } - return __node_insert_multi(__cp); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(const_iterator __p, __node_pointer __cp) { + if (__p != end() && key_eq()(*__p, __cp->__get_value())) { + __next_pointer __np = __p.__node_; + __cp->__hash_ = __np->__hash(); + size_type __bc = bucket_count(); + if (size() + 1 > __bc * max_load_factor() || __bc == 0) { + __rehash_multi(std::max( + 2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor())))); + __bc = bucket_count(); + } + size_t __chash = std::__constrain_hash(__cp->__hash_, __bc); + __next_pointer __pp = __bucket_list_[__chash]; + while (__pp->__next_ != __np) + __pp = __pp->__next_; + __cp->__next_ = __np; + __pp->__next_ = static_cast<__next_pointer>(__cp); + ++size(); + return iterator(static_cast<__next_pointer>(__cp)); + } + return __node_insert_multi(__cp); } - - template -template +template pair::iterator, bool> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) -{ - - size_t __hash = hash_function()(__k); - size_type __bc = bucket_count(); - bool __inserted = false; - __next_pointer __nd; - size_t __chash; - if (__bc != 0) - { - __chash = std::__constrain_hash(__hash, __bc); - __nd = __bucket_list_[__chash]; - if (__nd != nullptr) - { - for (__nd = __nd->__next_; __nd != nullptr && - (__nd->__hash() == __hash || std::__constrain_hash(__nd->__hash(), __bc) == __chash); - __nd = __nd->__next_) - { - if ((__nd->__hash() == __hash) && - key_eq()(__nd->__upcast()->__get_value(), __k)) - goto __done; - } - } +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) { + size_t __hash = hash_function()(__k); + size_type __bc = bucket_count(); + bool __inserted = false; + __next_pointer __nd; + size_t __chash; + if (__bc != 0) { + __chash = std::__constrain_hash(__hash, __bc); + __nd = __bucket_list_[__chash]; + if (__nd != nullptr) { + for (__nd = __nd->__next_; + __nd != nullptr && (__nd->__hash() == __hash || std::__constrain_hash(__nd->__hash(), __bc) == __chash); + __nd = __nd->__next_) { + if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k)) + goto __done; + } } - { - __node_holder __h = __construct_node_hash(__hash, std::forward<_Args>(__args)...); - if (size()+1 > __bc * max_load_factor() || __bc == 0) - { - __rehash_unique(std::max(2 * __bc + !std::__is_hash_power2(__bc), - size_type(std::ceil(float(size() + 1) / max_load_factor())))); - __bc = bucket_count(); - __chash = std::__constrain_hash(__hash, __bc); - } - // insert_after __bucket_list_[__chash], or __first_node if bucket is null - __next_pointer __pn = __bucket_list_[__chash]; - if (__pn == nullptr) - { - __pn = __p1_.first().__ptr(); - __h->__next_ = __pn->__next_; - __pn->__next_ = __h.get()->__ptr(); - // fix up __bucket_list_ - __bucket_list_[__chash] = __pn; - if (__h->__next_ != nullptr) - __bucket_list_[std::__constrain_hash(__h->__next_->__hash(), __bc)] - = __h.get()->__ptr(); - } - else - { - __h->__next_ = __pn->__next_; - __pn->__next_ = static_cast<__next_pointer>(__h.get()); - } - __nd = static_cast<__next_pointer>(__h.release()); - // increment size - ++size(); - __inserted = true; + } + { + __node_holder __h = __construct_node_hash(__hash, std::forward<_Args>(__args)...); + if (size() + 1 > __bc * max_load_factor() || __bc == 0) { + __rehash_unique(std::max( + 2 * __bc + !std::__is_hash_power2(__bc), size_type(std::ceil(float(size() + 1) / max_load_factor())))); + __bc = bucket_count(); + __chash = std::__constrain_hash(__hash, __bc); } + // insert_after __bucket_list_[__chash], or __first_node if bucket is null + __next_pointer __pn = __bucket_list_[__chash]; + if (__pn == nullptr) { + __pn = __p1_.first().__ptr(); + __h->__next_ = __pn->__next_; + __pn->__next_ = __h.get()->__ptr(); + // fix up __bucket_list_ + __bucket_list_[__chash] = __pn; + if (__h->__next_ != nullptr) + __bucket_list_[std::__constrain_hash(__h->__next_->__hash(), __bc)] = __h.get()->__ptr(); + } else { + __h->__next_ = __pn->__next_; + __pn->__next_ = static_cast<__next_pointer>(__h.get()); + } + __nd = static_cast<__next_pointer>(__h.release()); + // increment size + ++size(); + __inserted = true; + } __done: - return pair(iterator(__nd), __inserted); + return pair(iterator(__nd), __inserted); } template template pair::iterator, bool> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_impl(_Args&&... __args) -{ - __node_holder __h = __construct_node(std::forward<_Args>(__args)...); - pair __r = __node_insert_unique(__h.get()); - if (__r.second) - __h.release(); - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_impl(_Args&&... __args) { + __node_holder __h = __construct_node(std::forward<_Args>(__args)...); + pair __r = __node_insert_unique(__h.get()); + if (__r.second) + __h.release(); + return __r; } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_multi(_Args&&... __args) -{ - __node_holder __h = __construct_node(std::forward<_Args>(__args)...); - iterator __r = __node_insert_multi(__h.get()); - __h.release(); - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_multi(_Args&&... __args) { + __node_holder __h = __construct_node(std::forward<_Args>(__args)...); + iterator __r = __node_insert_multi(__h.get()); + __h.release(); + return __r; } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_hint_multi( - const_iterator __p, _Args&&... __args) -{ - __node_holder __h = __construct_node(std::forward<_Args>(__args)...); - iterator __r = __node_insert_multi(__p, __h.get()); - __h.release(); - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) { + __node_holder __h = __construct_node(std::forward<_Args>(__args)...); + iterator __r = __node_insert_multi(__p, __h.get()); + __h.release(); + return __r; } #if _LIBCPP_STD_VER >= 17 template template -_LIBCPP_HIDE_FROM_ABI -_InsertReturnType -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique( - _NodeHandle&& __nh) -{ - if (__nh.empty()) - return _InsertReturnType{end(), false, _NodeHandle()}; - pair __result = __node_insert_unique(__nh.__ptr_); - if (__result.second) - __nh.__release_ptr(); - return _InsertReturnType{__result.first, __result.second, std::move(__nh)}; +_LIBCPP_HIDE_FROM_ABI _InsertReturnType +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique(_NodeHandle&& __nh) { + if (__nh.empty()) + return _InsertReturnType{end(), false, _NodeHandle()}; + pair __result = __node_insert_unique(__nh.__ptr_); + if (__result.second) + __nh.__release_ptr(); + return _InsertReturnType{__result.first, __result.second, std::move(__nh)}; } template template -_LIBCPP_HIDE_FROM_ABI -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique( - const_iterator, _NodeHandle&& __nh) -{ - if (__nh.empty()) - return end(); - pair __result = __node_insert_unique(__nh.__ptr_); - if (__result.second) - __nh.__release_ptr(); - return __result.first; +_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_unique(const_iterator, _NodeHandle&& __nh) { + if (__nh.empty()) + return end(); + pair __result = __node_insert_unique(__nh.__ptr_); + if (__result.second) + __nh.__release_ptr(); + return __result.first; } template template -_LIBCPP_HIDE_FROM_ABI -_NodeHandle -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract( - key_type const& __key) -{ - iterator __i = find(__key); - if (__i == end()) - return _NodeHandle(); - return __node_handle_extract<_NodeHandle>(__i); +_LIBCPP_HIDE_FROM_ABI _NodeHandle +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract(key_type const& __key) { + iterator __i = find(__key); + if (__i == end()) + return _NodeHandle(); + return __node_handle_extract<_NodeHandle>(__i); } template template -_LIBCPP_HIDE_FROM_ABI -_NodeHandle -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract( - const_iterator __p) -{ - allocator_type __alloc(__node_alloc()); - return _NodeHandle(remove(__p).release(), __alloc); +_LIBCPP_HIDE_FROM_ABI _NodeHandle __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_extract(const_iterator __p) { + allocator_type __alloc(__node_alloc()); + return _NodeHandle(remove(__p).release(), __alloc); } template template -_LIBCPP_HIDE_FROM_ABI -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_unique( - _Table& __source) -{ - static_assert(is_same<__node, typename _Table::__node>::value, ""); - - for (typename _Table::iterator __it = __source.begin(); - __it != __source.end();) - { - __node_pointer __src_ptr = __it.__node_->__upcast(); - size_t __hash = hash_function()(__src_ptr->__get_value()); - __next_pointer __existing_node = - __node_insert_unique_prepare(__hash, __src_ptr->__get_value()); - auto __prev_iter = __it++; - if (__existing_node == nullptr) - { - (void)__source.remove(__prev_iter).release(); - __src_ptr->__hash_ = __hash; - __node_insert_unique_perform(__src_ptr); - } +_LIBCPP_HIDE_FROM_ABI void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_unique(_Table& __source) { + static_assert(is_same<__node, typename _Table::__node>::value, ""); + + for (typename _Table::iterator __it = __source.begin(); __it != __source.end();) { + __node_pointer __src_ptr = __it.__node_->__upcast(); + size_t __hash = hash_function()(__src_ptr->__get_value()); + __next_pointer __existing_node = __node_insert_unique_prepare(__hash, __src_ptr->__get_value()); + auto __prev_iter = __it++; + if (__existing_node == nullptr) { + (void)__source.remove(__prev_iter).release(); + __src_ptr->__hash_ = __hash; + __node_insert_unique_perform(__src_ptr); } + } } template template -_LIBCPP_HIDE_FROM_ABI -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi( - _NodeHandle&& __nh) -{ - if (__nh.empty()) - return end(); - iterator __result = __node_insert_multi(__nh.__ptr_); - __nh.__release_ptr(); - return __result; +_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi(_NodeHandle&& __nh) { + if (__nh.empty()) + return end(); + iterator __result = __node_insert_multi(__nh.__ptr_); + __nh.__release_ptr(); + return __result; } template template -_LIBCPP_HIDE_FROM_ABI -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi( - const_iterator __hint, _NodeHandle&& __nh) -{ - if (__nh.empty()) - return end(); - iterator __result = __node_insert_multi(__hint, __nh.__ptr_); - __nh.__release_ptr(); - return __result; +_LIBCPP_HIDE_FROM_ABI typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_insert_multi(const_iterator __hint, _NodeHandle&& __nh) { + if (__nh.empty()) + return end(); + iterator __result = __node_insert_multi(__hint, __nh.__ptr_); + __nh.__release_ptr(); + return __result; } template template -_LIBCPP_HIDE_FROM_ABI -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi( - _Table& __source) -{ - static_assert(is_same::value, ""); - - for (typename _Table::iterator __it = __source.begin(); - __it != __source.end();) - { - __node_pointer __src_ptr = __it.__node_->__upcast(); - size_t __src_hash = hash_function()(__src_ptr->__get_value()); - __next_pointer __pn = - __node_insert_multi_prepare(__src_hash, __src_ptr->__get_value()); - (void)__source.remove(__it++).release(); - __src_ptr->__hash_ = __src_hash; - __node_insert_multi_perform(__src_ptr, __pn); - } +_LIBCPP_HIDE_FROM_ABI void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi(_Table& __source) { + static_assert(is_same::value, ""); + + for (typename _Table::iterator __it = __source.begin(); __it != __source.end();) { + __node_pointer __src_ptr = __it.__node_->__upcast(); + size_t __src_hash = hash_function()(__src_ptr->__get_value()); + __next_pointer __pn = __node_insert_multi_prepare(__src_hash, __src_ptr->__get_value()); + (void)__source.remove(__it++).release(); + __src_ptr->__hash_ = __src_hash; + __node_insert_multi_perform(__src_ptr, __pn); + } } #endif // _LIBCPP_STD_VER >= 17 template template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n) -_LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK -{ - if (__n == 1) - __n = 2; - else if (__n & (__n - 1)) - __n = std::__next_prime(__n); - size_type __bc = bucket_count(); - if (__n > __bc) - __do_rehash<_UniqueKeys>(__n); - else if (__n < __bc) - { - __n = std::max - ( - __n, - std::__is_hash_power2(__bc) ? std::__next_hash_pow2(size_t(std::ceil(float(size()) / max_load_factor()))) : - std::__next_prime(size_t(std::ceil(float(size()) / max_load_factor()))) - ); - if (__n < __bc) - __do_rehash<_UniqueKeys>(__n); - } +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK { + if (__n == 1) + __n = 2; + else if (__n & (__n - 1)) + __n = std::__next_prime(__n); + size_type __bc = bucket_count(); + if (__n > __bc) + __do_rehash<_UniqueKeys>(__n); + else if (__n < __bc) { + __n = std::max( + __n, + std::__is_hash_power2(__bc) ? std::__next_hash_pow2(size_t(std::ceil(float(size()) / max_load_factor()))) + : std::__next_prime(size_t(std::ceil(float(size()) / max_load_factor())))); + if (__n < __bc) + __do_rehash<_UniqueKeys>(__n); + } } template template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc) -{ - __pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc(); - __bucket_list_.reset(__nbc > 0 ? - __pointer_alloc_traits::allocate(__npa, __nbc) : nullptr); - __bucket_list_.get_deleter().size() = __nbc; - if (__nbc > 0) - { - for (size_type __i = 0; __i < __nbc; ++__i) - __bucket_list_[__i] = nullptr; - __next_pointer __pp = __p1_.first().__ptr(); - __next_pointer __cp = __pp->__next_; - if (__cp != nullptr) - { - size_type __chash = std::__constrain_hash(__cp->__hash(), __nbc); +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc) { + __pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc(); + __bucket_list_.reset(__nbc > 0 ? __pointer_alloc_traits::allocate(__npa, __nbc) : nullptr); + __bucket_list_.get_deleter().size() = __nbc; + if (__nbc > 0) { + for (size_type __i = 0; __i < __nbc; ++__i) + __bucket_list_[__i] = nullptr; + __next_pointer __pp = __p1_.first().__ptr(); + __next_pointer __cp = __pp->__next_; + if (__cp != nullptr) { + size_type __chash = std::__constrain_hash(__cp->__hash(), __nbc); + __bucket_list_[__chash] = __pp; + size_type __phash = __chash; + for (__pp = __cp, void(), __cp = __cp->__next_; __cp != nullptr; __cp = __pp->__next_) { + __chash = std::__constrain_hash(__cp->__hash(), __nbc); + if (__chash == __phash) + __pp = __cp; + else { + if (__bucket_list_[__chash] == nullptr) { __bucket_list_[__chash] = __pp; - size_type __phash = __chash; - for (__pp = __cp, void(), __cp = __cp->__next_; __cp != nullptr; - __cp = __pp->__next_) - { - __chash = std::__constrain_hash(__cp->__hash(), __nbc); - if (__chash == __phash) - __pp = __cp; - else - { - if (__bucket_list_[__chash] == nullptr) - { - __bucket_list_[__chash] = __pp; - __pp = __cp; - __phash = __chash; - } - else - { - __next_pointer __np = __cp; - if _LIBCPP_CONSTEXPR_SINCE_CXX17 (!_UniqueKeys) - { - for (; __np->__next_ != nullptr && - key_eq()(__cp->__upcast()->__get_value(), - __np->__next_->__upcast()->__get_value()); - __np = __np->__next_) - ; - } - __pp->__next_ = __np->__next_; - __np->__next_ = __bucket_list_[__chash]->__next_; - __bucket_list_[__chash]->__next_ = __cp; - - } - } + __pp = __cp; + __phash = __chash; + } else { + __next_pointer __np = __cp; + if _LIBCPP_CONSTEXPR_SINCE_CXX17 (!_UniqueKeys) { + for (; __np->__next_ != nullptr && + key_eq()(__cp->__upcast()->__get_value(), __np->__next_->__upcast()->__get_value()); + __np = __np->__next_) + ; } + __pp->__next_ = __np->__next_; + __np->__next_ = __bucket_list_[__chash]->__next_; + __bucket_list_[__chash]->__next_ = __cp; + } } + } } + } } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) -{ - size_t __hash = hash_function()(__k); - size_type __bc = bucket_count(); - if (__bc != 0) - { - size_t __chash = std::__constrain_hash(__hash, __bc); - __next_pointer __nd = __bucket_list_[__chash]; - if (__nd != nullptr) - { - for (__nd = __nd->__next_; __nd != nullptr && - (__nd->__hash() == __hash - || std::__constrain_hash(__nd->__hash(), __bc) == __chash); - __nd = __nd->__next_) - { - if ((__nd->__hash() == __hash) - && key_eq()(__nd->__upcast()->__get_value(), __k)) - return iterator(__nd); - } - } +__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) { + size_t __hash = hash_function()(__k); + size_type __bc = bucket_count(); + if (__bc != 0) { + size_t __chash = std::__constrain_hash(__hash, __bc); + __next_pointer __nd = __bucket_list_[__chash]; + if (__nd != nullptr) { + for (__nd = __nd->__next_; + __nd != nullptr && (__nd->__hash() == __hash || std::__constrain_hash(__nd->__hash(), __bc) == __chash); + __nd = __nd->__next_) { + if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k)) + return iterator(__nd); + } } - return end(); + } + return end(); } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const -{ - size_t __hash = hash_function()(__k); - size_type __bc = bucket_count(); - if (__bc != 0) - { - size_t __chash = std::__constrain_hash(__hash, __bc); - __next_pointer __nd = __bucket_list_[__chash]; - if (__nd != nullptr) - { - for (__nd = __nd->__next_; __nd != nullptr && - (__hash == __nd->__hash() - || std::__constrain_hash(__nd->__hash(), __bc) == __chash); - __nd = __nd->__next_) - { - if ((__nd->__hash() == __hash) - && key_eq()(__nd->__upcast()->__get_value(), __k)) - return const_iterator(__nd); - } - } - +__hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const { + size_t __hash = hash_function()(__k); + size_type __bc = bucket_count(); + if (__bc != 0) { + size_t __chash = std::__constrain_hash(__hash, __bc); + __next_pointer __nd = __bucket_list_[__chash]; + if (__nd != nullptr) { + for (__nd = __nd->__next_; + __nd != nullptr && (__hash == __nd->__hash() || std::__constrain_hash(__nd->__hash(), __bc) == __chash); + __nd = __nd->__next_) { + if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k)) + return const_iterator(__nd); + } } - return end(); + } + return end(); } template -template +template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&& ...__args) -{ - static_assert(!__is_hash_value_type<_Args...>::value, - "Construct cannot be called with a hash value type"); - __node_allocator& __na = __node_alloc(); - __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); - - // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value - // held inside the node, since we need to use the allocator's construct() method for that. - // - // We don't use the allocator's construct() method to construct the node itself since the - // Cpp17FooInsertable named requirements don't require the allocator's construct() method - // to work on anything other than the value_type. - std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */0); - - // Now construct the value_type using the allocator's construct() method. - __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_Args>(__args)...); - __h.get_deleter().__value_constructed = true; - - __h->__hash_ = hash_function()(__h->__get_value()); - return __h; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&&... __args) { + static_assert(!__is_hash_value_type<_Args...>::value, "Construct cannot be called with a hash value type"); + __node_allocator& __na = __node_alloc(); + __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); + + // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value + // held inside the node, since we need to use the allocator's construct() method for that. + // + // We don't use the allocator's construct() method to construct the node itself since the + // Cpp17FooInsertable named requirements don't require the allocator's construct() method + // to work on anything other than the value_type. + std::__construct_at(std::addressof(*__h), /* next = */ nullptr, /* hash = */ 0); + + // Now construct the value_type using the allocator's construct() method. + __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_Args>(__args)...); + __h.get_deleter().__value_constructed = true; + + __h->__hash_ = hash_function()(__h->__get_value()); + return __h; } template -template +template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash( - size_t __hash, _First&& __f, _Rest&& ...__rest) -{ - static_assert(!__is_hash_value_type<_First, _Rest...>::value, - "Construct cannot be called with a hash value type"); - __node_allocator& __na = __node_alloc(); - __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); - std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */__hash); - __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), - std::forward<_First>(__f), - std::forward<_Rest>(__rest)...); - __h.get_deleter().__value_constructed = true; - return __h; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest) { + static_assert(!__is_hash_value_type<_First, _Rest...>::value, "Construct cannot be called with a hash value type"); + __node_allocator& __na = __node_alloc(); + __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); + std::__construct_at(std::addressof(*__h), /* next = */ nullptr, /* hash = */ __hash); + __node_traits::construct( + __na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_First>(__f), std::forward<_Rest>(__rest)...); + __h.get_deleter().__value_constructed = true; + return __h; } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __p) -{ - __next_pointer __np = __p.__node_; - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p != end(), - "unordered container::erase(iterator) called with a non-dereferenceable iterator"); - iterator __r(__np); - ++__r; - remove(__p); - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __p) { + __next_pointer __np = __p.__node_; + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __p != end(), "unordered container::erase(iterator) called with a non-dereferenceable iterator"); + iterator __r(__np); + ++__r; + remove(__p); + return __r; } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator -__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __first, - const_iterator __last) -{ - for (const_iterator __p = __first; __first != __last; __p = __first) - { - ++__first; - erase(__p); - } - __next_pointer __np = __last.__node_; - return iterator (__np); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __first, const_iterator __last) { + for (const_iterator __p = __first; __first != __last; __p = __first) { + ++__first; + erase(__p); + } + __next_pointer __np = __last.__node_; + return iterator(__np); } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_unique(const _Key& __k) -{ - iterator __i = find(__k); - if (__i == end()) - return 0; - erase(__i); - return 1; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_unique(const _Key& __k) { + iterator __i = find(__k); + if (__i == end()) + return 0; + erase(__i); + return 1; } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_multi(const _Key& __k) -{ - size_type __r = 0; - iterator __i = find(__k); - if (__i != end()) - { - iterator __e = end(); - do - { - erase(__i++); - ++__r; - } while (__i != __e && key_eq()(*__i, __k)); - } - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__erase_multi(const _Key& __k) { + size_type __r = 0; + iterator __i = find(__k); + if (__i != end()) { + iterator __e = end(); + do { + erase(__i++); + ++__r; + } while (__i != __e && key_eq()(*__i, __k)); + } + return __r; } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder -__hash_table<_Tp, _Hash, _Equal, _Alloc>::remove(const_iterator __p) _NOEXCEPT -{ - // current node - __next_pointer __cn = __p.__node_; - size_type __bc = bucket_count(); - size_t __chash = std::__constrain_hash(__cn->__hash(), __bc); - // find previous node - __next_pointer __pn = __bucket_list_[__chash]; - for (; __pn->__next_ != __cn; __pn = __pn->__next_) - ; - // Fix up __bucket_list_ - // if __pn is not in same bucket (before begin is not in same bucket) && - // if __cn->__next_ is not in same bucket (nullptr is not in same bucket) - if (__pn == __p1_.first().__ptr() - || std::__constrain_hash(__pn->__hash(), __bc) != __chash) - { - if (__cn->__next_ == nullptr - || std::__constrain_hash(__cn->__next_->__hash(), __bc) != __chash) - __bucket_list_[__chash] = nullptr; - } - // if __cn->__next_ is not in same bucket (nullptr is in same bucket) - if (__cn->__next_ != nullptr) - { - size_t __nhash = std::__constrain_hash(__cn->__next_->__hash(), __bc); - if (__nhash != __chash) - __bucket_list_[__nhash] = __pn; - } - // remove __cn - __pn->__next_ = __cn->__next_; - __cn->__next_ = nullptr; - --size(); - return __node_holder(__cn->__upcast(), _Dp(__node_alloc(), true)); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::remove(const_iterator __p) _NOEXCEPT { + // current node + __next_pointer __cn = __p.__node_; + size_type __bc = bucket_count(); + size_t __chash = std::__constrain_hash(__cn->__hash(), __bc); + // find previous node + __next_pointer __pn = __bucket_list_[__chash]; + for (; __pn->__next_ != __cn; __pn = __pn->__next_) + ; + // Fix up __bucket_list_ + // if __pn is not in same bucket (before begin is not in same bucket) && + // if __cn->__next_ is not in same bucket (nullptr is not in same bucket) + if (__pn == __p1_.first().__ptr() || std::__constrain_hash(__pn->__hash(), __bc) != __chash) { + if (__cn->__next_ == nullptr || std::__constrain_hash(__cn->__next_->__hash(), __bc) != __chash) + __bucket_list_[__chash] = nullptr; + } + // if __cn->__next_ is not in same bucket (nullptr is in same bucket) + if (__cn->__next_ != nullptr) { + size_t __nhash = std::__constrain_hash(__cn->__next_->__hash(), __bc); + if (__nhash != __chash) + __bucket_list_[__nhash] = __pn; + } + // remove __cn + __pn->__next_ = __cn->__next_; + __cn->__next_ = nullptr; + --size(); + return __node_holder(__cn->__upcast(), _Dp(__node_alloc(), true)); } template template -inline -typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_unique(const _Key& __k) const -{ - return static_cast(find(__k) != end()); +inline typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_unique(const _Key& __k) const { + return static_cast(find(__k) != end()); } template template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_multi(const _Key& __k) const -{ - size_type __r = 0; - const_iterator __i = find(__k); - if (__i != end()) - { - const_iterator __e = end(); - do - { - ++__i; - ++__r; - } while (__i != __e && key_eq()(*__i, __k)); - } - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__count_multi(const _Key& __k) const { + size_type __r = 0; + const_iterator __i = find(__k); + if (__i != end()) { + const_iterator __e = end(); + do { + ++__i; + ++__r; + } while (__i != __e && key_eq()(*__i, __k)); + } + return __r; } template template pair::iterator, typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique( - const _Key& __k) -{ - iterator __i = find(__k); - iterator __j = __i; - if (__i != end()) - ++__j; - return pair(__i, __j); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique(const _Key& __k) { + iterator __i = find(__k); + iterator __j = __i; + if (__i != end()) + ++__j; + return pair(__i, __j); } template template pair::const_iterator, typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique( - const _Key& __k) const -{ - const_iterator __i = find(__k); - const_iterator __j = __i; - if (__i != end()) - ++__j; - return pair(__i, __j); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_unique(const _Key& __k) const { + const_iterator __i = find(__k); + const_iterator __j = __i; + if (__i != end()) + ++__j; + return pair(__i, __j); } template template pair::iterator, typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi( - const _Key& __k) -{ - iterator __i = find(__k); - iterator __j = __i; - if (__i != end()) - { - iterator __e = end(); - do - { - ++__j; - } while (__j != __e && key_eq()(*__j, __k)); - } - return pair(__i, __j); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi(const _Key& __k) { + iterator __i = find(__k); + iterator __j = __i; + if (__i != end()) { + iterator __e = end(); + do { + ++__j; + } while (__j != __e && key_eq()(*__j, __k)); + } + return pair(__i, __j); } template template pair::const_iterator, typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator> -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi( - const _Key& __k) const -{ - const_iterator __i = find(__k); - const_iterator __j = __i; - if (__i != end()) - { - const_iterator __e = end(); - do - { - ++__j; - } while (__j != __e && key_eq()(*__j, __k)); - } - return pair(__i, __j); +__hash_table<_Tp, _Hash, _Equal, _Alloc>::__equal_range_multi(const _Key& __k) const { + const_iterator __i = find(__k); + const_iterator __j = __i; + if (__i != end()) { + const_iterator __e = end(); + do { + ++__j; + } while (__j != __e && key_eq()(*__j, __k)); + } + return pair(__i, __j); } template -void -__hash_table<_Tp, _Hash, _Equal, _Alloc>::swap(__hash_table& __u) +void __hash_table<_Tp, _Hash, _Equal, _Alloc>::swap(__hash_table& __u) #if _LIBCPP_STD_VER <= 11 - _NOEXCEPT_( - __is_nothrow_swappable::value && __is_nothrow_swappable::value - && (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value - || __is_nothrow_swappable<__pointer_allocator>::value) - && (!__node_traits::propagate_on_container_swap::value - || __is_nothrow_swappable<__node_allocator>::value) - ) + _NOEXCEPT_(__is_nothrow_swappable::value&& __is_nothrow_swappable::value && + (!allocator_traits<__pointer_allocator>::propagate_on_container_swap::value || + __is_nothrow_swappable<__pointer_allocator>::value) && + (!__node_traits::propagate_on_container_swap::value || __is_nothrow_swappable<__node_allocator>::value)) #else - _NOEXCEPT_(__is_nothrow_swappable::value && __is_nothrow_swappable::value) + _NOEXCEPT_(__is_nothrow_swappable::value&& __is_nothrow_swappable::value) #endif { - _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__node_traits::propagate_on_container_swap::value || - this->__node_alloc() == __u.__node_alloc(), - "unordered container::swap: Either propagate_on_container_swap " - "must be true or the allocators must compare equal"); - { + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( + __node_traits::propagate_on_container_swap::value || this->__node_alloc() == __u.__node_alloc(), + "unordered container::swap: Either propagate_on_container_swap " + "must be true or the allocators must compare equal"); + { __node_pointer_pointer __npp = __bucket_list_.release(); __bucket_list_.reset(__u.__bucket_list_.release()); __u.__bucket_list_.reset(__npp); - } - std::swap(__bucket_list_.get_deleter().size(), __u.__bucket_list_.get_deleter().size()); - std::__swap_allocator(__bucket_list_.get_deleter().__alloc(), - __u.__bucket_list_.get_deleter().__alloc()); - std::__swap_allocator(__node_alloc(), __u.__node_alloc()); - std::swap(__p1_.first().__next_, __u.__p1_.first().__next_); - __p2_.swap(__u.__p2_); - __p3_.swap(__u.__p3_); - if (size() > 0) - __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - if (__u.size() > 0) - __u.__bucket_list_[std::__constrain_hash(__u.__p1_.first().__next_->__hash(), __u.bucket_count())] = - __u.__p1_.first().__ptr(); + } + std::swap(__bucket_list_.get_deleter().size(), __u.__bucket_list_.get_deleter().size()); + std::__swap_allocator(__bucket_list_.get_deleter().__alloc(), __u.__bucket_list_.get_deleter().__alloc()); + std::__swap_allocator(__node_alloc(), __u.__node_alloc()); + std::swap(__p1_.first().__next_, __u.__p1_.first().__next_); + __p2_.swap(__u.__p2_); + __p3_.swap(__u.__p3_); + if (size() > 0) + __bucket_list_[std::__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = __p1_.first().__ptr(); + if (__u.size() > 0) + __u.__bucket_list_[std::__constrain_hash(__u.__p1_.first().__next_->__hash(), __u.bucket_count())] = + __u.__p1_.first().__ptr(); } template typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::size_type -__hash_table<_Tp, _Hash, _Equal, _Alloc>::bucket_size(size_type __n) const -{ - _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < bucket_count(), - "unordered container::bucket_size(n) called with n >= bucket_count()"); - __next_pointer __np = __bucket_list_[__n]; - size_type __bc = bucket_count(); - size_type __r = 0; - if (__np != nullptr) - { - for (__np = __np->__next_; __np != nullptr && - std::__constrain_hash(__np->__hash(), __bc) == __n; - __np = __np->__next_, (void) ++__r) - ; - } - return __r; +__hash_table<_Tp, _Hash, _Equal, _Alloc>::bucket_size(size_type __n) const { + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( + __n < bucket_count(), "unordered container::bucket_size(n) called with n >= bucket_count()"); + __next_pointer __np = __bucket_list_[__n]; + size_type __bc = bucket_count(); + size_type __r = 0; + if (__np != nullptr) { + for (__np = __np->__next_; __np != nullptr && std::__constrain_hash(__np->__hash(), __bc) == __n; + __np = __np->__next_, (void)++__r) + ; + } + return __r; } template -inline _LIBCPP_HIDE_FROM_ABI -void -swap(__hash_table<_Tp, _Hash, _Equal, _Alloc>& __x, - __hash_table<_Tp, _Hash, _Equal, _Alloc>& __y) - _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) -{ - __x.swap(__y); +inline _LIBCPP_HIDE_FROM_ABI void +swap(__hash_table<_Tp, _Hash, _Equal, _Alloc>& __x, __hash_table<_Tp, _Hash, _Equal, _Alloc>& __y) + _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) { + __x.swap(__y); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ios/fpos.h b/libcxx/include/__ios/fpos.h index ae578bdbb916c..1af1e23ee50da 100644 --- a/libcxx/include/__ios/fpos.h +++ b/libcxx/include/__ios/fpos.h @@ -57,20 +57,17 @@ class _LIBCPP_TEMPLATE_VIS fpos { }; template -inline _LIBCPP_HIDE_FROM_ABI -streamoff operator-(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { +inline _LIBCPP_HIDE_FROM_ABI streamoff operator-(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { return streamoff(__x) - streamoff(__y); } template -inline _LIBCPP_HIDE_FROM_ABI -bool operator==(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { +inline _LIBCPP_HIDE_FROM_ABI bool operator==(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { return streamoff(__x) == streamoff(__y); } template -inline _LIBCPP_HIDE_FROM_ABI -bool operator!=(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { +inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const fpos<_StateT>& __x, const fpos<_StateT>& __y) { return streamoff(__x) != streamoff(__y); } diff --git a/libcxx/include/__iterator/access.h b/libcxx/include/__iterator/access.h index 3ab3955e12a5b..5c6090eeb40c5 100644 --- a/libcxx/include/__iterator/access.h +++ b/libcxx/include/__iterator/access.h @@ -32,85 +32,60 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* end(_Tp (&__array)[_Np]) _NOEXCEPT #if !defined(_LIBCPP_CXX03_LANG) template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto -begin(_Cp& __c) -> decltype(__c.begin()) -{ - return __c.begin(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto begin(_Cp& __c) -> decltype(__c.begin()) { + return __c.begin(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto -begin(const _Cp& __c) -> decltype(__c.begin()) -{ - return __c.begin(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto begin(const _Cp& __c) -> decltype(__c.begin()) { + return __c.begin(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto -end(_Cp& __c) -> decltype(__c.end()) -{ - return __c.end(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto end(_Cp& __c) -> decltype(__c.end()) { + return __c.end(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto -end(const _Cp& __c) -> decltype(__c.end()) -{ - return __c.end(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto end(const _Cp& __c) -> decltype(__c.end()) { + return __c.end(); } -#if _LIBCPP_STD_VER >= 14 +# if _LIBCPP_STD_VER >= 14 template _LIBCPP_HIDE_FROM_ABI constexpr auto cbegin(const _Cp& __c) noexcept(noexcept(std::begin(__c))) -> decltype(std::begin(__c)) { - return std::begin(__c); + return std::begin(__c); } template _LIBCPP_HIDE_FROM_ABI constexpr auto cend(const _Cp& __c) noexcept(noexcept(std::end(__c))) -> decltype(std::end(__c)) { - return std::end(__c); + return std::end(__c); } -#endif - +# endif -#else // defined(_LIBCPP_CXX03_LANG) +#else // defined(_LIBCPP_CXX03_LANG) template -_LIBCPP_HIDE_FROM_ABI -typename _Cp::iterator -begin(_Cp& __c) -{ - return __c.begin(); +_LIBCPP_HIDE_FROM_ABI typename _Cp::iterator begin(_Cp& __c) { + return __c.begin(); } template -_LIBCPP_HIDE_FROM_ABI -typename _Cp::const_iterator -begin(const _Cp& __c) -{ - return __c.begin(); +_LIBCPP_HIDE_FROM_ABI typename _Cp::const_iterator begin(const _Cp& __c) { + return __c.begin(); } template -_LIBCPP_HIDE_FROM_ABI -typename _Cp::iterator -end(_Cp& __c) -{ - return __c.end(); +_LIBCPP_HIDE_FROM_ABI typename _Cp::iterator end(_Cp& __c) { + return __c.end(); } template -_LIBCPP_HIDE_FROM_ABI -typename _Cp::const_iterator -end(const _Cp& __c) -{ - return __c.end(); +_LIBCPP_HIDE_FROM_ABI typename _Cp::const_iterator end(const _Cp& __c) { + return __c.end(); } #endif // !defined(_LIBCPP_CXX03_LANG) diff --git a/libcxx/include/__iterator/advance.h b/libcxx/include/__iterator/advance.h index 76938970b0c34..64c8d249f78f3 100644 --- a/libcxx/include/__iterator/advance.h +++ b/libcxx/include/__iterator/advance.h @@ -35,15 +35,15 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -void __advance(_InputIter& __i, typename iterator_traits<_InputIter>::difference_type __n, input_iterator_tag) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void +__advance(_InputIter& __i, typename iterator_traits<_InputIter>::difference_type __n, input_iterator_tag) { for (; __n > 0; --__n) ++__i; } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -void __advance(_BiDirIter& __i, typename iterator_traits<_BiDirIter>::difference_type __n, bidirectional_iterator_tag) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void +__advance(_BiDirIter& __i, typename iterator_traits<_BiDirIter>::difference_type __n, bidirectional_iterator_tag) { if (__n >= 0) for (; __n > 0; --__n) ++__i; @@ -53,17 +53,16 @@ void __advance(_BiDirIter& __i, typename iterator_traits<_BiDirIter>::difference } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -void __advance(_RandIter& __i, typename iterator_traits<_RandIter>::difference_type __n, random_access_iterator_tag) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void +__advance(_RandIter& __i, typename iterator_traits<_RandIter>::difference_type __n, random_access_iterator_tag) { __i += __n; } -template < - class _InputIter, class _Distance, - class _IntegralDistance = decltype(std::__convert_to_integral(std::declval<_Distance>())), - class = __enable_if_t::value> > -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -void advance(_InputIter& __i, _Distance __orig_n) { +template < class _InputIter, + class _Distance, + class _IntegralDistance = decltype(std::__convert_to_integral(std::declval<_Distance>())), + class = __enable_if_t::value> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void advance(_InputIter& __i, _Distance __orig_n) { typedef typename iterator_traits<_InputIter>::difference_type _Difference; _Difference __n = static_cast<_Difference>(std::__convert_to_integral(__orig_n)); _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, @@ -81,8 +80,7 @@ namespace __advance { struct __fn { private: template - _LIBCPP_HIDE_FROM_ABI - static constexpr void __advance_forward(_Ip& __i, iter_difference_t<_Ip> __n) { + _LIBCPP_HIDE_FROM_ABI static constexpr void __advance_forward(_Ip& __i, iter_difference_t<_Ip> __n) { while (__n > 0) { --__n; ++__i; @@ -90,8 +88,7 @@ struct __fn { } template - _LIBCPP_HIDE_FROM_ABI - static constexpr void __advance_backward(_Ip& __i, iter_difference_t<_Ip> __n) { + _LIBCPP_HIDE_FROM_ABI static constexpr void __advance_backward(_Ip& __i, iter_difference_t<_Ip> __n) { while (__n < 0) { ++__n; --__i; @@ -101,10 +98,9 @@ struct __fn { public: // Preconditions: If `I` does not model `bidirectional_iterator`, `n` is not negative. template - _LIBCPP_HIDE_FROM_ABI - constexpr void operator()(_Ip& __i, iter_difference_t<_Ip> __n) const { - _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0 || bidirectional_iterator<_Ip>, - "If `n < 0`, then `bidirectional_iterator` must be true."); + _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Ip& __i, iter_difference_t<_Ip> __n) const { + _LIBCPP_ASSERT_UNCATEGORIZED( + __n >= 0 || bidirectional_iterator<_Ip>, "If `n < 0`, then `bidirectional_iterator` must be true."); // If `I` models `random_access_iterator`, equivalent to `i += n`. if constexpr (random_access_iterator<_Ip>) { @@ -123,14 +119,16 @@ struct __fn { } } - // Preconditions: Either `assignable_from || sized_sentinel_for` is modeled, or [i, bound_sentinel) denotes a range. + // Preconditions: Either `assignable_from || sized_sentinel_for` is modeled, or [i, bound_sentinel) + // denotes a range. template _Sp> _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Ip& __i, _Sp __bound_sentinel) const { // If `I` and `S` model `assignable_from`, equivalent to `i = std::move(bound_sentinel)`. if constexpr (assignable_from<_Ip&, _Sp>) { __i = std::move(__bound_sentinel); } - // Otherwise, if `S` and `I` model `sized_sentinel_for`, equivalent to `ranges::advance(i, bound_sentinel - i)`. + // Otherwise, if `S` and `I` model `sized_sentinel_for`, equivalent to `ranges::advance(i, bound_sentinel - + // i)`. else if constexpr (sized_sentinel_for<_Sp, _Ip>) { (*this)(__i, __bound_sentinel - __i); } @@ -145,22 +143,19 @@ struct __fn { // Preconditions: // * If `n > 0`, [i, bound_sentinel) denotes a range. // * If `n == 0`, [i, bound_sentinel) or [bound_sentinel, i) denotes a range. - // * If `n < 0`, [bound_sentinel, i) denotes a range, `I` models `bidirectional_iterator`, and `I` and `S` model `same_as`. + // * If `n < 0`, [bound_sentinel, i) denotes a range, `I` models `bidirectional_iterator`, and `I` and `S` model + // `same_as`. // Returns: `n - M`, where `M` is the difference between the ending and starting position. template _Sp> - _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip& __i, iter_difference_t<_Ip> __n, - _Sp __bound_sentinel) const { + _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> + operator()(_Ip& __i, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const { _LIBCPP_ASSERT_UNCATEGORIZED((__n >= 0) || (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>), "If `n < 0`, then `bidirectional_iterator && same_as` must be true."); // If `S` and `I` model `sized_sentinel_for`: if constexpr (sized_sentinel_for<_Sp, _Ip>) { // If |n| >= |bound_sentinel - i|, equivalent to `ranges::advance(i, bound_sentinel)`. // __magnitude_geq(a, b) returns |a| >= |b|, assuming they have the same sign. - auto __magnitude_geq = [](auto __a, auto __b) { - return __a == 0 ? __b == 0 : - __a > 0 ? __a >= __b : - __a <= __b; - }; + auto __magnitude_geq = [](auto __a, auto __b) { return __a == 0 ? __b == 0 : __a > 0 ? __a >= __b : __a <= __b; }; if (const auto __m = __bound_sentinel - __i; __magnitude_geq(__n, __m)) { (*this)(__i, __bound_sentinel); return __n - __m; @@ -194,7 +189,7 @@ struct __fn { } // namespace __advance inline namespace __cpo { - inline constexpr auto advance = __advance::__fn{}; +inline constexpr auto advance = __advance::__fn{}; } // namespace __cpo } // namespace ranges diff --git a/libcxx/include/__iterator/back_insert_iterator.h b/libcxx/include/__iterator/back_insert_iterator.h index f022d3bdf8dd1..6d3dd4b12966f 100644 --- a/libcxx/include/__iterator/back_insert_iterator.h +++ b/libcxx/include/__iterator/back_insert_iterator.h @@ -33,42 +33,49 @@ class _LIBCPP_TEMPLATE_VIS back_insert_iterator : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + protected: - _Container* container; + _Container* container; + public: - typedef output_iterator_tag iterator_category; - typedef void value_type; + typedef output_iterator_tag iterator_category; + typedef void value_type; #if _LIBCPP_STD_VER >= 20 - typedef ptrdiff_t difference_type; + typedef ptrdiff_t difference_type; #else - typedef void difference_type; + typedef void difference_type; #endif - typedef void pointer; - typedef void reference; - typedef _Container container_type; + typedef void pointer; + typedef void reference; + typedef _Container container_type; - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit back_insert_iterator(_Container& __x) : container(std::addressof(__x)) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator=(const typename _Container::value_type& __value) - {container->push_back(__value); return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit back_insert_iterator(_Container& __x) + : container(std::addressof(__x)) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& + operator=(const typename _Container::value_type& __value) { + container->push_back(__value); + return *this; + } #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator=(typename _Container::value_type&& __value) - {container->push_back(std::move(__value)); return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& + operator=(typename _Container::value_type&& __value) { + container->push_back(std::move(__value)); + return *this; + } #endif // _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator*() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator++() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator operator++(int) {return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator*() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator& operator++() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator operator++(int) { return *this; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Container* __get_container() const { return container; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Container* __get_container() const { return container; } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(back_insert_iterator); template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -back_insert_iterator<_Container> -back_inserter(_Container& __x) -{ - return back_insert_iterator<_Container>(__x); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 back_insert_iterator<_Container> +back_inserter(_Container& __x) { + return back_insert_iterator<_Container>(__x); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/common_iterator.h b/libcxx/include/__iterator/common_iterator.h index d7c308d8da00e..cc49d62cd04dd 100644 --- a/libcxx/include/__iterator/common_iterator.h +++ b/libcxx/include/__iterator/common_iterator.h @@ -41,13 +41,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template +template concept __can_use_postfix_proxy = - constructible_from, iter_reference_t<_Iter>> && - move_constructible>; + constructible_from, iter_reference_t<_Iter>> && move_constructible>; -template _Sent> - requires (!same_as<_Iter, _Sent> && copyable<_Iter>) +template _Sent> + requires(!same_as<_Iter, _Sent> && copyable<_Iter>) class common_iterator { struct __proxy { _LIBCPP_HIDE_FROM_ABI constexpr const iter_value_t<_Iter>* operator->() const noexcept { @@ -57,42 +56,42 @@ class common_iterator { }; struct __postfix_proxy { - _LIBCPP_HIDE_FROM_ABI constexpr const iter_value_t<_Iter>& operator*() const noexcept { - return __value_; - } + _LIBCPP_HIDE_FROM_ABI constexpr const iter_value_t<_Iter>& operator*() const noexcept { return __value_; } iter_value_t<_Iter> __value_; }; variant<_Iter, _Sent> __hold_; - template _OtherSent> - requires (!same_as<_OtherIter, _OtherSent> && copyable<_OtherIter>) + template _OtherSent> + requires(!same_as<_OtherIter, _OtherSent> && copyable<_OtherIter>) friend class common_iterator; public: - _LIBCPP_HIDE_FROM_ABI common_iterator() requires default_initializable<_Iter> = default; + _LIBCPP_HIDE_FROM_ABI common_iterator() + requires default_initializable<_Iter> + = default; _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(_Iter __i) : __hold_(in_place_type<_Iter>, std::move(__i)) {} _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(_Sent __s) : __hold_(in_place_type<_Sent>, std::move(__s)) {} - template + template requires convertible_to && convertible_to _LIBCPP_HIDE_FROM_ABI constexpr common_iterator(const common_iterator<_I2, _S2>& __other) - : __hold_([&]() -> variant<_Iter, _Sent> { - _LIBCPP_ASSERT_UNCATEGORIZED(!__other.__hold_.valueless_by_exception(), - "Attempted to construct from a valueless common_iterator"); - if (__other.__hold_.index() == 0) - return variant<_Iter, _Sent>{in_place_index<0>, std::__unchecked_get<0>(__other.__hold_)}; - return variant<_Iter, _Sent>{in_place_index<1>, std::__unchecked_get<1>(__other.__hold_)}; - }()) {} - - template + : __hold_([&]() -> variant<_Iter, _Sent> { + _LIBCPP_ASSERT_UNCATEGORIZED( + !__other.__hold_.valueless_by_exception(), "Attempted to construct from a valueless common_iterator"); + if (__other.__hold_.index() == 0) + return variant<_Iter, _Sent>{in_place_index<0>, std::__unchecked_get<0>(__other.__hold_)}; + return variant<_Iter, _Sent>{in_place_index<1>, std::__unchecked_get<1>(__other.__hold_)}; + }()) {} + + template requires convertible_to && convertible_to && assignable_from<_Iter&, const _I2&> && assignable_from<_Sent&, const _S2&> _LIBCPP_HIDE_FROM_ABI common_iterator& operator=(const common_iterator<_I2, _S2>& __other) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__other.__hold_.valueless_by_exception(), - "Attempted to assign from a valueless common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + !__other.__hold_.valueless_by_exception(), "Attempted to assign from a valueless common_iterator"); - auto __idx = __hold_.index(); + auto __idx = __hold_.index(); auto __other_idx = __other.__hold_.index(); // If they're the same index, just assign. @@ -110,31 +109,29 @@ class common_iterator { return *this; } - _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() - { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__hold_), - "Attempted to dereference a non-dereferenceable common_iterator"); + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() { + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator"); return *std::__unchecked_get<_Iter>(__hold_); } _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const requires __dereferenceable { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__hold_), - "Attempted to dereference a non-dereferenceable common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator"); return *std::__unchecked_get<_Iter>(__hold_); } - template + template _LIBCPP_HIDE_FROM_ABI decltype(auto) operator->() const - requires indirectly_readable && - (requires(const _I2& __i) { __i.operator->(); } || - is_reference_v> || - constructible_from, iter_reference_t<_I2>>) + requires indirectly_readable && (requires(const _I2& __i) { + __i.operator->(); + } || is_reference_v> || constructible_from, iter_reference_t<_I2>>) { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__hold_), - "Attempted to dereference a non-dereferenceable common_iterator"); - if constexpr (is_pointer_v<_Iter> || requires(const _Iter& __i) { __i.operator->(); }) { + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__hold_), "Attempted to dereference a non-dereferenceable common_iterator"); + if constexpr (is_pointer_v<_Iter> || requires(const _Iter& __i) { __i.operator->(); }) { return std::__unchecked_get<_Iter>(__hold_); } else if constexpr (is_reference_v>) { auto&& __tmp = *std::__unchecked_get<_Iter>(__hold_); @@ -145,20 +142,22 @@ class common_iterator { } _LIBCPP_HIDE_FROM_ABI common_iterator& operator++() { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__hold_), - "Attempted to increment a non-dereferenceable common_iterator"); - ++std::__unchecked_get<_Iter>(__hold_); return *this; + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__hold_), "Attempted to increment a non-dereferenceable common_iterator"); + ++std::__unchecked_get<_Iter>(__hold_); + return *this; } _LIBCPP_HIDE_FROM_ABI decltype(auto) operator++(int) { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__hold_), - "Attempted to increment a non-dereferenceable common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__hold_), "Attempted to increment a non-dereferenceable common_iterator"); if constexpr (forward_iterator<_Iter>) { auto __tmp = *this; ++*this; return __tmp; - } else if constexpr (requires (_Iter& __i) { { *__i++ } -> __can_reference; } || - !__can_use_postfix_proxy<_Iter>) { + } else if constexpr (requires(_Iter& __i) { + { *__i++ } -> __can_reference; + } || !__can_use_postfix_proxy<_Iter>) { return std::__unchecked_get<_Iter>(__hold_)++; } else { auto __p = __postfix_proxy{**this}; @@ -167,14 +166,14 @@ class common_iterator { } } - template _S2> + template _S2> requires sentinel_for<_Sent, _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__x.__hold_.valueless_by_exception(), - "Attempted to compare a valueless common_iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(!__y.__hold_.valueless_by_exception(), - "Attempted to compare a valueless common_iterator"); + _LIBCPP_HIDE_FROM_ABI friend constexpr bool + operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { + _LIBCPP_ASSERT_UNCATEGORIZED( + !__x.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + !__y.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator"); auto __x_index = __x.__hold_.index(); auto __y_index = __y.__hold_.index(); @@ -188,14 +187,14 @@ class common_iterator { return std::__unchecked_get<_Sent>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_); } - template _S2> + template _S2> requires sentinel_for<_Sent, _I2> && equality_comparable_with<_Iter, _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__x.__hold_.valueless_by_exception(), - "Attempted to compare a valueless common_iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(!__y.__hold_.valueless_by_exception(), - "Attempted to compare a valueless common_iterator"); + _LIBCPP_HIDE_FROM_ABI friend constexpr bool + operator==(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { + _LIBCPP_ASSERT_UNCATEGORIZED( + !__x.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + !__y.__hold_.valueless_by_exception(), "Attempted to compare a valueless common_iterator"); auto __x_index = __x.__hold_.index(); auto __y_index = __y.__hold_.index(); @@ -204,22 +203,22 @@ class common_iterator { return true; if (__x_index == 0 && __y_index == 0) - return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_); + return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_); if (__x_index == 0) - return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_S2>(__y.__hold_); + return std::__unchecked_get<_Iter>(__x.__hold_) == std::__unchecked_get<_S2>(__y.__hold_); - return std::__unchecked_get<_Sent>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_); + return std::__unchecked_get<_Sent>(__x.__hold_) == std::__unchecked_get<_I2>(__y.__hold_); } - template _I2, sized_sentinel_for<_Iter> _S2> + template _I2, sized_sentinel_for<_Iter> _S2> requires sized_sentinel_for<_Sent, _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr iter_difference_t<_I2> operator-(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__x.__hold_.valueless_by_exception(), - "Attempted to subtract from a valueless common_iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(!__y.__hold_.valueless_by_exception(), - "Attempted to subtract a valueless common_iterator"); + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_I2> + operator-(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) { + _LIBCPP_ASSERT_UNCATEGORIZED( + !__x.__hold_.valueless_by_exception(), "Attempted to subtract from a valueless common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + !__y.__hold_.valueless_by_exception(), "Attempted to subtract a valueless common_iterator"); auto __x_index = __x.__hold_.index(); auto __y_index = __y.__hold_.index(); @@ -228,73 +227,67 @@ class common_iterator { return 0; if (__x_index == 0 && __y_index == 0) - return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_I2>(__y.__hold_); + return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_I2>(__y.__hold_); if (__x_index == 0) - return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_S2>(__y.__hold_); + return std::__unchecked_get<_Iter>(__x.__hold_) - std::__unchecked_get<_S2>(__y.__hold_); return std::__unchecked_get<_Sent>(__x.__hold_) - std::__unchecked_get<_I2>(__y.__hold_); } - _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> iter_move(const common_iterator& __i) - noexcept(noexcept(ranges::iter_move(std::declval()))) - requires input_iterator<_Iter> + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> + iter_move(const common_iterator& __i) noexcept(noexcept(ranges::iter_move(std::declval()))) + requires input_iterator<_Iter> { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__i.__hold_), - "Attempted to iter_move a non-dereferenceable common_iterator"); - return ranges::iter_move( std::__unchecked_get<_Iter>(__i.__hold_)); + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__i.__hold_), "Attempted to iter_move a non-dereferenceable common_iterator"); + return ranges::iter_move(std::__unchecked_get<_Iter>(__i.__hold_)); } - template _I2, class _S2> - _LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) - noexcept(noexcept(ranges::iter_swap(std::declval(), std::declval()))) - { - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_Iter>(__x.__hold_), - "Attempted to iter_swap a non-dereferenceable common_iterator"); - _LIBCPP_ASSERT_UNCATEGORIZED(std::holds_alternative<_I2>(__y.__hold_), - "Attempted to iter_swap a non-dereferenceable common_iterator"); + template _I2, class _S2> + _LIBCPP_HIDE_FROM_ABI friend constexpr void + iter_swap(const common_iterator& __x, const common_iterator<_I2, _S2>& __y) noexcept( + noexcept(ranges::iter_swap(std::declval(), std::declval()))) { + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_Iter>(__x.__hold_), "Attempted to iter_swap a non-dereferenceable common_iterator"); + _LIBCPP_ASSERT_UNCATEGORIZED( + std::holds_alternative<_I2>(__y.__hold_), "Attempted to iter_swap a non-dereferenceable common_iterator"); return ranges::iter_swap(std::__unchecked_get<_Iter>(__x.__hold_), std::__unchecked_get<_I2>(__y.__hold_)); } }; -template +template struct incrementable_traits> { using difference_type = iter_difference_t<_Iter>; }; -template -concept __denotes_forward_iter = - requires { typename iterator_traits<_Iter>::iterator_category; } && - derived_from::iterator_category, forward_iterator_tag>; +template +concept __denotes_forward_iter = requires { + typename iterator_traits<_Iter>::iterator_category; +} && derived_from::iterator_category, forward_iterator_tag>; -template -concept __common_iter_has_ptr_op = requires(const common_iterator<_Iter, _Sent>& __a) { - __a.operator->(); -}; +template +concept __common_iter_has_ptr_op = requires(const common_iterator<_Iter, _Sent>& __a) { __a.operator->(); }; -template +template struct __arrow_type_or_void { - using type = void; + using type = void; }; -template +template requires __common_iter_has_ptr_op<_Iter, _Sent> struct __arrow_type_or_void<_Iter, _Sent> { - using type = decltype(std::declval&>().operator->()); + using type = decltype(std::declval&>().operator->()); }; -template +template struct iterator_traits> { - using iterator_concept = _If, - forward_iterator_tag, - input_iterator_tag>; - using iterator_category = _If<__denotes_forward_iter<_Iter>, - forward_iterator_tag, - input_iterator_tag>; - using pointer = typename __arrow_type_or_void<_Iter, _Sent>::type; - using value_type = iter_value_t<_Iter>; - using difference_type = iter_difference_t<_Iter>; - using reference = iter_reference_t<_Iter>; + using iterator_concept = _If, forward_iterator_tag, input_iterator_tag>; + using iterator_category = _If<__denotes_forward_iter<_Iter>, forward_iterator_tag, input_iterator_tag>; + using pointer = typename __arrow_type_or_void<_Iter, _Sent>::type; + using value_type = iter_value_t<_Iter>; + using difference_type = iter_difference_t<_Iter>; + using reference = iter_reference_t<_Iter>; }; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/concepts.h b/libcxx/include/__iterator/concepts.h index 614de85847a09..afb7b821a99ce 100644 --- a/libcxx/include/__iterator/concepts.h +++ b/libcxx/include/__iterator/concepts.h @@ -49,246 +49,198 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 // [iterator.concept.readable] -template +template concept __indirectly_readable_impl = - requires(const _In __i) { - typename iter_value_t<_In>; - typename iter_reference_t<_In>; - typename iter_rvalue_reference_t<_In>; - { *__i } -> same_as>; - { ranges::iter_move(__i) } -> same_as>; - } && - common_reference_with&&, iter_value_t<_In>&> && - common_reference_with&&, iter_rvalue_reference_t<_In>&&> && - common_reference_with&&, const iter_value_t<_In>&>; - -template + requires(const _In __i) { + typename iter_value_t<_In>; + typename iter_reference_t<_In>; + typename iter_rvalue_reference_t<_In>; + { *__i } -> same_as>; + { ranges::iter_move(__i) } -> same_as>; + } && common_reference_with&&, iter_value_t<_In>&> && + common_reference_with&&, iter_rvalue_reference_t<_In>&&> && + common_reference_with&&, const iter_value_t<_In>&>; + +template concept indirectly_readable = __indirectly_readable_impl>; -template +template using iter_common_reference_t = common_reference_t, iter_value_t<_Tp>&>; // [iterator.concept.writable] -template -concept indirectly_writable = - requires(_Out&& __o, _Tp&& __t) { - *__o = std::forward<_Tp>(__t); // not required to be equality-preserving - *std::forward<_Out>(__o) = std::forward<_Tp>(__t); // not required to be equality-preserving - const_cast&&>(*__o) = std::forward<_Tp>(__t); // not required to be equality-preserving - const_cast&&>(*std::forward<_Out>(__o)) = std::forward<_Tp>(__t); // not required to be equality-preserving - }; +template +concept indirectly_writable = requires(_Out&& __o, _Tp&& __t) { + *__o = std::forward<_Tp>(__t); // not required to be equality-preserving + *std::forward<_Out>(__o) = std::forward<_Tp>(__t); // not required to be equality-preserving + const_cast&&>(*__o) = std::forward<_Tp>(__t); // not required to be equality-preserving + const_cast&&>(*std::forward<_Out>(__o)) = + std::forward<_Tp>(__t); // not required to be equality-preserving +}; // [iterator.concept.winc] -template +template concept __integer_like = integral<_Tp> && !same_as<_Tp, bool>; -template +template concept __signed_integer_like = signed_integral<_Tp>; -template +template concept weakly_incrementable = - // TODO: remove this once the clang bug is fixed (bugs.llvm.org/PR48173). - !same_as<_Ip, bool> && // Currently, clang does not handle bool correctly. - movable<_Ip> && - requires(_Ip __i) { - typename iter_difference_t<_Ip>; - requires __signed_integer_like>; - { ++__i } -> same_as<_Ip&>; // not required to be equality-preserving - __i++; // not required to be equality-preserving - }; + // TODO: remove this once the clang bug is fixed (bugs.llvm.org/PR48173). + !same_as<_Ip, bool> && // Currently, clang does not handle bool correctly. + movable<_Ip> && requires(_Ip __i) { + typename iter_difference_t<_Ip>; + requires __signed_integer_like>; + { ++__i } -> same_as<_Ip&>; // not required to be equality-preserving + __i++; // not required to be equality-preserving + }; // [iterator.concept.inc] -template -concept incrementable = - regular<_Ip> && - weakly_incrementable<_Ip> && - requires(_Ip __i) { - { __i++ } -> same_as<_Ip>; - }; +template +concept incrementable = regular<_Ip> && weakly_incrementable<_Ip> && requires(_Ip __i) { + { __i++ } -> same_as<_Ip>; +}; // [iterator.concept.iterator] -template -concept input_or_output_iterator = - requires(_Ip __i) { - { *__i } -> __can_reference; - } && - weakly_incrementable<_Ip>; +template +concept input_or_output_iterator = requires(_Ip __i) { + { *__i } -> __can_reference; +} && weakly_incrementable<_Ip>; // [iterator.concept.sentinel] -template -concept sentinel_for = - semiregular<_Sp> && - input_or_output_iterator<_Ip> && - __weakly_equality_comparable_with<_Sp, _Ip>; +template +concept sentinel_for = semiregular<_Sp> && input_or_output_iterator<_Ip> && __weakly_equality_comparable_with<_Sp, _Ip>; -template +template inline constexpr bool disable_sized_sentinel_for = false; -template +template concept sized_sentinel_for = - sentinel_for<_Sp, _Ip> && - !disable_sized_sentinel_for, remove_cv_t<_Ip>> && - requires(const _Ip& __i, const _Sp& __s) { - { __s - __i } -> same_as>; - { __i - __s } -> same_as>; - }; + sentinel_for<_Sp, _Ip> && !disable_sized_sentinel_for, remove_cv_t<_Ip>> && + requires(const _Ip& __i, const _Sp& __s) { + { __s - __i } -> same_as>; + { __i - __s } -> same_as>; + }; // [iterator.concept.input] -template -concept input_iterator = - input_or_output_iterator<_Ip> && - indirectly_readable<_Ip> && - requires { typename _ITER_CONCEPT<_Ip>; } && - derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>; +template +concept input_iterator = input_or_output_iterator<_Ip> && indirectly_readable<_Ip> && requires { + typename _ITER_CONCEPT<_Ip>; +} && derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>; // [iterator.concept.output] -template +template concept output_iterator = - input_or_output_iterator<_Ip> && - indirectly_writable<_Ip, _Tp> && - requires (_Ip __it, _Tp&& __t) { - *__it++ = std::forward<_Tp>(__t); // not required to be equality-preserving - }; + input_or_output_iterator<_Ip> && indirectly_writable<_Ip, _Tp> && requires(_Ip __it, _Tp&& __t) { + *__it++ = std::forward<_Tp>(__t); // not required to be equality-preserving + }; // [iterator.concept.forward] -template +template concept forward_iterator = - input_iterator<_Ip> && - derived_from<_ITER_CONCEPT<_Ip>, forward_iterator_tag> && - incrementable<_Ip> && - sentinel_for<_Ip, _Ip>; + input_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, forward_iterator_tag> && incrementable<_Ip> && + sentinel_for<_Ip, _Ip>; // [iterator.concept.bidir] -template +template concept bidirectional_iterator = - forward_iterator<_Ip> && - derived_from<_ITER_CONCEPT<_Ip>, bidirectional_iterator_tag> && - requires(_Ip __i) { - { --__i } -> same_as<_Ip&>; - { __i-- } -> same_as<_Ip>; - }; - -template + forward_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, bidirectional_iterator_tag> && requires(_Ip __i) { + { --__i } -> same_as<_Ip&>; + { __i-- } -> same_as<_Ip>; + }; + +template concept random_access_iterator = - bidirectional_iterator<_Ip> && - derived_from<_ITER_CONCEPT<_Ip>, random_access_iterator_tag> && - totally_ordered<_Ip> && - sized_sentinel_for<_Ip, _Ip> && - requires(_Ip __i, const _Ip __j, const iter_difference_t<_Ip> __n) { - { __i += __n } -> same_as<_Ip&>; - { __j + __n } -> same_as<_Ip>; - { __n + __j } -> same_as<_Ip>; - { __i -= __n } -> same_as<_Ip&>; - { __j - __n } -> same_as<_Ip>; - { __j[__n] } -> same_as>; - }; - -template + bidirectional_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, random_access_iterator_tag> && + totally_ordered<_Ip> && sized_sentinel_for<_Ip, _Ip> && + requires(_Ip __i, const _Ip __j, const iter_difference_t<_Ip> __n) { + { __i += __n } -> same_as<_Ip&>; + { __j + __n } -> same_as<_Ip>; + { __n + __j } -> same_as<_Ip>; + { __i -= __n } -> same_as<_Ip&>; + { __j - __n } -> same_as<_Ip>; + { __j[__n] } -> same_as>; + }; + +template concept contiguous_iterator = - random_access_iterator<_Ip> && - derived_from<_ITER_CONCEPT<_Ip>, contiguous_iterator_tag> && - is_lvalue_reference_v> && - same_as, remove_cvref_t>> && - requires(const _Ip& __i) { - { std::to_address(__i) } -> same_as>>; - }; - -template + random_access_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, contiguous_iterator_tag> && + is_lvalue_reference_v> && same_as, remove_cvref_t>> && + requires(const _Ip& __i) { + { std::to_address(__i) } -> same_as>>; + }; + +template concept __has_arrow = input_iterator<_Ip> && (is_pointer_v<_Ip> || requires(_Ip __i) { __i.operator->(); }); // [indirectcallable.indirectinvocable] -template +template concept indirectly_unary_invocable = - indirectly_readable<_It> && - copy_constructible<_Fp> && - invocable<_Fp&, iter_value_t<_It>&> && - invocable<_Fp&, iter_reference_t<_It>> && - invocable<_Fp&, iter_common_reference_t<_It>> && - common_reference_with< - invoke_result_t<_Fp&, iter_value_t<_It>&>, - invoke_result_t<_Fp&, iter_reference_t<_It>>>; - -template + indirectly_readable<_It> && copy_constructible<_Fp> && invocable<_Fp&, iter_value_t<_It>&> && + invocable<_Fp&, iter_reference_t<_It>> && invocable<_Fp&, iter_common_reference_t<_It>> && + common_reference_with< invoke_result_t<_Fp&, iter_value_t<_It>&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>; + +template concept indirectly_regular_unary_invocable = - indirectly_readable<_It> && - copy_constructible<_Fp> && - regular_invocable<_Fp&, iter_value_t<_It>&> && - regular_invocable<_Fp&, iter_reference_t<_It>> && - regular_invocable<_Fp&, iter_common_reference_t<_It>> && - common_reference_with< - invoke_result_t<_Fp&, iter_value_t<_It>&>, - invoke_result_t<_Fp&, iter_reference_t<_It>>>; - -template + indirectly_readable<_It> && copy_constructible<_Fp> && regular_invocable<_Fp&, iter_value_t<_It>&> && + regular_invocable<_Fp&, iter_reference_t<_It>> && regular_invocable<_Fp&, iter_common_reference_t<_It>> && + common_reference_with< invoke_result_t<_Fp&, iter_value_t<_It>&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>; + +template concept indirect_unary_predicate = - indirectly_readable<_It> && - copy_constructible<_Fp> && - predicate<_Fp&, iter_value_t<_It>&> && - predicate<_Fp&, iter_reference_t<_It>> && - predicate<_Fp&, iter_common_reference_t<_It>>; + indirectly_readable<_It> && copy_constructible<_Fp> && predicate<_Fp&, iter_value_t<_It>&> && + predicate<_Fp&, iter_reference_t<_It>> && predicate<_Fp&, iter_common_reference_t<_It>>; -template +template concept indirect_binary_predicate = - indirectly_readable<_It1> && indirectly_readable<_It2> && - copy_constructible<_Fp> && - predicate<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && - predicate<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && - predicate<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && - predicate<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && - predicate<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; - -template + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> && + predicate<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && + predicate<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && + predicate<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && + predicate<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && + predicate<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template concept indirect_equivalence_relation = - indirectly_readable<_It1> && indirectly_readable<_It2> && - copy_constructible<_Fp> && - equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && - equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && - equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && - equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && - equivalence_relation<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; - -template + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> && + equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && + equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && + equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && + equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && + equivalence_relation<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template concept indirect_strict_weak_order = - indirectly_readable<_It1> && indirectly_readable<_It2> && - copy_constructible<_Fp> && - strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && - strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && - strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && - strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && - strict_weak_order<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; - -template - requires (indirectly_readable<_Its> && ...) && invocable<_Fp, iter_reference_t<_Its>...> + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> && + strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> && + strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> && + strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> && + strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> && + strict_weak_order<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template + requires(indirectly_readable<_Its> && ...) && invocable<_Fp, iter_reference_t<_Its>...> using indirect_result_t = invoke_result_t<_Fp, iter_reference_t<_Its>...>; -template -concept indirectly_movable = - indirectly_readable<_In> && - indirectly_writable<_Out, iter_rvalue_reference_t<_In>>; +template +concept indirectly_movable = indirectly_readable<_In> && indirectly_writable<_Out, iter_rvalue_reference_t<_In>>; -template +template concept indirectly_movable_storable = - indirectly_movable<_In, _Out> && - indirectly_writable<_Out, iter_value_t<_In>> && - movable> && - constructible_from, iter_rvalue_reference_t<_In>> && - assignable_from&, iter_rvalue_reference_t<_In>>; - -template -concept indirectly_copyable = - indirectly_readable<_In> && - indirectly_writable<_Out, iter_reference_t<_In>>; - -template + indirectly_movable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>> && movable> && + constructible_from, iter_rvalue_reference_t<_In>> && + assignable_from&, iter_rvalue_reference_t<_In>>; + +template +concept indirectly_copyable = indirectly_readable<_In> && indirectly_writable<_Out, iter_reference_t<_In>>; + +template concept indirectly_copyable_storable = - indirectly_copyable<_In, _Out> && - indirectly_writable<_Out, iter_value_t<_In>&> && - indirectly_writable<_Out, const iter_value_t<_In>&> && - indirectly_writable<_Out, iter_value_t<_In>&&> && - indirectly_writable<_Out, const iter_value_t<_In>&&> && - copyable> && - constructible_from, iter_reference_t<_In>> && - assignable_from&, iter_reference_t<_In>>; + indirectly_copyable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>&> && + indirectly_writable<_Out, const iter_value_t<_In>&> && indirectly_writable<_Out, iter_value_t<_In>&&> && + indirectly_writable<_Out, const iter_value_t<_In>&&> && copyable> && + constructible_from, iter_reference_t<_In>> && + assignable_from&, iter_reference_t<_In>>; // Note: indirectly_swappable is located in iter_swap.h to prevent a dependency cycle // (both iter_swap and indirectly_swappable require indirectly_readable). @@ -298,10 +250,10 @@ concept indirectly_copyable_storable = template using __has_random_access_iterator_category_or_concept #if _LIBCPP_STD_VER >= 20 - = integral_constant>; -#else // _LIBCPP_STD_VER < 20 - = __has_random_access_iterator_category<_Tp>; -#endif // _LIBCPP_STD_VER + = integral_constant>; +#else // _LIBCPP_STD_VER < 20 + = __has_random_access_iterator_category<_Tp>; +#endif // _LIBCPP_STD_VER _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/counted_iterator.h b/libcxx/include/__iterator/counted_iterator.h index 171c0d7f0e772..c72ac677ff2f8 100644 --- a/libcxx/include/__iterator/counted_iterator.h +++ b/libcxx/include/__iterator/counted_iterator.h @@ -41,118 +41,110 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template +template struct __counted_iterator_concept {}; -template +template requires requires { typename _Iter::iterator_concept; } struct __counted_iterator_concept<_Iter> { using iterator_concept = typename _Iter::iterator_concept; }; -template +template struct __counted_iterator_category {}; -template +template requires requires { typename _Iter::iterator_category; } struct __counted_iterator_category<_Iter> { using iterator_category = typename _Iter::iterator_category; }; -template +template struct __counted_iterator_value_type {}; -template +template struct __counted_iterator_value_type<_Iter> { using value_type = iter_value_t<_Iter>; }; -template +template class counted_iterator - : public __counted_iterator_concept<_Iter> - , public __counted_iterator_category<_Iter> - , public __counted_iterator_value_type<_Iter> -{ + : public __counted_iterator_concept<_Iter>, + public __counted_iterator_category<_Iter>, + public __counted_iterator_value_type<_Iter> { public: - using iterator_type = _Iter; + using iterator_type = _Iter; using difference_type = iter_difference_t<_Iter>; - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator() requires default_initializable<_Iter> = default; + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator() + requires default_initializable<_Iter> + = default; - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator(_Iter __iter, iter_difference_t<_Iter> __n) - : __current_(std::move(__iter)), __count_(__n) { + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator(_Iter __iter, iter_difference_t<_Iter> __n) + : __current_(std::move(__iter)), __count_(__n) { _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0, "__n must not be negative."); } - template + template requires convertible_to - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator(const counted_iterator<_I2>& __other) - : __current_(__other.__current_), __count_(__other.__count_) {} + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator(const counted_iterator<_I2>& __other) + : __current_(__other.__current_), __count_(__other.__count_) {} - template + template requires assignable_from<_Iter&, const _I2&> - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator& operator=(const counted_iterator<_I2>& __other) { + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator=(const counted_iterator<_I2>& __other) { __current_ = __other.__current_; - __count_ = __other.__count_; + __count_ = __other.__count_; return *this; } - _LIBCPP_HIDE_FROM_ABI - constexpr const _Iter& base() const& noexcept { return __current_; } + _LIBCPP_HIDE_FROM_ABI constexpr const _Iter& base() const& noexcept { return __current_; } - _LIBCPP_HIDE_FROM_ABI - constexpr _Iter base() && { return std::move(__current_); } + _LIBCPP_HIDE_FROM_ABI constexpr _Iter base() && { return std::move(__current_); } - _LIBCPP_HIDE_FROM_ABI - constexpr iter_difference_t<_Iter> count() const noexcept { return __count_; } + _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter> count() const noexcept { return __count_; } - _LIBCPP_HIDE_FROM_ABI - constexpr decltype(auto) operator*() { + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() { _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator is equal to or past end."); return *__current_; } - _LIBCPP_HIDE_FROM_ABI - constexpr decltype(auto) operator*() const + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const requires __dereferenceable { _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator is equal to or past end."); return *__current_; } - _LIBCPP_HIDE_FROM_ABI - constexpr auto operator->() const noexcept + _LIBCPP_HIDE_FROM_ABI constexpr auto operator->() const noexcept requires contiguous_iterator<_Iter> { return std::to_address(__current_); } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator& operator++() { + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator++() { _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end."); ++__current_; --__count_; return *this; } - _LIBCPP_HIDE_FROM_ABI - decltype(auto) operator++(int) { + _LIBCPP_HIDE_FROM_ABI decltype(auto) operator++(int) { _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end."); --__count_; -#ifndef _LIBCPP_HAS_NO_EXCEPTIONS - try { return __current_++; } - catch(...) { ++__count_; throw; } -#else +# ifndef _LIBCPP_HAS_NO_EXCEPTIONS + try { + return __current_++; + } catch (...) { + ++__count_; + throw; + } +# else return __current_++; -#endif // _LIBCPP_HAS_NO_EXCEPTIONS +# endif // _LIBCPP_HAS_NO_EXCEPTIONS } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator operator++(int) + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator++(int) requires forward_iterator<_Iter> { _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end."); @@ -161,8 +153,7 @@ class counted_iterator return __tmp; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator& operator--() + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator--() requires bidirectional_iterator<_Iter> { --__current_; @@ -170,8 +161,7 @@ class counted_iterator return *this; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator operator--(int) + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator--(int) requires bidirectional_iterator<_Iter> { counted_iterator __tmp = *this; @@ -179,23 +169,20 @@ class counted_iterator return __tmp; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator operator+(iter_difference_t<_Iter> __n) const + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator+(iter_difference_t<_Iter> __n) const requires random_access_iterator<_Iter> { return counted_iterator(__current_ + __n, __count_ - __n); } - _LIBCPP_HIDE_FROM_ABI - friend constexpr counted_iterator operator+( - iter_difference_t<_Iter> __n, const counted_iterator& __x) + _LIBCPP_HIDE_FROM_ABI friend constexpr counted_iterator + operator+(iter_difference_t<_Iter> __n, const counted_iterator& __x) requires random_access_iterator<_Iter> { return __x + __n; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator& operator+=(iter_difference_t<_Iter> __n) + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator+=(iter_difference_t<_Iter> __n) requires random_access_iterator<_Iter> { _LIBCPP_ASSERT_UNCATEGORIZED(__n <= __count_, "Cannot advance iterator past end."); @@ -204,110 +191,92 @@ class counted_iterator return *this; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator operator-(iter_difference_t<_Iter> __n) const + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator-(iter_difference_t<_Iter> __n) const requires random_access_iterator<_Iter> { return counted_iterator(__current_ - __n, __count_ + __n); } - template _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr iter_difference_t<_I2> operator-( - const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) - { + template _I2> + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_I2> + operator-(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) { return __rhs.__count_ - __lhs.__count_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr iter_difference_t<_Iter> operator-( - const counted_iterator& __lhs, default_sentinel_t) - { + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Iter> + operator-(const counted_iterator& __lhs, default_sentinel_t) { return -__lhs.__count_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr iter_difference_t<_Iter> operator-( - default_sentinel_t, const counted_iterator& __rhs) - { + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Iter> + operator-(default_sentinel_t, const counted_iterator& __rhs) { return __rhs.__count_; } - _LIBCPP_HIDE_FROM_ABI - constexpr counted_iterator& operator-=(iter_difference_t<_Iter> __n) + _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator& operator-=(iter_difference_t<_Iter> __n) requires random_access_iterator<_Iter> { - _LIBCPP_ASSERT_UNCATEGORIZED(-__n <= __count_, - "Attempt to subtract too large of a size: " - "counted_iterator would be decremented before the " - "first element of its range."); + _LIBCPP_ASSERT_UNCATEGORIZED( + -__n <= __count_, + "Attempt to subtract too large of a size: " + "counted_iterator would be decremented before the " + "first element of its range."); __current_ -= __n; __count_ += __n; return *this; } - _LIBCPP_HIDE_FROM_ABI - constexpr decltype(auto) operator[](iter_difference_t<_Iter> __n) const + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](iter_difference_t<_Iter> __n) const requires random_access_iterator<_Iter> { _LIBCPP_ASSERT_UNCATEGORIZED(__n < __count_, "Subscript argument must be less than size."); return __current_[__n]; } - template _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==( - const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) - { + template _I2> + _LIBCPP_HIDE_FROM_ABI friend constexpr bool + operator==(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) { return __lhs.__count_ == __rhs.__count_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr bool operator==( - const counted_iterator& __lhs, default_sentinel_t) - { + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const counted_iterator& __lhs, default_sentinel_t) { return __lhs.__count_ == 0; } - template _I2> - _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering operator<=>( - const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) - { + template _I2> + _LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering + operator<=>(const counted_iterator& __lhs, const counted_iterator<_I2>& __rhs) { return __rhs.__count_ <=> __lhs.__count_; } - _LIBCPP_HIDE_FROM_ABI - friend constexpr iter_rvalue_reference_t<_Iter> iter_move(const counted_iterator& __i) - noexcept(noexcept(ranges::iter_move(__i.__current_))) - requires input_iterator<_Iter> + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> + iter_move(const counted_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__current_))) + requires input_iterator<_Iter> { _LIBCPP_ASSERT_UNCATEGORIZED(__i.__count_ > 0, "Iterator must not be past end of range."); return ranges::iter_move(__i.__current_); } - template _I2> - _LIBCPP_HIDE_FROM_ABI - friend constexpr void iter_swap(const counted_iterator& __x, const counted_iterator<_I2>& __y) - noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) - { - _LIBCPP_ASSERT_UNCATEGORIZED(__x.__count_ > 0 && __y.__count_ > 0, - "Iterators must not be past end of range."); + template _I2> + _LIBCPP_HIDE_FROM_ABI friend constexpr void + iter_swap(const counted_iterator& __x, + const counted_iterator<_I2>& __y) noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) { + _LIBCPP_ASSERT_UNCATEGORIZED(__x.__count_ > 0 && __y.__count_ > 0, "Iterators must not be past end of range."); return ranges::iter_swap(__x.__current_, __y.__current_); } private: _LIBCPP_NO_UNIQUE_ADDRESS _Iter __current_ = _Iter(); - iter_difference_t<_Iter> __count_ = 0; - template + iter_difference_t<_Iter> __count_ = 0; + template friend class counted_iterator; }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(counted_iterator); -template +template requires same_as<_ITER_TRAITS<_Iter>, iterator_traits<_Iter>> struct iterator_traits> : iterator_traits<_Iter> { - using pointer = conditional_t, - add_pointer_t>, void>; + using pointer = conditional_t, add_pointer_t>, void>; }; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/data.h b/libcxx/include/__iterator/data.h index 894de49991d6e..3986739061017 100644 --- a/libcxx/include/__iterator/data.h +++ b/libcxx/include/__iterator/data.h @@ -22,27 +22,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 -template constexpr -_LIBCPP_HIDE_FROM_ABI -auto data(_Cont& __c) -_NOEXCEPT_(noexcept(__c.data())) --> decltype (__c.data()) -{ return __c.data(); } - -template constexpr -_LIBCPP_HIDE_FROM_ABI -auto data(const _Cont& __c) -_NOEXCEPT_(noexcept(__c.data())) --> decltype (__c.data()) -{ return __c.data(); } +template +constexpr _LIBCPP_HIDE_FROM_ABI auto data(_Cont& __c) _NOEXCEPT_(noexcept(__c.data())) -> decltype(__c.data()) { + return __c.data(); +} + +template +constexpr _LIBCPP_HIDE_FROM_ABI auto data(const _Cont& __c) _NOEXCEPT_(noexcept(__c.data())) -> decltype(__c.data()) { + return __c.data(); +} template -_LIBCPP_HIDE_FROM_ABI -constexpr _Tp* data(_Tp (&__array)[_Sz]) noexcept { return __array; } +_LIBCPP_HIDE_FROM_ABI constexpr _Tp* data(_Tp (&__array)[_Sz]) noexcept { + return __array; +} template -_LIBCPP_HIDE_FROM_ABI -constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept { return __il.begin(); } +_LIBCPP_HIDE_FROM_ABI constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept { + return __il.begin(); +} #endif diff --git a/libcxx/include/__iterator/default_sentinel.h b/libcxx/include/__iterator/default_sentinel.h index d5fb2b699ad44..3b65f442f1a85 100644 --- a/libcxx/include/__iterator/default_sentinel.h +++ b/libcxx/include/__iterator/default_sentinel.h @@ -20,7 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -struct default_sentinel_t { }; +struct default_sentinel_t {}; inline constexpr default_sentinel_t default_sentinel{}; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/distance.h b/libcxx/include/__iterator/distance.h index 83a9265390956..75bd49c9ae732 100644 --- a/libcxx/include/__iterator/distance.h +++ b/libcxx/include/__iterator/distance.h @@ -27,30 +27,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -typename iterator_traits<_InputIter>::difference_type -__distance(_InputIter __first, _InputIter __last, input_iterator_tag) -{ - typename iterator_traits<_InputIter>::difference_type __r(0); - for (; __first != __last; ++__first) - ++__r; - return __r; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type +__distance(_InputIter __first, _InputIter __last, input_iterator_tag) { + typename iterator_traits<_InputIter>::difference_type __r(0); + for (; __first != __last; ++__first) + ++__r; + return __r; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -typename iterator_traits<_RandIter>::difference_type -__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) -{ - return __last - __first; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_RandIter>::difference_type +__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) { + return __last - __first; } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -typename iterator_traits<_InputIter>::difference_type -distance(_InputIter __first, _InputIter __last) -{ - return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category()); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type +distance(_InputIter __first, _InputIter __last) { + return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category()); } #if _LIBCPP_STD_VER >= 20 @@ -61,10 +55,9 @@ namespace ranges { namespace __distance { struct __fn { - template _Sp> - requires (!sized_sentinel_for<_Sp, _Ip>) - _LIBCPP_HIDE_FROM_ABI - constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const { + template _Sp> + requires(!sized_sentinel_for<_Sp, _Ip>) + _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const { iter_difference_t<_Ip> __n = 0; while (__first != __last) { ++__first; @@ -73,9 +66,8 @@ struct __fn { return __n; } - template> _Sp> - _LIBCPP_HIDE_FROM_ABI - constexpr iter_difference_t<_Ip> operator()(_Ip&& __first, _Sp __last) const { + template > _Sp> + _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip&& __first, _Sp __last) const { if constexpr (sized_sentinel_for<_Sp, __remove_cvref_t<_Ip>>) { return __last - __first; } else { @@ -83,9 +75,8 @@ struct __fn { } } - template - _LIBCPP_HIDE_FROM_ABI - constexpr range_difference_t<_Rp> operator()(_Rp&& __r) const { + template + _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Rp> operator()(_Rp&& __r) const { if constexpr (sized_range<_Rp>) { return static_cast>(ranges::size(__r)); } else { @@ -97,7 +88,7 @@ struct __fn { } // namespace __distance inline namespace __cpo { - inline constexpr auto distance = __distance::__fn{}; +inline constexpr auto distance = __distance::__fn{}; } // namespace __cpo } // namespace ranges diff --git a/libcxx/include/__iterator/empty.h b/libcxx/include/__iterator/empty.h index 6c7b0dcb1ebca..3ca0aff6be46e 100644 --- a/libcxx/include/__iterator/empty.h +++ b/libcxx/include/__iterator/empty.h @@ -23,19 +23,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 17 template -_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI -constexpr auto empty(const _Cont& __c) -_NOEXCEPT_(noexcept(__c.empty())) --> decltype (__c.empty()) -{ return __c.empty(); } +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI constexpr auto empty(const _Cont& __c) + _NOEXCEPT_(noexcept(__c.empty())) -> decltype(__c.empty()) { + return __c.empty(); +} template -_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI -constexpr bool empty(const _Tp (&)[_Sz]) noexcept { return false; } +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI constexpr bool empty(const _Tp (&)[_Sz]) noexcept { + return false; +} template -_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI -constexpr bool empty(initializer_list<_Ep> __il) noexcept { return __il.size() == 0; } +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_HIDE_FROM_ABI constexpr bool empty(initializer_list<_Ep> __il) noexcept { + return __il.size() == 0; +} #endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/include/__iterator/erase_if_container.h b/libcxx/include/__iterator/erase_if_container.h index 532ec9c1de0be..0f87f50cd1c16 100644 --- a/libcxx/include/__iterator/erase_if_container.h +++ b/libcxx/include/__iterator/erase_if_container.h @@ -22,9 +22,7 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template -_LIBCPP_HIDE_FROM_ABI -typename _Container::size_type -__libcpp_erase_if_container(_Container& __c, _Predicate& __pred) { +_LIBCPP_HIDE_FROM_ABI typename _Container::size_type __libcpp_erase_if_container(_Container& __c, _Predicate& __pred) { typename _Container::size_type __old_size = __c.size(); const typename _Container::iterator __last = __c.end(); diff --git a/libcxx/include/__iterator/front_insert_iterator.h b/libcxx/include/__iterator/front_insert_iterator.h index f9aa442177b2c..7f2c54ec87442 100644 --- a/libcxx/include/__iterator/front_insert_iterator.h +++ b/libcxx/include/__iterator/front_insert_iterator.h @@ -33,40 +33,47 @@ class _LIBCPP_TEMPLATE_VIS front_insert_iterator : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + protected: - _Container* container; + _Container* container; + public: - typedef output_iterator_tag iterator_category; - typedef void value_type; + typedef output_iterator_tag iterator_category; + typedef void value_type; #if _LIBCPP_STD_VER >= 20 - typedef ptrdiff_t difference_type; + typedef ptrdiff_t difference_type; #else - typedef void difference_type; + typedef void difference_type; #endif - typedef void pointer; - typedef void reference; - typedef _Container container_type; + typedef void pointer; + typedef void reference; + typedef _Container container_type; - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit front_insert_iterator(_Container& __x) : container(std::addressof(__x)) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator=(const typename _Container::value_type& __value) - {container->push_front(__value); return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit front_insert_iterator(_Container& __x) + : container(std::addressof(__x)) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& + operator=(const typename _Container::value_type& __value) { + container->push_front(__value); + return *this; + } #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator=(typename _Container::value_type&& __value) - {container->push_front(std::move(__value)); return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& + operator=(typename _Container::value_type&& __value) { + container->push_front(std::move(__value)); + return *this; + } #endif // _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator*() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator++() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator operator++(int) {return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator*() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator& operator++() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator operator++(int) { return *this; } }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(front_insert_iterator); template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -front_insert_iterator<_Container> -front_inserter(_Container& __x) -{ - return front_insert_iterator<_Container>(__x); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 front_insert_iterator<_Container> +front_inserter(_Container& __x) { + return front_insert_iterator<_Container>(__x); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/incrementable_traits.h b/libcxx/include/__iterator/incrementable_traits.h index 604e9580ef7ca..a228b228f6e55 100644 --- a/libcxx/include/__iterator/incrementable_traits.h +++ b/libcxx/include/__iterator/incrementable_traits.h @@ -29,33 +29,33 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 // [incrementable.traits] -template struct incrementable_traits {}; +template +struct incrementable_traits {}; -template -requires is_object_v<_Tp> +template + requires is_object_v<_Tp> struct incrementable_traits<_Tp*> { using difference_type = ptrdiff_t; }; -template +template struct incrementable_traits : incrementable_traits<_Ip> {}; -template +template concept __has_member_difference_type = requires { typename _Tp::difference_type; }; -template<__has_member_difference_type _Tp> +template <__has_member_difference_type _Tp> struct incrementable_traits<_Tp> { using difference_type = typename _Tp::difference_type; }; -template -concept __has_integral_minus = - requires(const _Tp& __x, const _Tp& __y) { - { __x - __y } -> integral; - }; +template +concept __has_integral_minus = requires(const _Tp& __x, const _Tp& __y) { + { __x - __y } -> integral; +}; -template<__has_integral_minus _Tp> -requires (!__has_member_difference_type<_Tp>) +template <__has_integral_minus _Tp> + requires(!__has_member_difference_type<_Tp>) struct incrementable_traits<_Tp> { using difference_type = make_signed_t() - std::declval<_Tp>())>; }; @@ -67,9 +67,10 @@ struct iterator_traits; // `incrementable_traits::difference_type` if `iterator_traits` names a specialization // generated from the primary template, and `iterator_traits::difference_type` otherwise. template -using iter_difference_t = typename conditional_t<__is_primary_template > >::value, - incrementable_traits >, - iterator_traits > >::difference_type; +using iter_difference_t = + typename conditional_t<__is_primary_template > >::value, + incrementable_traits >, + iterator_traits > >::difference_type; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/indirectly_comparable.h b/libcxx/include/__iterator/indirectly_comparable.h index e60ba25ca7687..e8a7398bacd2b 100644 --- a/libcxx/include/__iterator/indirectly_comparable.h +++ b/libcxx/include/__iterator/indirectly_comparable.h @@ -24,8 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -concept indirectly_comparable = - indirect_binary_predicate<_Rp, projected<_I1, _P1>, projected<_I2, _P2>>; +concept indirectly_comparable = indirect_binary_predicate<_Rp, projected<_I1, _P1>, projected<_I2, _P2>>; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/insert_iterator.h b/libcxx/include/__iterator/insert_iterator.h index 1a9c74a16f4fd..8b7574dc9ec0a 100644 --- a/libcxx/include/__iterator/insert_iterator.h +++ b/libcxx/include/__iterator/insert_iterator.h @@ -42,41 +42,50 @@ class _LIBCPP_TEMPLATE_VIS insert_iterator : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + protected: - _Container* container; - __insert_iterator_iter_t<_Container> iter; + _Container* container; + __insert_iterator_iter_t<_Container> iter; + public: - typedef output_iterator_tag iterator_category; - typedef void value_type; + typedef output_iterator_tag iterator_category; + typedef void value_type; #if _LIBCPP_STD_VER >= 20 - typedef ptrdiff_t difference_type; + typedef ptrdiff_t difference_type; #else - typedef void difference_type; + typedef void difference_type; #endif - typedef void pointer; - typedef void reference; - typedef _Container container_type; + typedef void pointer; + typedef void reference; + typedef _Container container_type; - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator(_Container& __x, __insert_iterator_iter_t<_Container> __i) - : container(std::addressof(__x)), iter(__i) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator=(const typename _Container::value_type& __value) - {iter = container->insert(iter, __value); ++iter; return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 + insert_iterator(_Container& __x, __insert_iterator_iter_t<_Container> __i) + : container(std::addressof(__x)), iter(__i) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& + operator=(const typename _Container::value_type& __value) { + iter = container->insert(iter, __value); + ++iter; + return *this; + } #ifndef _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator=(typename _Container::value_type&& __value) - {iter = container->insert(iter, std::move(__value)); ++iter; return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& + operator=(typename _Container::value_type&& __value) { + iter = container->insert(iter, std::move(__value)); + ++iter; + return *this; + } #endif // _LIBCPP_CXX03_LANG - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator*() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++() {return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++(int) {return *this;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator*() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++() { return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator& operator++(int) { return *this; } }; template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 -insert_iterator<_Container> -inserter(_Container& __x, __insert_iterator_iter_t<_Container> __i) -{ - return insert_iterator<_Container>(__x, __i); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 insert_iterator<_Container> +inserter(_Container& __x, __insert_iterator_iter_t<_Container> __i) { + return insert_iterator<_Container>(__x, __i); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/istream_iterator.h b/libcxx/include/__iterator/istream_iterator.h index 2efe5772ba614..58c9ac6d4ccce 100644 --- a/libcxx/include/__iterator/istream_iterator.h +++ b/libcxx/include/__iterator/istream_iterator.h @@ -26,78 +26,73 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_SUPPRESS_DEPRECATED_PUSH -template , class _Distance = ptrdiff_t> +template , class _Distance = ptrdiff_t> class _LIBCPP_TEMPLATE_VIS istream_iterator #if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES) : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + public: - typedef input_iterator_tag iterator_category; - typedef _Tp value_type; - typedef _Distance difference_type; - typedef const _Tp* pointer; - typedef const _Tp& reference; - typedef _CharT char_type; - typedef _Traits traits_type; - typedef basic_istream<_CharT,_Traits> istream_type; + typedef input_iterator_tag iterator_category; + typedef _Tp value_type; + typedef _Distance difference_type; + typedef const _Tp* pointer; + typedef const _Tp& reference; + typedef _CharT char_type; + typedef _Traits traits_type; + typedef basic_istream<_CharT, _Traits> istream_type; + private: - istream_type* __in_stream_; - _Tp __value_; + istream_type* __in_stream_; + _Tp __value_; + public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istream_iterator() : __in_stream_(nullptr), __value_() {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istream_iterator() : __in_stream_(nullptr), __value_() {} #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI constexpr istream_iterator(default_sentinel_t) : istream_iterator() {} + _LIBCPP_HIDE_FROM_ABI constexpr istream_iterator(default_sentinel_t) : istream_iterator() {} #endif // _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI istream_iterator(istream_type& __s) : __in_stream_(std::addressof(__s)) - { - if (!(*__in_stream_ >> __value_)) - __in_stream_ = nullptr; - } + _LIBCPP_HIDE_FROM_ABI istream_iterator(istream_type& __s) : __in_stream_(std::addressof(__s)) { + if (!(*__in_stream_ >> __value_)) + __in_stream_ = nullptr; + } - _LIBCPP_HIDE_FROM_ABI const _Tp& operator*() const {return __value_;} - _LIBCPP_HIDE_FROM_ABI const _Tp* operator->() const {return std::addressof((operator*()));} - _LIBCPP_HIDE_FROM_ABI istream_iterator& operator++() - { - if (!(*__in_stream_ >> __value_)) - __in_stream_ = nullptr; - return *this; - } - _LIBCPP_HIDE_FROM_ABI istream_iterator operator++(int) - {istream_iterator __t(*this); ++(*this); return __t;} + _LIBCPP_HIDE_FROM_ABI const _Tp& operator*() const { return __value_; } + _LIBCPP_HIDE_FROM_ABI const _Tp* operator->() const { return std::addressof((operator*())); } + _LIBCPP_HIDE_FROM_ABI istream_iterator& operator++() { + if (!(*__in_stream_ >> __value_)) + __in_stream_ = nullptr; + return *this; + } + _LIBCPP_HIDE_FROM_ABI istream_iterator operator++(int) { + istream_iterator __t(*this); + ++(*this); + return __t; + } - template - friend _LIBCPP_HIDE_FROM_ABI - bool - operator==(const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __x, - const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __y); + template + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __x, + const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __y); #if _LIBCPP_STD_VER >= 20 - friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator& __i, default_sentinel_t) { - return __i.__in_stream_ == nullptr; - } + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator& __i, default_sentinel_t) { + return __i.__in_stream_ == nullptr; + } #endif // _LIBCPP_STD_VER >= 20 }; template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator==(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, - const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) -{ - return __x.__in_stream_ == __y.__in_stream_; +inline _LIBCPP_HIDE_FROM_ABI bool operator==(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, + const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) { + return __x.__in_stream_ == __y.__in_stream_; } #if _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI -bool -operator!=(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, - const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) -{ - return !(__x == __y); +inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, + const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) { + return !(__x == __y); } #endif // _LIBCPP_STD_VER <= 17 diff --git a/libcxx/include/__iterator/istreambuf_iterator.h b/libcxx/include/__iterator/istreambuf_iterator.h index ed5012a68d82d..51c4ecff351f5 100644 --- a/libcxx/include/__iterator/istreambuf_iterator.h +++ b/libcxx/include/__iterator/istreambuf_iterator.h @@ -24,95 +24,84 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_SUPPRESS_DEPRECATED_PUSH -template +template class _LIBCPP_TEMPLATE_VIS istreambuf_iterator #if _LIBCPP_STD_VER <= 14 || !defined(_LIBCPP_ABI_NO_ITERATOR_BASES) - : public iterator + : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + public: - typedef input_iterator_tag iterator_category; - typedef _CharT value_type; - typedef typename _Traits::off_type difference_type; - typedef _CharT* pointer; - typedef _CharT reference; - typedef _CharT char_type; - typedef _Traits traits_type; - typedef typename _Traits::int_type int_type; - typedef basic_streambuf<_CharT,_Traits> streambuf_type; - typedef basic_istream<_CharT,_Traits> istream_type; + typedef input_iterator_tag iterator_category; + typedef _CharT value_type; + typedef typename _Traits::off_type difference_type; + typedef _CharT* pointer; + typedef _CharT reference; + typedef _CharT char_type; + typedef _Traits traits_type; + typedef typename _Traits::int_type int_type; + typedef basic_streambuf<_CharT, _Traits> streambuf_type; + typedef basic_istream<_CharT, _Traits> istream_type; + private: - mutable streambuf_type* __sbuf_; - - class __proxy - { - char_type __keep_; - streambuf_type* __sbuf_; - _LIBCPP_HIDE_FROM_ABI - explicit __proxy(char_type __c, streambuf_type* __s) - : __keep_(__c), __sbuf_(__s) {} - friend class istreambuf_iterator; - public: - _LIBCPP_HIDE_FROM_ABI char_type operator*() const {return __keep_;} - }; - - _LIBCPP_HIDE_FROM_ABI - bool __test_for_eof() const - { - if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sgetc(), traits_type::eof())) - __sbuf_ = nullptr; - return __sbuf_ == nullptr; - } + mutable streambuf_type* __sbuf_; + + class __proxy { + char_type __keep_; + streambuf_type* __sbuf_; + _LIBCPP_HIDE_FROM_ABI explicit __proxy(char_type __c, streambuf_type* __s) : __keep_(__c), __sbuf_(__s) {} + friend class istreambuf_iterator; + + public: + _LIBCPP_HIDE_FROM_ABI char_type operator*() const { return __keep_; } + }; + + _LIBCPP_HIDE_FROM_ABI bool __test_for_eof() const { + if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sgetc(), traits_type::eof())) + __sbuf_ = nullptr; + return __sbuf_ == nullptr; + } + public: - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istreambuf_iterator() _NOEXCEPT : __sbuf_(nullptr) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR istreambuf_iterator() _NOEXCEPT : __sbuf_(nullptr) {} #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI constexpr istreambuf_iterator(default_sentinel_t) noexcept - : istreambuf_iterator() {} + _LIBCPP_HIDE_FROM_ABI constexpr istreambuf_iterator(default_sentinel_t) noexcept : istreambuf_iterator() {} #endif // _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(istream_type& __s) _NOEXCEPT - : __sbuf_(__s.rdbuf()) {} - _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(streambuf_type* __s) _NOEXCEPT - : __sbuf_(__s) {} - _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(const __proxy& __p) _NOEXCEPT - : __sbuf_(__p.__sbuf_) {} - - _LIBCPP_HIDE_FROM_ABI char_type operator*() const - {return static_cast(__sbuf_->sgetc());} - _LIBCPP_HIDE_FROM_ABI istreambuf_iterator& operator++() - { - __sbuf_->sbumpc(); - return *this; - } - _LIBCPP_HIDE_FROM_ABI __proxy operator++(int) - { - return __proxy(__sbuf_->sbumpc(), __sbuf_); - } - - _LIBCPP_HIDE_FROM_ABI bool equal(const istreambuf_iterator& __b) const - {return __test_for_eof() == __b.__test_for_eof();} + _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(istream_type& __s) _NOEXCEPT : __sbuf_(__s.rdbuf()) {} + _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(streambuf_type* __s) _NOEXCEPT : __sbuf_(__s) {} + _LIBCPP_HIDE_FROM_ABI istreambuf_iterator(const __proxy& __p) _NOEXCEPT : __sbuf_(__p.__sbuf_) {} + + _LIBCPP_HIDE_FROM_ABI char_type operator*() const { return static_cast(__sbuf_->sgetc()); } + _LIBCPP_HIDE_FROM_ABI istreambuf_iterator& operator++() { + __sbuf_->sbumpc(); + return *this; + } + _LIBCPP_HIDE_FROM_ABI __proxy operator++(int) { return __proxy(__sbuf_->sbumpc(), __sbuf_); } + + _LIBCPP_HIDE_FROM_ABI bool equal(const istreambuf_iterator& __b) const { + return __test_for_eof() == __b.__test_for_eof(); + } #if _LIBCPP_STD_VER >= 20 - friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istreambuf_iterator& __i, default_sentinel_t) { - return __i.__test_for_eof(); - } + friend _LIBCPP_HIDE_FROM_ABI bool operator==(const istreambuf_iterator& __i, default_sentinel_t) { + return __i.__test_for_eof(); + } #endif // _LIBCPP_STD_VER >= 20 }; template -inline _LIBCPP_HIDE_FROM_ABI -bool operator==(const istreambuf_iterator<_CharT,_Traits>& __a, - const istreambuf_iterator<_CharT,_Traits>& __b) - {return __a.equal(__b);} +inline _LIBCPP_HIDE_FROM_ABI bool +operator==(const istreambuf_iterator<_CharT, _Traits>& __a, const istreambuf_iterator<_CharT, _Traits>& __b) { + return __a.equal(__b); +} #if _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI -bool operator!=(const istreambuf_iterator<_CharT,_Traits>& __a, - const istreambuf_iterator<_CharT,_Traits>& __b) - {return !__a.equal(__b);} +inline _LIBCPP_HIDE_FROM_ABI bool +operator!=(const istreambuf_iterator<_CharT, _Traits>& __a, const istreambuf_iterator<_CharT, _Traits>& __b) { + return !__a.equal(__b); +} #endif // _LIBCPP_STD_VER <= 17 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/iter_move.h b/libcxx/include/__iterator/iter_move.h index d9ff90bf65456..202b94cccc5ac 100644 --- a/libcxx/include/__iterator/iter_move.h +++ b/libcxx/include/__iterator/iter_move.h @@ -38,66 +38,60 @@ namespace __iter_move { void iter_move(); template -concept __unqualified_iter_move = - __class_or_enum> && - requires (_Tp&& __t) { - // NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap - iter_move(std::forward<_Tp>(__t)); - }; - -template -concept __move_deref = - !__unqualified_iter_move<_Tp> && - requires (_Tp&& __t) { - *__t; - requires is_lvalue_reference_v; - }; - -template -concept __just_deref = - !__unqualified_iter_move<_Tp> && - !__move_deref<_Tp> && - requires (_Tp&& __t) { - *__t; - requires (!is_lvalue_reference_v); - }; +concept __unqualified_iter_move = __class_or_enum> && requires(_Tp&& __t) { + // NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap + iter_move(std::forward<_Tp>(__t)); +}; + +template +concept __move_deref = !__unqualified_iter_move<_Tp> && requires(_Tp&& __t) { + *__t; + requires is_lvalue_reference_v; +}; + +template +concept __just_deref = !__unqualified_iter_move<_Tp> && !__move_deref<_Tp> && requires(_Tp&& __t) { + *__t; + requires(!is_lvalue_reference_v); +}; // [iterator.cust.move] struct __fn { // NOLINTBEGIN(libcpp-robust-against-adl) iter_move ADL calls should only be made through ranges::iter_move - template + template requires __unqualified_iter_move<_Ip> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator()(_Ip&& __i) const - noexcept(noexcept(iter_move(std::forward<_Ip>(__i)))) - { + noexcept(noexcept(iter_move(std::forward<_Ip>(__i)))) { return iter_move(std::forward<_Ip>(__i)); } // NOLINTEND(libcpp-robust-against-adl) - template + template requires __move_deref<_Ip> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ip&& __i) const - noexcept(noexcept(std::move(*std::forward<_Ip>(__i)))) - -> decltype( std::move(*std::forward<_Ip>(__i))) - { return std::move(*std::forward<_Ip>(__i)); } + noexcept(noexcept(std::move(*std::forward<_Ip>(__i)))) -> decltype(std::move(*std::forward<_Ip>(__i))) { + return std::move(*std::forward<_Ip>(__i)); + } - template + template requires __just_deref<_Ip> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ip&& __i) const - noexcept(noexcept(*std::forward<_Ip>(__i))) - -> decltype( *std::forward<_Ip>(__i)) - { return *std::forward<_Ip>(__i); } + noexcept(noexcept(*std::forward<_Ip>(__i))) -> decltype(*std::forward<_Ip>(__i)) { + return *std::forward<_Ip>(__i); + } }; } // namespace __iter_move inline namespace __cpo { - inline constexpr auto iter_move = __iter_move::__fn{}; +inline constexpr auto iter_move = __iter_move::__fn{}; } // namespace __cpo } // namespace ranges -template<__dereferenceable _Tp> - requires requires(_Tp& __t) { { ranges::iter_move(__t) } -> __can_reference; } +template <__dereferenceable _Tp> + requires requires(_Tp& __t) { + { ranges::iter_move(__t) } -> __can_reference; + } using iter_rvalue_reference_t = decltype(ranges::iter_move(std::declval<_Tp&>())); #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/iter_swap.h b/libcxx/include/__iterator/iter_swap.h index 23d3e4009f536..52c3f095e7fb1 100644 --- a/libcxx/include/__iterator/iter_swap.h +++ b/libcxx/include/__iterator/iter_swap.h @@ -37,77 +37,67 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace ranges { namespace __iter_swap { - template - void iter_swap(_I1, _I2) = delete; +template +void iter_swap(_I1, _I2) = delete; - template - concept __unqualified_iter_swap = - (__class_or_enum> || __class_or_enum>) && - requires (_T1&& __x, _T2&& __y) { +template +concept __unqualified_iter_swap = + (__class_or_enum> || __class_or_enum>)&&requires(_T1&& __x, _T2&& __y) { // NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)); }; - template - concept __readable_swappable = +template +concept __readable_swappable = indirectly_readable<_T1> && indirectly_readable<_T2> && swappable_with, iter_reference_t<_T2>>; - - struct __fn { - // NOLINTBEGIN(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap - template - requires __unqualified_iter_swap<_T1, _T2> - _LIBCPP_HIDE_FROM_ABI - constexpr void operator()(_T1&& __x, _T2&& __y) const - noexcept(noexcept(iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)))) - { - (void)iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)); - } - // NOLINTEND(libcpp-robust-against-adl) - - template - requires (!__unqualified_iter_swap<_T1, _T2>) && - __readable_swappable<_T1, _T2> - _LIBCPP_HIDE_FROM_ABI - constexpr void operator()(_T1&& __x, _T2&& __y) const - noexcept(noexcept(ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y)))) - { - ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y)); - } - - template - requires (!__unqualified_iter_swap<_T1, _T2> && // - !__readable_swappable<_T1, _T2>) && // - indirectly_movable_storable<_T1, _T2> && // - indirectly_movable_storable<_T2, _T1> - _LIBCPP_HIDE_FROM_ABI - constexpr void operator()(_T1&& __x, _T2&& __y) const +struct __fn { + // NOLINTBEGIN(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap + template + requires __unqualified_iter_swap<_T1, _T2> + _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const + noexcept(noexcept(iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)))) { + (void)iter_swap(std::forward<_T1>(__x), std::forward<_T2>(__y)); + } + // NOLINTEND(libcpp-robust-against-adl) + + template + requires(!__unqualified_iter_swap<_T1, _T2>) && __readable_swappable<_T1, _T2> + _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const + noexcept(noexcept(ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y)))) { + ranges::swap(*std::forward<_T1>(__x), *std::forward<_T2>(__y)); + } + + template + requires(!__unqualified_iter_swap<_T1, _T2> && // + !__readable_swappable<_T1, _T2>) && // + indirectly_movable_storable<_T1, _T2> && // + indirectly_movable_storable<_T2, _T1> + _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_T1&& __x, _T2&& __y) const noexcept(noexcept(iter_value_t<_T2>(ranges::iter_move(__y))) && // - noexcept(*__y = ranges::iter_move(__x)) && // - noexcept(*std::forward<_T1>(__x) = std::declval>())) - { - iter_value_t<_T2> __old(ranges::iter_move(__y)); - *__y = ranges::iter_move(__x); - *std::forward<_T1>(__x) = std::move(__old); - } - }; + noexcept(*__y = ranges::iter_move(__x)) && // + noexcept(*std::forward<_T1>(__x) = std::declval>())) { + iter_value_t<_T2> __old(ranges::iter_move(__y)); + *__y = ranges::iter_move(__x); + *std::forward<_T1>(__x) = std::move(__old); + } +}; } // namespace __iter_swap inline namespace __cpo { - inline constexpr auto iter_swap = __iter_swap::__fn{}; +inline constexpr auto iter_swap = __iter_swap::__fn{}; } // namespace __cpo } // namespace ranges -template +template concept indirectly_swappable = - indirectly_readable<_I1> && indirectly_readable<_I2> && - requires(const _I1 __i1, const _I2 __i2) { - ranges::iter_swap(__i1, __i1); - ranges::iter_swap(__i2, __i2); - ranges::iter_swap(__i1, __i2); - ranges::iter_swap(__i2, __i1); - }; + indirectly_readable<_I1> && indirectly_readable<_I2> && requires(const _I1 __i1, const _I2 __i2) { + ranges::iter_swap(__i1, __i1); + ranges::iter_swap(__i2, __i2); + ranges::iter_swap(__i1, __i2); + ranges::iter_swap(__i2, __i1); + }; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/iterator.h b/libcxx/include/__iterator/iterator.h index b417eeab79bfe..ba9308f3c2243 100644 --- a/libcxx/include/__iterator/iterator.h +++ b/libcxx/include/__iterator/iterator.h @@ -19,15 +19,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 iterator -{ - typedef _Tp value_type; - typedef _Distance difference_type; - typedef _Pointer pointer; - typedef _Reference reference; - typedef _Category iterator_category; +template +struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 iterator { + typedef _Tp value_type; + typedef _Distance difference_type; + typedef _Pointer pointer; + typedef _Reference reference; + typedef _Category iterator_category; }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/iterator_traits.h b/libcxx/include/__iterator/iterator_traits.h index 248987e9b5605..dae8cc75ae349 100644 --- a/libcxx/include/__iterator/iterator_traits.h +++ b/libcxx/include/__iterator/iterator_traits.h @@ -49,9 +49,7 @@ template using __with_reference = _Tp&; template -concept __can_reference = requires { - typename __with_reference<_Tp>; -}; +concept __can_reference = requires { typename __with_reference<_Tp>; }; template concept __dereferenceable = requires(_Tp& __t) { @@ -59,7 +57,7 @@ concept __dereferenceable = requires(_Tp& __t) { }; // [iterator.traits] -template<__dereferenceable _Tp> +template <__dereferenceable _Tp> using iter_reference_t = decltype(*std::declval<_Tp&>()); #endif // _LIBCPP_STD_VER >= 20 @@ -69,20 +67,16 @@ struct _LIBCPP_TEMPLATE_VIS iterator_traits; struct _LIBCPP_TEMPLATE_VIS input_iterator_tag {}; struct _LIBCPP_TEMPLATE_VIS output_iterator_tag {}; -struct _LIBCPP_TEMPLATE_VIS forward_iterator_tag : public input_iterator_tag {}; +struct _LIBCPP_TEMPLATE_VIS forward_iterator_tag : public input_iterator_tag {}; struct _LIBCPP_TEMPLATE_VIS bidirectional_iterator_tag : public forward_iterator_tag {}; struct _LIBCPP_TEMPLATE_VIS random_access_iterator_tag : public bidirectional_iterator_tag {}; #if _LIBCPP_STD_VER >= 20 -struct _LIBCPP_TEMPLATE_VIS contiguous_iterator_tag : public random_access_iterator_tag {}; +struct _LIBCPP_TEMPLATE_VIS contiguous_iterator_tag : public random_access_iterator_tag {}; #endif template struct __iter_traits_cache { - using type = _If< - __is_primary_template >::value, - _Iter, - iterator_traits<_Iter> - >; + using type = _If< __is_primary_template >::value, _Iter, iterator_traits<_Iter> >; }; template using _ITER_TRAITS = typename __iter_traits_cache<_Iter>::type; @@ -97,64 +91,61 @@ struct __iter_concept_category_test { }; struct __iter_concept_random_fallback { template - using _Apply = __enable_if_t< - __is_primary_template >::value, - random_access_iterator_tag - >; + using _Apply = __enable_if_t< __is_primary_template >::value, random_access_iterator_tag >; }; -template struct __test_iter_concept - : _IsValidExpansion<_Tester::template _Apply, _Iter>, - _Tester -{ -}; +template +struct __test_iter_concept : _IsValidExpansion<_Tester::template _Apply, _Iter>, _Tester {}; template struct __iter_concept_cache { - using type = _Or< - __test_iter_concept<_Iter, __iter_concept_concept_test>, - __test_iter_concept<_Iter, __iter_concept_category_test>, - __test_iter_concept<_Iter, __iter_concept_random_fallback> - >; + using type = _Or< __test_iter_concept<_Iter, __iter_concept_concept_test>, + __test_iter_concept<_Iter, __iter_concept_category_test>, + __test_iter_concept<_Iter, __iter_concept_random_fallback> >; }; template using _ITER_CONCEPT = typename __iter_concept_cache<_Iter>::type::template _Apply<_Iter>; - template -struct __has_iterator_typedefs -{ +struct __has_iterator_typedefs { private: - template static false_type __test(...); - template static true_type __test(__void_t* = nullptr, - __void_t* = nullptr, - __void_t* = nullptr, - __void_t* = nullptr, - __void_t* = nullptr); + template + static false_type __test(...); + template + static true_type + __test(__void_t* = nullptr, + __void_t* = nullptr, + __void_t* = nullptr, + __void_t* = nullptr, + __void_t* = nullptr); + public: - static const bool value = decltype(__test<_Tp>(0,0,0,0,0))::value; + static const bool value = decltype(__test<_Tp>(0, 0, 0, 0, 0))::value; }; - template -struct __has_iterator_category -{ +struct __has_iterator_category { private: - template static false_type __test(...); - template static true_type __test(typename _Up::iterator_category* = nullptr); + template + static false_type __test(...); + template + static true_type __test(typename _Up::iterator_category* = nullptr); + public: - static const bool value = decltype(__test<_Tp>(nullptr))::value; + static const bool value = decltype(__test<_Tp>(nullptr))::value; }; template -struct __has_iterator_concept -{ +struct __has_iterator_concept { private: - template static false_type __test(...); - template static true_type __test(typename _Up::iterator_concept* = nullptr); + template + static false_type __test(...); + template + static true_type __test(typename _Up::iterator_concept* = nullptr); + public: - static const bool value = decltype(__test<_Tp>(nullptr))::value; + static const bool value = decltype(__test<_Tp>(nullptr))::value; }; #if _LIBCPP_STD_VER >= 20 @@ -163,200 +154,194 @@ struct __has_iterator_concept // from `[iterator.cpp17]`. To avoid confusion between the two, the exposition-only concepts have been banished to // a "detail" namespace indicating they have a niche use-case. namespace __iterator_traits_detail { -template -concept __cpp17_iterator = - requires(_Ip __i) { - { *__i } -> __can_reference; - { ++__i } -> same_as<_Ip&>; - { *__i++ } -> __can_reference; - } && - copyable<_Ip>; - -template -concept __cpp17_input_iterator = - __cpp17_iterator<_Ip> && - equality_comparable<_Ip> && - requires(_Ip __i) { - typename incrementable_traits<_Ip>::difference_type; - typename indirectly_readable_traits<_Ip>::value_type; - typename common_reference_t&&, - typename indirectly_readable_traits<_Ip>::value_type&>; - typename common_reference_t::value_type&>; - requires signed_integral::difference_type>; - }; - -template +template +concept __cpp17_iterator = requires(_Ip __i) { + { *__i } -> __can_reference; + { ++__i } -> same_as<_Ip&>; + { *__i++ } -> __can_reference; +} && copyable<_Ip>; + +template +concept __cpp17_input_iterator = __cpp17_iterator<_Ip> && equality_comparable<_Ip> && requires(_Ip __i) { + typename incrementable_traits<_Ip>::difference_type; + typename indirectly_readable_traits<_Ip>::value_type; + typename common_reference_t&&, typename indirectly_readable_traits<_Ip>::value_type&>; + typename common_reference_t::value_type&>; + requires signed_integral::difference_type>; +}; + +template concept __cpp17_forward_iterator = - __cpp17_input_iterator<_Ip> && - constructible_from<_Ip> && - is_reference_v> && - same_as>, - typename indirectly_readable_traits<_Ip>::value_type> && - requires(_Ip __i) { - { __i++ } -> convertible_to<_Ip const&>; - { *__i++ } -> same_as>; - }; - -template -concept __cpp17_bidirectional_iterator = - __cpp17_forward_iterator<_Ip> && - requires(_Ip __i) { - { --__i } -> same_as<_Ip&>; - { __i-- } -> convertible_to<_Ip const&>; - { *__i-- } -> same_as>; - }; - -template + __cpp17_input_iterator<_Ip> && constructible_from<_Ip> && is_reference_v> && + same_as>, typename indirectly_readable_traits<_Ip>::value_type> && + requires(_Ip __i) { + { __i++ } -> convertible_to<_Ip const&>; + { *__i++ } -> same_as>; + }; + +template +concept __cpp17_bidirectional_iterator = __cpp17_forward_iterator<_Ip> && requires(_Ip __i) { + { --__i } -> same_as<_Ip&>; + { __i-- } -> convertible_to<_Ip const&>; + { *__i-- } -> same_as>; +}; + +template concept __cpp17_random_access_iterator = - __cpp17_bidirectional_iterator<_Ip> && - totally_ordered<_Ip> && - requires(_Ip __i, typename incrementable_traits<_Ip>::difference_type __n) { - { __i += __n } -> same_as<_Ip&>; - { __i -= __n } -> same_as<_Ip&>; - { __i + __n } -> same_as<_Ip>; - { __n + __i } -> same_as<_Ip>; - { __i - __n } -> same_as<_Ip>; - { __i - __i } -> same_as; // NOLINT(misc-redundant-expression) ; This is llvm.org/PR54114 - { __i[__n] } -> convertible_to>; - }; + __cpp17_bidirectional_iterator<_Ip> && totally_ordered<_Ip> && + requires(_Ip __i, typename incrementable_traits<_Ip>::difference_type __n) { + { __i += __n } -> same_as<_Ip&>; + { __i -= __n } -> same_as<_Ip&>; + { __i + __n } -> same_as<_Ip>; + { __n + __i } -> same_as<_Ip>; + { __i - __n } -> same_as<_Ip>; + { __i - __i } -> same_as; // NOLINT(misc-redundant-expression) ; This is llvm.org/PR54114 + { __i[__n] } -> convertible_to>; + }; } // namespace __iterator_traits_detail -template +template concept __has_member_reference = requires { typename _Ip::reference; }; -template +template concept __has_member_pointer = requires { typename _Ip::pointer; }; -template +template concept __has_member_iterator_category = requires { typename _Ip::iterator_category; }; -template +template concept __specifies_members = requires { - typename _Ip::value_type; - typename _Ip::difference_type; - requires __has_member_reference<_Ip>; - requires __has_member_iterator_category<_Ip>; - }; + typename _Ip::value_type; + typename _Ip::difference_type; + requires __has_member_reference<_Ip>; + requires __has_member_iterator_category<_Ip>; +}; -template +template struct __iterator_traits_member_pointer_or_void { using type = void; }; -template<__has_member_pointer _Tp> +template <__has_member_pointer _Tp> struct __iterator_traits_member_pointer_or_void<_Tp> { using type = typename _Tp::pointer; }; -template -concept __cpp17_iterator_missing_members = - !__specifies_members<_Tp> && - __iterator_traits_detail::__cpp17_iterator<_Tp>; +template +concept __cpp17_iterator_missing_members = !__specifies_members<_Tp> && __iterator_traits_detail::__cpp17_iterator<_Tp>; -template +template concept __cpp17_input_iterator_missing_members = - __cpp17_iterator_missing_members<_Tp> && - __iterator_traits_detail::__cpp17_input_iterator<_Tp>; + __cpp17_iterator_missing_members<_Tp> && __iterator_traits_detail::__cpp17_input_iterator<_Tp>; // Otherwise, `pointer` names `void`. -template -struct __iterator_traits_member_pointer_or_arrow_or_void { using type = void; }; +template +struct __iterator_traits_member_pointer_or_arrow_or_void { + using type = void; +}; // [iterator.traits]/3.2.1 // If the qualified-id `I::pointer` is valid and denotes a type, `pointer` names that type. -template<__has_member_pointer _Ip> -struct __iterator_traits_member_pointer_or_arrow_or_void<_Ip> { using type = typename _Ip::pointer; }; +template <__has_member_pointer _Ip> +struct __iterator_traits_member_pointer_or_arrow_or_void<_Ip> { + using type = typename _Ip::pointer; +}; // Otherwise, if `decltype(declval().operator->())` is well-formed, then `pointer` names that // type. -template +template requires requires(_Ip& __i) { __i.operator->(); } && (!__has_member_pointer<_Ip>) struct __iterator_traits_member_pointer_or_arrow_or_void<_Ip> { using type = decltype(std::declval<_Ip&>().operator->()); }; // Otherwise, `reference` names `iter-reference-t`. -template -struct __iterator_traits_member_reference { using type = iter_reference_t<_Ip>; }; +template +struct __iterator_traits_member_reference { + using type = iter_reference_t<_Ip>; +}; // [iterator.traits]/3.2.2 // If the qualified-id `I::reference` is valid and denotes a type, `reference` names that type. -template<__has_member_reference _Ip> -struct __iterator_traits_member_reference<_Ip> { using type = typename _Ip::reference; }; +template <__has_member_reference _Ip> +struct __iterator_traits_member_reference<_Ip> { + using type = typename _Ip::reference; +}; // [iterator.traits]/3.2.3.4 // input_iterator_tag -template +template struct __deduce_iterator_category { using type = input_iterator_tag; }; // [iterator.traits]/3.2.3.1 // `random_access_iterator_tag` if `I` satisfies `cpp17-random-access-iterator`, or otherwise -template<__iterator_traits_detail::__cpp17_random_access_iterator _Ip> +template <__iterator_traits_detail::__cpp17_random_access_iterator _Ip> struct __deduce_iterator_category<_Ip> { using type = random_access_iterator_tag; }; // [iterator.traits]/3.2.3.2 // `bidirectional_iterator_tag` if `I` satisfies `cpp17-bidirectional-iterator`, or otherwise -template<__iterator_traits_detail::__cpp17_bidirectional_iterator _Ip> +template <__iterator_traits_detail::__cpp17_bidirectional_iterator _Ip> struct __deduce_iterator_category<_Ip> { using type = bidirectional_iterator_tag; }; // [iterator.traits]/3.2.3.3 // `forward_iterator_tag` if `I` satisfies `cpp17-forward-iterator`, or otherwise -template<__iterator_traits_detail::__cpp17_forward_iterator _Ip> +template <__iterator_traits_detail::__cpp17_forward_iterator _Ip> struct __deduce_iterator_category<_Ip> { using type = forward_iterator_tag; }; -template +template struct __iterator_traits_iterator_category : __deduce_iterator_category<_Ip> {}; // [iterator.traits]/3.2.3 // If the qualified-id `I::iterator-category` is valid and denotes a type, `iterator-category` names // that type. -template<__has_member_iterator_category _Ip> +template <__has_member_iterator_category _Ip> struct __iterator_traits_iterator_category<_Ip> { using type = typename _Ip::iterator_category; }; // otherwise, it names void. -template -struct __iterator_traits_difference_type { using type = void; }; +template +struct __iterator_traits_difference_type { + using type = void; +}; // If the qualified-id `incrementable_traits::difference_type` is valid and denotes a type, then // `difference_type` names that type; -template -requires requires { typename incrementable_traits<_Ip>::difference_type; } +template + requires requires { typename incrementable_traits<_Ip>::difference_type; } struct __iterator_traits_difference_type<_Ip> { using type = typename incrementable_traits<_Ip>::difference_type; }; // [iterator.traits]/3.4 // Otherwise, `iterator_traits` has no members by any of the above names. -template +template struct __iterator_traits {}; // [iterator.traits]/3.1 // If `I` has valid ([temp.deduct]) member types `difference-type`, `value-type`, `reference`, and // `iterator-category`, then `iterator-traits` has the following publicly accessible members: -template<__specifies_members _Ip> +template <__specifies_members _Ip> struct __iterator_traits<_Ip> { - using iterator_category = typename _Ip::iterator_category; - using value_type = typename _Ip::value_type; - using difference_type = typename _Ip::difference_type; - using pointer = typename __iterator_traits_member_pointer_or_void<_Ip>::type; - using reference = typename _Ip::reference; + using iterator_category = typename _Ip::iterator_category; + using value_type = typename _Ip::value_type; + using difference_type = typename _Ip::difference_type; + using pointer = typename __iterator_traits_member_pointer_or_void<_Ip>::type; + using reference = typename _Ip::reference; }; // [iterator.traits]/3.2 // Otherwise, if `I` satisfies the exposition-only concept `cpp17-input-iterator`, // `iterator-traits` has the following publicly accessible members: -template<__cpp17_input_iterator_missing_members _Ip> +template <__cpp17_input_iterator_missing_members _Ip> struct __iterator_traits<_Ip> { using iterator_category = typename __iterator_traits_iterator_category<_Ip>::type; using value_type = typename indirectly_readable_traits<_Ip>::value_type; @@ -367,7 +352,7 @@ struct __iterator_traits<_Ip> { // Otherwise, if `I` satisfies the exposition-only concept `cpp17-iterator`, then // `iterator_traits` has the following publicly accessible members: -template<__cpp17_iterator_missing_members _Ip> +template <__cpp17_iterator_missing_members _Ip> struct __iterator_traits<_Ip> { using iterator_category = output_iterator_tag; using value_type = void; @@ -376,36 +361,33 @@ struct __iterator_traits<_Ip> { using reference = void; }; -template +template struct iterator_traits : __iterator_traits<_Ip> { using __primary_template = iterator_traits; }; -#else // _LIBCPP_STD_VER >= 20 +#else // _LIBCPP_STD_VER >= 20 -template struct __iterator_traits {}; +template +struct __iterator_traits {}; -template struct __iterator_traits_impl {}; +template +struct __iterator_traits_impl {}; template -struct __iterator_traits_impl<_Iter, true> -{ - typedef typename _Iter::difference_type difference_type; - typedef typename _Iter::value_type value_type; - typedef typename _Iter::pointer pointer; - typedef typename _Iter::reference reference; - typedef typename _Iter::iterator_category iterator_category; +struct __iterator_traits_impl<_Iter, true> { + typedef typename _Iter::difference_type difference_type; + typedef typename _Iter::value_type value_type; + typedef typename _Iter::pointer pointer; + typedef typename _Iter::reference reference; + typedef typename _Iter::iterator_category iterator_category; }; template struct __iterator_traits<_Iter, true> - : __iterator_traits_impl - < - _Iter, - is_convertible::value || - is_convertible::value - > -{}; + : __iterator_traits_impl< _Iter, + is_convertible::value || + is_convertible::value > {}; // iterator_traits will only have the nested types if Iterator::iterator_category // exists. Else iterator_traits will be an empty class. This is a @@ -413,41 +395,35 @@ struct __iterator_traits<_Iter, true> // the client expects instead of failing at compile time. template -struct _LIBCPP_TEMPLATE_VIS iterator_traits - : __iterator_traits<_Iter, __has_iterator_typedefs<_Iter>::value> { - +struct _LIBCPP_TEMPLATE_VIS iterator_traits : __iterator_traits<_Iter, __has_iterator_typedefs<_Iter>::value> { using __primary_template = iterator_traits; }; #endif // _LIBCPP_STD_VER >= 20 -template +template #if _LIBCPP_STD_VER >= 20 -requires is_object_v<_Tp> + requires is_object_v<_Tp> #endif -struct _LIBCPP_TEMPLATE_VIS iterator_traits<_Tp*> -{ - typedef ptrdiff_t difference_type; - typedef __remove_cv_t<_Tp> value_type; - typedef _Tp* pointer; - typedef _Tp& reference; - typedef random_access_iterator_tag iterator_category; +struct _LIBCPP_TEMPLATE_VIS iterator_traits<_Tp*> { + typedef ptrdiff_t difference_type; + typedef __remove_cv_t<_Tp> value_type; + typedef _Tp* pointer; + typedef _Tp& reference; + typedef random_access_iterator_tag iterator_category; #if _LIBCPP_STD_VER >= 20 - typedef contiguous_iterator_tag iterator_concept; + typedef contiguous_iterator_tag iterator_concept; #endif }; template >::value> -struct __has_iterator_category_convertible_to - : is_convertible::iterator_category, _Up> -{}; +struct __has_iterator_category_convertible_to : is_convertible::iterator_category, _Up> { +}; template struct __has_iterator_category_convertible_to<_Tp, _Up, false> : false_type {}; template ::value> -struct __has_iterator_concept_convertible_to - : is_convertible -{}; +struct __has_iterator_concept_convertible_to : is_convertible {}; template struct __has_iterator_concept_convertible_to<_Tp, _Up, false> : false_type {}; @@ -473,10 +449,9 @@ using __has_random_access_iterator_category = __has_iterator_category_convertibl // #if _LIBCPP_STD_VER >= 20 template -struct __libcpp_is_contiguous_iterator : _Or< - __has_iterator_category_convertible_to<_Tp, contiguous_iterator_tag>, - __has_iterator_concept_convertible_to<_Tp, contiguous_iterator_tag> -> {}; +struct __libcpp_is_contiguous_iterator + : _Or< __has_iterator_category_convertible_to<_Tp, contiguous_iterator_tag>, + __has_iterator_concept_convertible_to<_Tp, contiguous_iterator_tag> > {}; #else template struct __libcpp_is_contiguous_iterator : false_type {}; @@ -486,41 +461,40 @@ struct __libcpp_is_contiguous_iterator : false_type {}; template struct __libcpp_is_contiguous_iterator<_Up*> : true_type {}; - template class __wrap_iter; template -using __has_exactly_input_iterator_category - = integral_constant::value && - !__has_iterator_category_convertible_to<_Tp, forward_iterator_tag>::value>; +using __has_exactly_input_iterator_category = + integral_constant::value && + !__has_iterator_category_convertible_to<_Tp, forward_iterator_tag>::value>; template -using __has_exactly_forward_iterator_category - = integral_constant::value && - !__has_iterator_category_convertible_to<_Tp, bidirectional_iterator_tag>::value>; +using __has_exactly_forward_iterator_category = + integral_constant::value && + !__has_iterator_category_convertible_to<_Tp, bidirectional_iterator_tag>::value>; template -using __has_exactly_bidirectional_iterator_category - = integral_constant::value && - !__has_iterator_category_convertible_to<_Tp, random_access_iterator_tag>::value>; +using __has_exactly_bidirectional_iterator_category = + integral_constant::value && + !__has_iterator_category_convertible_to<_Tp, random_access_iterator_tag>::value>; -template +template using __iter_value_type = typename iterator_traits<_InputIterator>::value_type; -template +template using __iter_key_type = __remove_const_t::value_type::first_type>; -template +template using __iter_mapped_type = typename iterator_traits<_InputIterator>::value_type::second_type; -template -using __iter_to_alloc_type = pair< - typename add_const::value_type::first_type>::type, - typename iterator_traits<_InputIterator>::value_type::second_type>; +template +using __iter_to_alloc_type = + pair< typename add_const::value_type::first_type>::type, + typename iterator_traits<_InputIterator>::value_type::second_type>; template using __iterator_category_type = typename iterator_traits<_Iter>::iterator_category; @@ -543,9 +517,10 @@ using __iter_reference = typename iterator_traits<_Iter>::reference; // generated from the primary template, and `iterator_traits::value_type` otherwise. // This has to be in this file and not readable_traits.h to break the include cycle between the two. template -using iter_value_t = typename conditional_t<__is_primary_template > >::value, - indirectly_readable_traits >, - iterator_traits > >::value_type; +using iter_value_t = + typename conditional_t<__is_primary_template > >::value, + indirectly_readable_traits >, + iterator_traits > >::value_type; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/mergeable.h b/libcxx/include/__iterator/mergeable.h index 494fda956e277..7976d751095e5 100644 --- a/libcxx/include/__iterator/mergeable.h +++ b/libcxx/include/__iterator/mergeable.h @@ -24,14 +24,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template +template concept mergeable = - input_iterator<_Input1> && - input_iterator<_Input2> && - weakly_incrementable<_Output> && - indirectly_copyable<_Input1, _Output> && - indirectly_copyable<_Input2, _Output> && + input_iterator<_Input1> && input_iterator<_Input2> && weakly_incrementable<_Output> && + indirectly_copyable<_Input1, _Output> && indirectly_copyable<_Input2, _Output> && indirect_strict_weak_order<_Comp, projected<_Input1, _Proj1>, projected<_Input2, _Proj2>>; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/move_iterator.h b/libcxx/include/__iterator/move_iterator.h index 2f79d9a3d263a..d1bd0138bdda1 100644 --- a/libcxx/include/__iterator/move_iterator.h +++ b/libcxx/include/__iterator/move_iterator.h @@ -45,22 +45,21 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 -template +template struct __move_iter_category_base {}; -template +template requires requires { typename iterator_traits<_Iter>::iterator_category; } struct __move_iter_category_base<_Iter> { - using iterator_category = _If< - derived_from::iterator_category, random_access_iterator_tag>, - random_access_iterator_tag, - typename iterator_traits<_Iter>::iterator_category - >; + using iterator_category = + _If< derived_from::iterator_category, random_access_iterator_tag>, + random_access_iterator_tag, + typename iterator_traits<_Iter>::iterator_category >; }; -template +template concept __move_iter_comparable = requires { - { std::declval() == std::declval<_Sent>() } -> convertible_to; + { std::declval() == std::declval<_Sent>() } -> convertible_to; }; #endif // _LIBCPP_STD_VER >= 20 @@ -70,282 +69,270 @@ class _LIBCPP_TEMPLATE_VIS move_iterator : public __move_iter_category_base<_Iter> #endif { - #if _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 20 + private: - _LIBCPP_HIDE_FROM_ABI - static constexpr auto __get_iter_concept() { - if constexpr (random_access_iterator<_Iter>) { - return random_access_iterator_tag{}; - } else if constexpr (bidirectional_iterator<_Iter>) { - return bidirectional_iterator_tag{}; - } else if constexpr (forward_iterator<_Iter>) { - return forward_iterator_tag{}; - } else { - return input_iterator_tag{}; - } + _LIBCPP_HIDE_FROM_ABI static constexpr auto __get_iter_concept() { + if constexpr (random_access_iterator<_Iter>) { + return random_access_iterator_tag{}; + } else if constexpr (bidirectional_iterator<_Iter>) { + return bidirectional_iterator_tag{}; + } else if constexpr (forward_iterator<_Iter>) { + return forward_iterator_tag{}; + } else { + return input_iterator_tag{}; } + } #endif // _LIBCPP_STD_VER >= 20 + public: #if _LIBCPP_STD_VER >= 20 - using iterator_type = _Iter; - using iterator_concept = decltype(__get_iter_concept()); - // iterator_category is inherited and not always present - using value_type = iter_value_t<_Iter>; - using difference_type = iter_difference_t<_Iter>; - using pointer = _Iter; - using reference = iter_rvalue_reference_t<_Iter>; + using iterator_type = _Iter; + using iterator_concept = decltype(__get_iter_concept()); + // iterator_category is inherited and not always present + using value_type = iter_value_t<_Iter>; + using difference_type = iter_difference_t<_Iter>; + using pointer = _Iter; + using reference = iter_rvalue_reference_t<_Iter>; #else - typedef _Iter iterator_type; - typedef _If< - __has_random_access_iterator_category<_Iter>::value, - random_access_iterator_tag, - typename iterator_traits<_Iter>::iterator_category - > iterator_category; - typedef typename iterator_traits::value_type value_type; - typedef typename iterator_traits::difference_type difference_type; - typedef iterator_type pointer; - - typedef typename iterator_traits::reference __reference; - typedef typename conditional< - is_reference<__reference>::value, - __libcpp_remove_reference_t<__reference>&&, - __reference - >::type reference; + typedef _Iter iterator_type; + typedef _If< __has_random_access_iterator_category<_Iter>::value, + random_access_iterator_tag, + typename iterator_traits<_Iter>::iterator_category > + iterator_category; + typedef typename iterator_traits::value_type value_type; + typedef typename iterator_traits::difference_type difference_type; + typedef iterator_type pointer; + + typedef typename iterator_traits::reference __reference; + typedef typename conditional< is_reference<__reference>::value, + __libcpp_remove_reference_t<__reference>&&, + __reference >::type reference; #endif // _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - explicit move_iterator(_Iter __i) : __current_(std::move(__i)) {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit move_iterator(_Iter __i) : __current_(std::move(__i)) {} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator& operator++() { ++__current_; return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator++() { + ++__current_; + return *this; + } - _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - pointer operator->() const { return __current_; } + _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pointer operator->() const { + return __current_; + } #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI constexpr - move_iterator() requires is_constructible_v<_Iter> : __current_() {} - - template - requires (!_IsSame<_Up, _Iter>::value) && convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr - move_iterator(const move_iterator<_Up>& __u) : __current_(__u.base()) {} - - template - requires (!_IsSame<_Up, _Iter>::value) && - convertible_to && - assignable_from<_Iter&, const _Up&> - _LIBCPP_HIDE_FROM_ABI constexpr - move_iterator& operator=(const move_iterator<_Up>& __u) { - __current_ = __u.base(); - return *this; - } - - _LIBCPP_HIDE_FROM_ABI constexpr const _Iter& base() const & noexcept { return __current_; } - _LIBCPP_HIDE_FROM_ABI constexpr _Iter base() && { return std::move(__current_); } - - _LIBCPP_HIDE_FROM_ABI constexpr - reference operator*() const { return ranges::iter_move(__current_); } - _LIBCPP_HIDE_FROM_ABI constexpr - reference operator[](difference_type __n) const { return ranges::iter_move(__current_ + __n); } - - _LIBCPP_HIDE_FROM_ABI constexpr - auto operator++(int) - requires forward_iterator<_Iter> - { - move_iterator __tmp(*this); ++__current_; return __tmp; - } - - _LIBCPP_HIDE_FROM_ABI constexpr - void operator++(int) { ++__current_; } + _LIBCPP_HIDE_FROM_ABI constexpr move_iterator() + requires is_constructible_v<_Iter> + : __current_() {} + + template + requires(!_IsSame<_Up, _Iter>::value) && convertible_to + _LIBCPP_HIDE_FROM_ABI constexpr move_iterator(const move_iterator<_Up>& __u) : __current_(__u.base()) {} + + template + requires(!_IsSame<_Up, _Iter>::value) && convertible_to && assignable_from<_Iter&, const _Up&> + _LIBCPP_HIDE_FROM_ABI constexpr move_iterator& operator=(const move_iterator<_Up>& __u) { + __current_ = __u.base(); + return *this; + } + + _LIBCPP_HIDE_FROM_ABI constexpr const _Iter& base() const& noexcept { return __current_; } + _LIBCPP_HIDE_FROM_ABI constexpr _Iter base() && { return std::move(__current_); } + + _LIBCPP_HIDE_FROM_ABI constexpr reference operator*() const { return ranges::iter_move(__current_); } + _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](difference_type __n) const { + return ranges::iter_move(__current_ + __n); + } + + _LIBCPP_HIDE_FROM_ABI constexpr auto operator++(int) + requires forward_iterator<_Iter> + { + move_iterator __tmp(*this); + ++__current_; + return __tmp; + } + + _LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++__current_; } #else - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator() : __current_() {} - - template ::value && is_convertible::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator(const move_iterator<_Up>& __u) : __current_(__u.base()) {} - - template ::value && - is_convertible::value && - is_assignable<_Iter&, const _Up&>::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator& operator=(const move_iterator<_Up>& __u) { - __current_ = __u.base(); - return *this; - } - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - _Iter base() const { return __current_; } - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reference operator*() const { return static_cast(*__current_); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reference operator[](difference_type __n) const { return static_cast(__current_[__n]); } - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator operator++(int) { move_iterator __tmp(*this); ++__current_; return __tmp; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator() : __current_() {} + + template ::value && is_convertible::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator(const move_iterator<_Up>& __u) + : __current_(__u.base()) {} + + template ::value && is_convertible::value && + is_assignable<_Iter&, const _Up&>::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator=(const move_iterator<_Up>& __u) { + __current_ = __u.base(); + return *this; + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter base() const { return __current_; } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator*() const { + return static_cast(*__current_); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](difference_type __n) const { + return static_cast(__current_[__n]); + } + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator++(int) { + move_iterator __tmp(*this); + ++__current_; + return __tmp; + } #endif // _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator& operator--() { --__current_; return *this; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator operator--(int) { move_iterator __tmp(*this); --__current_; return __tmp; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator operator+(difference_type __n) const { return move_iterator(__current_ + __n); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator& operator+=(difference_type __n) { __current_ += __n; return *this; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator operator-(difference_type __n) const { return move_iterator(__current_ - __n); } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - move_iterator& operator-=(difference_type __n) { __current_ -= __n; return *this; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator--() { + --__current_; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator--(int) { + move_iterator __tmp(*this); + --__current_; + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator+(difference_type __n) const { + return move_iterator(__current_ + __n); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator+=(difference_type __n) { + __current_ += __n; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator operator-(difference_type __n) const { + return move_iterator(__current_ - __n); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator& operator-=(difference_type __n) { + __current_ -= __n; + return *this; + } #if _LIBCPP_STD_VER >= 20 - template _Sent> - friend _LIBCPP_HIDE_FROM_ABI constexpr - bool operator==(const move_iterator& __x, const move_sentinel<_Sent>& __y) - requires __move_iter_comparable<_Iter, _Sent> - { - return __x.base() == __y.base(); - } - - template _Sent> - friend _LIBCPP_HIDE_FROM_ABI constexpr - iter_difference_t<_Iter> operator-(const move_sentinel<_Sent>& __x, const move_iterator& __y) - { - return __x.base() - __y.base(); - } - - template _Sent> - friend _LIBCPP_HIDE_FROM_ABI constexpr - iter_difference_t<_Iter> operator-(const move_iterator& __x, const move_sentinel<_Sent>& __y) - { - return __x.base() - __y.base(); - } + template _Sent> + friend _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const move_iterator& __x, const move_sentinel<_Sent>& __y) + requires __move_iter_comparable<_Iter, _Sent> + { + return __x.base() == __y.base(); + } - friend _LIBCPP_HIDE_FROM_ABI constexpr - iter_rvalue_reference_t<_Iter> iter_move(const move_iterator& __i) - noexcept(noexcept(ranges::iter_move(__i.__current_))) - { - return ranges::iter_move(__i.__current_); - } + template _Sent> + friend _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter> + operator-(const move_sentinel<_Sent>& __x, const move_iterator& __y) { + return __x.base() - __y.base(); + } - template _It2> - friend _LIBCPP_HIDE_FROM_ABI constexpr - void iter_swap(const move_iterator& __x, const move_iterator<_It2>& __y) - noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) - { - return ranges::iter_swap(__x.__current_, __y.__current_); - } + template _Sent> + friend _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter> + operator-(const move_iterator& __x, const move_sentinel<_Sent>& __y) { + return __x.base() - __y.base(); + } + + friend _LIBCPP_HIDE_FROM_ABI constexpr iter_rvalue_reference_t<_Iter> + iter_move(const move_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__current_))) { + return ranges::iter_move(__i.__current_); + } + + template _It2> + friend _LIBCPP_HIDE_FROM_ABI constexpr void + iter_swap(const move_iterator& __x, + const move_iterator<_It2>& __y) noexcept(noexcept(ranges::iter_swap(__x.__current_, __y.__current_))) { + return ranges::iter_swap(__x.__current_, __y.__current_); + } #endif // _LIBCPP_STD_VER >= 20 private: - template friend class move_iterator; + template + friend class move_iterator; - _Iter __current_; + _Iter __current_; }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(move_iterator); template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator==(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() == __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator==(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() == __y.base(); } #if _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator!=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() != __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator!=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() != __y.base(); } #endif // _LIBCPP_STD_VER <= 17 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator<(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() < __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator<(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() < __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator>(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() > __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator>(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() > __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator<=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() <= __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator<=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() <= __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool operator>=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() >= __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool +operator>=(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() >= __y.base(); } #if _LIBCPP_STD_VER >= 20 template _Iter2> -inline _LIBCPP_HIDE_FROM_ABI constexpr -auto operator<=>(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) - -> compare_three_way_result_t<_Iter1, _Iter2> -{ - return __x.base() <=> __y.base(); +inline _LIBCPP_HIDE_FROM_ABI constexpr auto +operator<=>(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) + -> compare_three_way_result_t<_Iter1, _Iter2> { + return __x.base() <=> __y.base(); } #endif // _LIBCPP_STD_VER >= 20 #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) - -> decltype(__x.base() - __y.base()) -{ - return __x.base() - __y.base(); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto +operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -> decltype(__x.base() - __y.base()) { + return __x.base() - __y.base(); } #else template -inline _LIBCPP_HIDE_FROM_ABI -typename move_iterator<_Iter1>::difference_type -operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) -{ - return __x.base() - __y.base(); +inline _LIBCPP_HIDE_FROM_ABI typename move_iterator<_Iter1>::difference_type +operator-(const move_iterator<_Iter1>& __x, const move_iterator<_Iter2>& __y) { + return __x.base() - __y.base(); } #endif // !_LIBCPP_CXX03_LANG #if _LIBCPP_STD_VER >= 20 template -inline _LIBCPP_HIDE_FROM_ABI constexpr -move_iterator<_Iter> operator+(iter_difference_t<_Iter> __n, const move_iterator<_Iter>& __x) - requires requires { { __x.base() + __n } -> same_as<_Iter>; } +inline _LIBCPP_HIDE_FROM_ABI constexpr move_iterator<_Iter> +operator+(iter_difference_t<_Iter> __n, const move_iterator<_Iter>& __x) + requires requires { + { __x.base() + __n } -> same_as<_Iter>; + } { - return __x + __n; + return __x + __n; } #else template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -move_iterator<_Iter> -operator+(typename move_iterator<_Iter>::difference_type __n, const move_iterator<_Iter>& __x) -{ - return move_iterator<_Iter>(__x.base() + __n); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator<_Iter> +operator+(typename move_iterator<_Iter>::difference_type __n, const move_iterator<_Iter>& __x) { + return move_iterator<_Iter>(__x.base() + __n); } #endif // _LIBCPP_STD_VER >= 20 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -move_iterator<_Iter> -make_move_iterator(_Iter __i) -{ - return move_iterator<_Iter>(std::move(__i)); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator<_Iter> make_move_iterator(_Iter __i) { + return move_iterator<_Iter>(std::move(__i)); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/move_sentinel.h b/libcxx/include/__iterator/move_sentinel.h index 34b3c25417c44..4a2a09ef0611d 100644 --- a/libcxx/include/__iterator/move_sentinel.h +++ b/libcxx/include/__iterator/move_sentinel.h @@ -27,30 +27,27 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 template -class _LIBCPP_TEMPLATE_VIS move_sentinel -{ +class _LIBCPP_TEMPLATE_VIS move_sentinel { public: - _LIBCPP_HIDE_FROM_ABI - move_sentinel() = default; + _LIBCPP_HIDE_FROM_ABI move_sentinel() = default; - _LIBCPP_HIDE_FROM_ABI constexpr - explicit move_sentinel(_Sent __s) : __last_(std::move(__s)) {} + _LIBCPP_HIDE_FROM_ABI constexpr explicit move_sentinel(_Sent __s) : __last_(std::move(__s)) {} template requires convertible_to - _LIBCPP_HIDE_FROM_ABI constexpr - move_sentinel(const move_sentinel<_S2>& __s) : __last_(__s.base()) {} + _LIBCPP_HIDE_FROM_ABI constexpr move_sentinel(const move_sentinel<_S2>& __s) : __last_(__s.base()) {} template requires assignable_from<_Sent&, const _S2&> - _LIBCPP_HIDE_FROM_ABI constexpr - move_sentinel& operator=(const move_sentinel<_S2>& __s) - { __last_ = __s.base(); return *this; } + _LIBCPP_HIDE_FROM_ABI constexpr move_sentinel& operator=(const move_sentinel<_S2>& __s) { + __last_ = __s.base(); + return *this; + } _LIBCPP_HIDE_FROM_ABI constexpr _Sent base() const { return __last_; } private: - _Sent __last_ = _Sent(); + _Sent __last_ = _Sent(); }; _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(move_sentinel); diff --git a/libcxx/include/__iterator/next.h b/libcxx/include/__iterator/next.h index ae98229bdabf4..da60aacfd08d2 100644 --- a/libcxx/include/__iterator/next.h +++ b/libcxx/include/__iterator/next.h @@ -25,9 +25,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - _InputIter - next(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter +next(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, "Attempt to next(it, n) with negative n on a non-bidirectional iterator"); @@ -44,15 +43,13 @@ namespace __next { struct __fn { template - _LIBCPP_HIDE_FROM_ABI - constexpr _Ip operator()(_Ip __x) const { + _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x) const { ++__x; return __x; } template - _LIBCPP_HIDE_FROM_ABI - constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const { + _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const { ranges::advance(__x, __n); return __x; } @@ -73,7 +70,7 @@ struct __fn { } // namespace __next inline namespace __cpo { - inline constexpr auto next = __next::__fn{}; +inline constexpr auto next = __next::__fn{}; } // namespace __cpo } // namespace ranges diff --git a/libcxx/include/__iterator/ostream_iterator.h b/libcxx/include/__iterator/ostream_iterator.h index 381f75832fc65..05697e62d9dcb 100644 --- a/libcxx/include/__iterator/ostream_iterator.h +++ b/libcxx/include/__iterator/ostream_iterator.h @@ -31,40 +31,43 @@ class _LIBCPP_TEMPLATE_VIS ostream_iterator : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + public: - typedef output_iterator_tag iterator_category; - typedef void value_type; + typedef output_iterator_tag iterator_category; + typedef void value_type; #if _LIBCPP_STD_VER >= 20 - typedef ptrdiff_t difference_type; + typedef ptrdiff_t difference_type; #else - typedef void difference_type; + typedef void difference_type; #endif - typedef void pointer; - typedef void reference; - typedef _CharT char_type; - typedef _Traits traits_type; - typedef basic_ostream<_CharT, _Traits> ostream_type; + typedef void pointer; + typedef void reference; + typedef _CharT char_type; + typedef _Traits traits_type; + typedef basic_ostream<_CharT, _Traits> ostream_type; private: - ostream_type* __out_stream_; - const char_type* __delim_; + ostream_type* __out_stream_; + const char_type* __delim_; + public: - _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s) _NOEXCEPT - : __out_stream_(std::addressof(__s)), __delim_(nullptr) {} - _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s, const _CharT* __delimiter) _NOEXCEPT - : __out_stream_(std::addressof(__s)), __delim_(__delimiter) {} - _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator=(const _Tp& __value) - { - *__out_stream_ << __value; - if (__delim_) - *__out_stream_ << __delim_; - return *this; - } + _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s) _NOEXCEPT + : __out_stream_(std::addressof(__s)), + __delim_(nullptr) {} + _LIBCPP_HIDE_FROM_ABI ostream_iterator(ostream_type& __s, const _CharT* __delimiter) _NOEXCEPT + : __out_stream_(std::addressof(__s)), + __delim_(__delimiter) {} + _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator=(const _Tp& __value) { + *__out_stream_ << __value; + if (__delim_) + *__out_stream_ << __delim_; + return *this; + } - _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator*() {return *this;} - _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++() {return *this;} - _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++(int) {return *this;} + _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator*() { return *this; } + _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++() { return *this; } + _LIBCPP_HIDE_FROM_ABI ostream_iterator& operator++(int) { return *this; } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/ostreambuf_iterator.h b/libcxx/include/__iterator/ostreambuf_iterator.h index 6f7491eda5b49..dda0094dc3f53 100644 --- a/libcxx/include/__iterator/ostreambuf_iterator.h +++ b/libcxx/include/__iterator/ostreambuf_iterator.h @@ -29,47 +29,42 @@ class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator : public iterator #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + public: - typedef output_iterator_tag iterator_category; - typedef void value_type; + typedef output_iterator_tag iterator_category; + typedef void value_type; #if _LIBCPP_STD_VER >= 20 - typedef ptrdiff_t difference_type; + typedef ptrdiff_t difference_type; #else - typedef void difference_type; + typedef void difference_type; #endif - typedef void pointer; - typedef void reference; - typedef _CharT char_type; - typedef _Traits traits_type; - typedef basic_streambuf<_CharT, _Traits> streambuf_type; - typedef basic_ostream<_CharT, _Traits> ostream_type; + typedef void pointer; + typedef void reference; + typedef _CharT char_type; + typedef _Traits traits_type; + typedef basic_streambuf<_CharT, _Traits> streambuf_type; + typedef basic_ostream<_CharT, _Traits> ostream_type; private: - streambuf_type* __sbuf_; + streambuf_type* __sbuf_; + public: - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(ostream_type& __s) _NOEXCEPT - : __sbuf_(__s.rdbuf()) {} - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(streambuf_type* __s) _NOEXCEPT - : __sbuf_(__s) {} - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator=(_CharT __c) - { - if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sputc(__c), traits_type::eof())) - __sbuf_ = nullptr; - return *this; - } - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator*() {return *this;} - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++() {return *this;} - _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++(int) {return *this;} - _LIBCPP_HIDE_FROM_ABI bool failed() const _NOEXCEPT {return __sbuf_ == nullptr;} + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(ostream_type& __s) _NOEXCEPT : __sbuf_(__s.rdbuf()) {} + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator(streambuf_type* __s) _NOEXCEPT : __sbuf_(__s) {} + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator=(_CharT __c) { + if (__sbuf_ && traits_type::eq_int_type(__sbuf_->sputc(__c), traits_type::eof())) + __sbuf_ = nullptr; + return *this; + } + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator*() { return *this; } + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++() { return *this; } + _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator& operator++(int) { return *this; } + _LIBCPP_HIDE_FROM_ABI bool failed() const _NOEXCEPT { return __sbuf_ == nullptr; } - template - friend - _LIBCPP_HIDE_FROM_ABI - ostreambuf_iterator<_Ch, _Tr> - __pad_and_output(ostreambuf_iterator<_Ch, _Tr> __s, - const _Ch* __ob, const _Ch* __op, const _Ch* __oe, - ios_base& __iob, _Ch __fl); + template + friend _LIBCPP_HIDE_FROM_ABI ostreambuf_iterator<_Ch, _Tr> __pad_and_output( + ostreambuf_iterator<_Ch, _Tr> __s, const _Ch* __ob, const _Ch* __op, const _Ch* __oe, ios_base& __iob, _Ch __fl); }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/permutable.h b/libcxx/include/__iterator/permutable.h index adf88f506529f..f65ba3bfbbaad 100644 --- a/libcxx/include/__iterator/permutable.h +++ b/libcxx/include/__iterator/permutable.h @@ -24,8 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD template concept permutable = - forward_iterator<_Iterator> && - indirectly_movable_storable<_Iterator, _Iterator> && + forward_iterator<_Iterator> && indirectly_movable_storable<_Iterator, _Iterator> && indirectly_swappable<_Iterator, _Iterator>; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h index 3bc29a27e743e..1651942acea9e 100644 --- a/libcxx/include/__iterator/prev.h +++ b/libcxx/include/__iterator/prev.h @@ -25,9 +25,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - _InputIter - prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter +prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { _LIBCPP_ASSERT_UNCATEGORIZED(__n <= 0 || __has_bidirectional_iterator_category<_InputIter>::value, "Attempt to prev(it, n) with a positive n on a non-bidirectional iterator"); std::advance(__x, -__n); @@ -43,15 +42,13 @@ namespace __prev { struct __fn { template - _LIBCPP_HIDE_FROM_ABI - constexpr _Ip operator()(_Ip __x) const { + _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x) const { --__x; return __x; } template - _LIBCPP_HIDE_FROM_ABI - constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const { + _LIBCPP_HIDE_FROM_ABI constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const { ranges::advance(__x, -__n); return __x; } @@ -66,7 +63,7 @@ struct __fn { } // namespace __prev inline namespace __cpo { - inline constexpr auto prev = __prev::__fn{}; +inline constexpr auto prev = __prev::__fn{}; } // namespace __cpo } // namespace ranges diff --git a/libcxx/include/__iterator/readable_traits.h b/libcxx/include/__iterator/readable_traits.h index 9e101fc28a6f1..25e74567fff11 100644 --- a/libcxx/include/__iterator/readable_traits.h +++ b/libcxx/include/__iterator/readable_traits.h @@ -29,50 +29,50 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 20 // [readable.traits] -template struct __cond_value_type {}; +template +struct __cond_value_type {}; -template -requires is_object_v<_Tp> -struct __cond_value_type<_Tp> { using value_type = remove_cv_t<_Tp>; }; +template + requires is_object_v<_Tp> +struct __cond_value_type<_Tp> { + using value_type = remove_cv_t<_Tp>; +}; -template +template concept __has_member_value_type = requires { typename _Tp::value_type; }; -template +template concept __has_member_element_type = requires { typename _Tp::element_type; }; -template struct indirectly_readable_traits {}; +template +struct indirectly_readable_traits {}; -template -requires is_array_v<_Ip> +template + requires is_array_v<_Ip> struct indirectly_readable_traits<_Ip> { using value_type = remove_cv_t>; }; -template +template struct indirectly_readable_traits : indirectly_readable_traits<_Ip> {}; -template +template struct indirectly_readable_traits<_Tp*> : __cond_value_type<_Tp> {}; -template<__has_member_value_type _Tp> -struct indirectly_readable_traits<_Tp> - : __cond_value_type {}; +template <__has_member_value_type _Tp> +struct indirectly_readable_traits<_Tp> : __cond_value_type {}; -template<__has_member_element_type _Tp> -struct indirectly_readable_traits<_Tp> - : __cond_value_type {}; +template <__has_member_element_type _Tp> +struct indirectly_readable_traits<_Tp> : __cond_value_type {}; -template<__has_member_value_type _Tp> +template <__has_member_value_type _Tp> requires __has_member_element_type<_Tp> struct indirectly_readable_traits<_Tp> {}; -template<__has_member_value_type _Tp> +template <__has_member_value_type _Tp> requires __has_member_element_type<_Tp> && - same_as, - remove_cv_t> -struct indirectly_readable_traits<_Tp> - : __cond_value_type {}; + same_as, remove_cv_t> +struct indirectly_readable_traits<_Tp> : __cond_value_type {}; #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__iterator/reverse_access.h b/libcxx/include/__iterator/reverse_access.h index 5e01f667e115f..54d7270b04a53 100644 --- a/libcxx/include/__iterator/reverse_access.h +++ b/libcxx/include/__iterator/reverse_access.h @@ -24,73 +24,53 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if _LIBCPP_STD_VER >= 14 template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator<_Tp*> rbegin(_Tp (&__array)[_Np]) -{ - return reverse_iterator<_Tp*>(__array + _Np); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Tp*> rbegin(_Tp (&__array)[_Np]) { + return reverse_iterator<_Tp*>(__array + _Np); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator<_Tp*> rend(_Tp (&__array)[_Np]) -{ - return reverse_iterator<_Tp*>(__array); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Tp*> rend(_Tp (&__array)[_Np]) { + return reverse_iterator<_Tp*>(__array); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator rbegin(initializer_list<_Ep> __il) -{ - return reverse_iterator(__il.end()); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin(initializer_list<_Ep> __il) { + return reverse_iterator(__il.end()); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator rend(initializer_list<_Ep> __il) -{ - return reverse_iterator(__il.begin()); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend(initializer_list<_Ep> __il) { + return reverse_iterator(__il.begin()); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto rbegin(_Cp& __c) -> decltype(__c.rbegin()) -{ - return __c.rbegin(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rbegin(_Cp& __c) -> decltype(__c.rbegin()) { + return __c.rbegin(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto rbegin(const _Cp& __c) -> decltype(__c.rbegin()) -{ - return __c.rbegin(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rbegin(const _Cp& __c) -> decltype(__c.rbegin()) { + return __c.rbegin(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto rend(_Cp& __c) -> decltype(__c.rend()) -{ - return __c.rend(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rend(_Cp& __c) -> decltype(__c.rend()) { + return __c.rend(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto rend(const _Cp& __c) -> decltype(__c.rend()) -{ - return __c.rend(); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto rend(const _Cp& __c) -> decltype(__c.rend()) { + return __c.rend(); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto crbegin(const _Cp& __c) -> decltype(std::rbegin(__c)) -{ - return std::rbegin(__c); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto crbegin(const _Cp& __c) -> decltype(std::rbegin(__c)) { + return std::rbegin(__c); } template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto crend(const _Cp& __c) -> decltype(std::rend(__c)) -{ - return std::rend(__c); +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto crend(const _Cp& __c) -> decltype(std::rend(__c)) { + return std::rend(__c); } #endif // _LIBCPP_STD_VER >= 14 diff --git a/libcxx/include/__iterator/reverse_iterator.h b/libcxx/include/__iterator/reverse_iterator.h index 552699bfafa4e..79b48bcea57a1 100644 --- a/libcxx/include/__iterator/reverse_iterator.h +++ b/libcxx/include/__iterator/reverse_iterator.h @@ -57,279 +57,262 @@ class _LIBCPP_TEMPLATE_VIS reverse_iterator typename iterator_traits<_Iter>::reference> #endif { -_LIBCPP_SUPPRESS_DEPRECATED_POP + _LIBCPP_SUPPRESS_DEPRECATED_POP + private: #ifndef _LIBCPP_ABI_NO_ITERATOR_BASES - _Iter __t_; // no longer used as of LWG #2360, not removed due to ABI break + _Iter __t_; // no longer used as of LWG #2360, not removed due to ABI break #endif #if _LIBCPP_STD_VER >= 20 - static_assert(__has_bidirectional_iterator_category<_Iter>::value || bidirectional_iterator<_Iter>, - "reverse_iterator requires It to be a bidirectional iterator."); + static_assert(__has_bidirectional_iterator_category<_Iter>::value || bidirectional_iterator<_Iter>, + "reverse_iterator requires It to be a bidirectional iterator."); #endif // _LIBCPP_STD_VER >= 20 protected: - _Iter current; + _Iter current; + public: - using iterator_type = _Iter; + using iterator_type = _Iter; - using iterator_category = _If<__has_random_access_iterator_category<_Iter>::value, - random_access_iterator_tag, - typename iterator_traits<_Iter>::iterator_category>; - using pointer = typename iterator_traits<_Iter>::pointer; + using iterator_category = + _If<__has_random_access_iterator_category<_Iter>::value, + random_access_iterator_tag, + typename iterator_traits<_Iter>::iterator_category>; + using pointer = typename iterator_traits<_Iter>::pointer; #if _LIBCPP_STD_VER >= 20 - using iterator_concept = _If, random_access_iterator_tag, bidirectional_iterator_tag>; - using value_type = iter_value_t<_Iter>; - using difference_type = iter_difference_t<_Iter>; - using reference = iter_reference_t<_Iter>; + using iterator_concept = _If, random_access_iterator_tag, bidirectional_iterator_tag>; + using value_type = iter_value_t<_Iter>; + using difference_type = iter_difference_t<_Iter>; + using reference = iter_reference_t<_Iter>; #else - using value_type = typename iterator_traits<_Iter>::value_type; - using difference_type = typename iterator_traits<_Iter>::difference_type; - using reference = typename iterator_traits<_Iter>::reference; + using value_type = typename iterator_traits<_Iter>::value_type; + using difference_type = typename iterator_traits<_Iter>::difference_type; + using reference = typename iterator_traits<_Iter>::reference; #endif #ifndef _LIBCPP_ABI_NO_ITERATOR_BASES - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator() : __t_(), current() {} - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - explicit reverse_iterator(_Iter __x) : __t_(__x), current(__x) {} - - template ::value && is_convertible<_Up const&, _Iter>::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator(const reverse_iterator<_Up>& __u) - : __t_(__u.base()), current(__u.base()) - { } - - template ::value && - is_convertible<_Up const&, _Iter>::value && - is_assignable<_Iter&, _Up const&>::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator=(const reverse_iterator<_Up>& __u) { - __t_ = current = __u.base(); - return *this; - } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator() : __t_(), current() {} + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit reverse_iterator(_Iter __x) : __t_(__x), current(__x) {} + + template ::value && is_convertible<_Up const&, _Iter>::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator(const reverse_iterator<_Up>& __u) + : __t_(__u.base()), current(__u.base()) {} + + template ::value && is_convertible<_Up const&, _Iter>::value && + is_assignable<_Iter&, _Up const&>::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator=(const reverse_iterator<_Up>& __u) { + __t_ = current = __u.base(); + return *this; + } #else - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator() : current() {} - - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - explicit reverse_iterator(_Iter __x) : current(__x) {} - - template ::value && is_convertible<_Up const&, _Iter>::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator(const reverse_iterator<_Up>& __u) - : current(__u.base()) - { } - - template ::value && - is_convertible<_Up const&, _Iter>::value && - is_assignable<_Iter&, _Up const&>::value - > > - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator=(const reverse_iterator<_Up>& __u) { - current = __u.base(); - return *this; - } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator() : current() {} + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit reverse_iterator(_Iter __x) : current(__x) {} + + template ::value && is_convertible<_Up const&, _Iter>::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator(const reverse_iterator<_Up>& __u) + : current(__u.base()) {} + + template ::value && is_convertible<_Up const&, _Iter>::value && + is_assignable<_Iter&, _Up const&>::value > > + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator=(const reverse_iterator<_Up>& __u) { + current = __u.base(); + return *this; + } #endif - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - _Iter base() const {return current;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reference operator*() const {_Iter __tmp = current; return *--__tmp;} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter base() const { return current; } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator*() const { + _Iter __tmp = current; + return *--__tmp; + } #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI - constexpr pointer operator->() const - requires is_pointer_v<_Iter> || requires(const _Iter __i) { __i.operator->(); } - { - if constexpr (is_pointer_v<_Iter>) { - return std::prev(current); - } else { - return std::prev(current).operator->(); - } + _LIBCPP_HIDE_FROM_ABI constexpr pointer operator->() const + requires is_pointer_v<_Iter> || requires(const _Iter __i) { __i.operator->(); } + { + if constexpr (is_pointer_v<_Iter>) { + return std::prev(current); + } else { + return std::prev(current).operator->(); } + } #else - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - pointer operator->() const { - return std::addressof(operator*()); - } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pointer operator->() const { return std::addressof(operator*()); } #endif // _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator++() {--current; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator operator++(int) {reverse_iterator __tmp(*this); --current; return __tmp;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator--() {++current; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator operator--(int) {reverse_iterator __tmp(*this); ++current; return __tmp;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator operator+(difference_type __n) const {return reverse_iterator(current - __n);} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator+=(difference_type __n) {current -= __n; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator operator-(difference_type __n) const {return reverse_iterator(current + __n);} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reverse_iterator& operator-=(difference_type __n) {current += __n; return *this;} - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 - reference operator[](difference_type __n) const {return *(*this + __n);} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator++() { + --current; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator++(int) { + reverse_iterator __tmp(*this); + --current; + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator--() { + ++current; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator--(int) { + reverse_iterator __tmp(*this); + ++current; + return __tmp; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator+(difference_type __n) const { + return reverse_iterator(current - __n); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator+=(difference_type __n) { + current -= __n; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator operator-(difference_type __n) const { + return reverse_iterator(current + __n); + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator& operator-=(difference_type __n) { + current += __n; + return *this; + } + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](difference_type __n) const { + return *(*this + __n); + } #if _LIBCPP_STD_VER >= 20 - _LIBCPP_HIDE_FROM_ABI friend constexpr - iter_rvalue_reference_t<_Iter> iter_move(const reverse_iterator& __i) - noexcept(is_nothrow_copy_constructible_v<_Iter> && - noexcept(ranges::iter_move(--std::declval<_Iter&>()))) { - auto __tmp = __i.base(); - return ranges::iter_move(--__tmp); - } + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> iter_move(const reverse_iterator& __i) noexcept( + is_nothrow_copy_constructible_v<_Iter>&& noexcept(ranges::iter_move(--std::declval<_Iter&>()))) { + auto __tmp = __i.base(); + return ranges::iter_move(--__tmp); + } - template _Iter2> - _LIBCPP_HIDE_FROM_ABI friend constexpr - void iter_swap(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) - noexcept(is_nothrow_copy_constructible_v<_Iter> && - is_nothrow_copy_constructible_v<_Iter2> && - noexcept(ranges::iter_swap(--std::declval<_Iter&>(), --std::declval<_Iter2&>()))) { - auto __xtmp = __x.base(); - auto __ytmp = __y.base(); - ranges::iter_swap(--__xtmp, --__ytmp); - } + template _Iter2> + _LIBCPP_HIDE_FROM_ABI friend constexpr void + iter_swap(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) noexcept( + is_nothrow_copy_constructible_v<_Iter> && + is_nothrow_copy_constructible_v<_Iter2>&& noexcept( + ranges::iter_swap(--std::declval<_Iter&>(), --std::declval<_Iter2&>()))) { + auto __xtmp = __x.base(); + auto __ytmp = __y.base(); + ranges::iter_swap(--__xtmp, --__ytmp); + } #endif // _LIBCPP_STD_VER >= 20 }; template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator==(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() == __y.base() } -> convertible_to; - } + requires requires { + { __x.base() == __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() == __y.base(); + return __x.base() == __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator<(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() > __y.base() } -> convertible_to; - } + requires requires { + { __x.base() > __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() > __y.base(); + return __x.base() > __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator!=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() != __y.base() } -> convertible_to; - } + requires requires { + { __x.base() != __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() != __y.base(); + return __x.base() != __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator>(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() < __y.base() } -> convertible_to; - } + requires requires { + { __x.base() < __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() < __y.base(); + return __x.base() < __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator>=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() <= __y.base() } -> convertible_to; - } + requires requires { + { __x.base() <= __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() <= __y.base(); + return __x.base() <= __y.base(); } template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -bool +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 bool operator<=(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) #if _LIBCPP_STD_VER >= 20 - requires requires { - { __x.base() >= __y.base() } -> convertible_to; - } + requires requires { + { __x.base() >= __y.base() } -> convertible_to; + } #endif // _LIBCPP_STD_VER >= 20 { - return __x.base() >= __y.base(); + return __x.base() >= __y.base(); } #if _LIBCPP_STD_VER >= 20 template _Iter2> -_LIBCPP_HIDE_FROM_ABI constexpr -compare_three_way_result_t<_Iter1, _Iter2> -operator<=>(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) -{ - return __y.base() <=> __x.base(); +_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Iter1, _Iter2> +operator<=>(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) { + return __y.base() <=> __x.base(); } #endif // _LIBCPP_STD_VER >= 20 #ifndef _LIBCPP_CXX03_LANG template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -auto +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 auto operator-(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) --> decltype(__y.base() - __x.base()) -{ - return __y.base() - __x.base(); + -> decltype(__y.base() - __x.base()) { + return __y.base() - __x.base(); } #else template -inline _LIBCPP_HIDE_FROM_ABI -typename reverse_iterator<_Iter1>::difference_type -operator-(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) -{ - return __y.base() - __x.base(); +inline _LIBCPP_HIDE_FROM_ABI typename reverse_iterator<_Iter1>::difference_type +operator-(const reverse_iterator<_Iter1>& __x, const reverse_iterator<_Iter2>& __y) { + return __y.base() - __x.base(); } #endif template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator<_Iter> -operator+(typename reverse_iterator<_Iter>::difference_type __n, const reverse_iterator<_Iter>& __x) -{ - return reverse_iterator<_Iter>(__x.base() - __n); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Iter> +operator+(typename reverse_iterator<_Iter>::difference_type __n, const reverse_iterator<_Iter>& __x) { + return reverse_iterator<_Iter>(__x.base() - __n); } #if _LIBCPP_STD_VER >= 20 template - requires (!sized_sentinel_for<_Iter1, _Iter2>) + requires(!sized_sentinel_for<_Iter1, _Iter2>) inline constexpr bool disable_sized_sentinel_for, reverse_iterator<_Iter2>> = true; #endif // _LIBCPP_STD_VER >= 20 #if _LIBCPP_STD_VER >= 14 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -reverse_iterator<_Iter> make_reverse_iterator(_Iter __i) -{ - return reverse_iterator<_Iter>(__i); +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator<_Iter> make_reverse_iterator(_Iter __i) { + return reverse_iterator<_Iter>(__i); } #endif @@ -369,13 +352,15 @@ class __unconstrained_reverse_iterator { using iterator_type = _Iter; using iterator_category = - _If<__has_random_access_iterator_category<_Iter>::value, random_access_iterator_tag, __iterator_category_type<_Iter>>; - using pointer = __iterator_pointer_type<_Iter>; - using value_type = iter_value_t<_Iter>; + _If<__has_random_access_iterator_category<_Iter>::value, + random_access_iterator_tag, + __iterator_category_type<_Iter>>; + using pointer = __iterator_pointer_type<_Iter>; + using value_type = iter_value_t<_Iter>; using difference_type = iter_difference_t<_Iter>; - using reference = iter_reference_t<_Iter>; + using reference = iter_reference_t<_Iter>; - _LIBCPP_HIDE_FROM_ABI constexpr __unconstrained_reverse_iterator() = default; + _LIBCPP_HIDE_FROM_ABI constexpr __unconstrained_reverse_iterator() = default; _LIBCPP_HIDE_FROM_ABI constexpr __unconstrained_reverse_iterator(const __unconstrained_reverse_iterator&) = default; _LIBCPP_HIDE_FROM_ABI constexpr explicit __unconstrained_reverse_iterator(_Iter __iter) : __iter_(__iter) {} @@ -393,10 +378,9 @@ class __unconstrained_reverse_iterator { } } - _LIBCPP_HIDE_FROM_ABI friend constexpr - iter_rvalue_reference_t<_Iter> iter_move(const __unconstrained_reverse_iterator& __i) - noexcept(is_nothrow_copy_constructible_v<_Iter> && - noexcept(ranges::iter_move(--std::declval<_Iter&>()))) { + _LIBCPP_HIDE_FROM_ABI friend constexpr iter_rvalue_reference_t<_Iter> + iter_move(const __unconstrained_reverse_iterator& __i) noexcept( + is_nothrow_copy_constructible_v<_Iter>&& noexcept(ranges::iter_move(--std::declval<_Iter&>()))) { auto __tmp = __i.base(); return ranges::iter_move(--__tmp); } @@ -484,7 +468,7 @@ class __unconstrained_reverse_iterator { template