diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index dbbb5e75adc66..e2dcf643082d5 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -437,6 +437,13 @@ class VPBuilder { new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL)); } + VPInstruction *createScalarGEP(Type *SourceElementTy, ArrayRef Ops, + DebugLoc DL, const VPIRFlags &Flags, + const VPIRMetadata &Metadata = {}) { + return tryInsertInstruction( + new VPGEPInstruction(SourceElementTy, Ops, Flags, Metadata, DL)); + } + VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy, DebugLoc DL) { if (ResultTy == SrcTy) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 83475129cdebf..d32caa15b3c0b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1596,6 +1596,51 @@ class VPInstructionWithType : public VPInstruction { #endif }; +class VPGEPInstruction : public VPInstruction { + /// The source element type of the GEP. + Type *SourceElementTy; + +public: + VPGEPInstruction(Type *SourceElementTy, ArrayRef Operands, + const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, + DebugLoc DL = DebugLoc::getUnknown(), Twine Name = "") + : VPInstruction(Instruction::GetElementPtr, Operands, Flags, Metadata, DL, + Name, Operands[0]->getScalarType()), + SourceElementTy(SourceElementTy) {} + + static inline bool classof(const VPRecipeBase *R) { + auto *VPI = dyn_cast(R); + return VPI && VPI->getOpcode() == Instruction::GetElementPtr; + } + + Type *getSourceElementType() const { return SourceElementTy; } + + VPGEPInstruction *clone() override { + auto *New = new VPGEPInstruction(getSourceElementType(), operands(), *this, + *this, getDebugLoc(), getName()); + New->setUnderlyingValue(getUnderlyingValue()); + return New; + } + + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override { + // We mark this instruction as zero-cost because the cost of GEPs in + // vectorized code depends on whether the corresponding memory instruction + // is scalarized or not. Therefore, we handle GEPs with the memory + // instruction cost. + return 0; + } + + void execute(VPTransformState &State) override; + +protected: +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + void printRecipe(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif +}; + /// Helper type to provide functions to access incoming values and blocks for /// phi-like recipes. class VPPhiAccessors { diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index aa14c0329be66..804da3ada2cab 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -271,6 +271,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, CI->getType(), CI->getDebugLoc(), VPIRFlags(*CI), MD); NewR->setUnderlyingValue(CI); + } else if (auto *GEP = dyn_cast(Inst)) { + NewR = VPIRBuilder.createScalarGEP(GEP->getSourceElementType(), + VPOperands, GEP->getDebugLoc(), + VPIRFlags(*GEP), MD); + NewR->setUnderlyingValue(GEP); } else if (auto *LI = dyn_cast(Inst)) { NewR = VPIRBuilder.createScalarLoad(LI->getType(), VPOperands[0], LI->getDebugLoc(), MD); diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 50a3e8abde71a..4a2ef379be36e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -774,6 +774,7 @@ inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) { Recipe_match, Instruction::GetElementPtr, /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>( Op0, Op1), + VPInstruction_match(Op0, Op1), VPInstruction_match(Op0, Op1), VPInstruction_match(Op0, Op1)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index f78651cf8de6c..f3875b5f61883 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1454,6 +1454,7 @@ bool VPInstruction::isVectorToScalar() const { bool VPInstruction::isSingleScalar() const { switch (getOpcode()) { + case Instruction::GetElementPtr: case Instruction::Load: case Instruction::PHI: case VPInstruction::ExplicitVectorLength: @@ -1612,6 +1613,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const { case Instruction::ICmp: case Instruction::Select: case Instruction::Or: + case Instruction::GetElementPtr: case Instruction::Freeze: case VPInstruction::Not: // TODO: Cover additional opcodes. @@ -1854,6 +1856,27 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent, } #endif +void VPGEPInstruction::execute(VPTransformState &State) { + auto Ops = map_to_vector(operands(), + [&](VPValue *Op) { return State.get(Op, true); }); + Value *GEP = + State.Builder.CreateGEP(getSourceElementType(), Ops.front(), + drop_begin(Ops), "", getGEPNoWrapFlags()); + State.set(this, GEP, true); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPGEPInstruction::printRecipe(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "EMIT-SCALAR "; + printAsOperand(O, SlotTracker); + O << " = getelementptr"; + printFlags(O); + O << *getSourceElementType() << " "; + printOperands(O, SlotTracker); +} +#endif + void VPPhi::execute(VPTransformState &State) { PHINode *NewPhi = State.Builder.CreatePHI(getScalarType(), 2, getName()); unsigned NumIncoming = getNumIncoming(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 0fa411450ea92..0a6ec78c708c6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1260,6 +1260,23 @@ getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) { .Default([](auto *) { return std::nullopt; }); } +/// If recipe \p R will lower to a GEP with a non-i8 source element type, +/// return that source element type. +static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) { + // All VPInstructions that lower to GEPs must have the i8 source element + // type (as they are PtrAdds), so we omit it. + return TypeSwitch(R) + .Case([](const VPReplicateRecipe *I) -> Type * { + if (auto *GEP = dyn_cast(I->getUnderlyingValue())) + return GEP->getSourceElementType(); + return nullptr; + }) + .Case( + [](auto *I) { return I->getSourceElementType(); }) + .Case([](auto *I) { return I->getSourceElementType(); }) + .Default([](auto *) { return nullptr; }); +} + /// Try to fold \p R using InstSimplifyFolder. Will succeed and return a /// non-nullptr VPValue for a handled opcode or intrinsic ID if corresponding \p /// Operands are foldable live-ins. @@ -1310,8 +1327,7 @@ static VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, Ops[1]); case Instruction::GetElementPtr: { auto &RFlags = cast(R); - auto *GEP = cast(RFlags.getUnderlyingInstr()); - return Folder.FoldGEP(GEP->getSourceElementType(), Ops[0], + return Folder.FoldGEP(getGEPSourceElementType(&RFlags), Ops[0], drop_begin(Ops), RFlags.getGEPNoWrapFlags()); } case VPInstruction::PtrAdd: @@ -2409,22 +2425,6 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { namespace { struct VPCSEDenseMapInfo : public DenseMapInfo { - /// If recipe \p R will lower to a GEP with a non-i8 source element type, - /// return that source element type. - static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) { - // All VPInstructions that lower to GEPs must have the i8 source element - // type (as they are PtrAdds), so we omit it. - return TypeSwitch(R) - .Case([](const VPReplicateRecipe *I) -> Type * { - if (auto *GEP = dyn_cast(I->getUnderlyingValue())) - return GEP->getSourceElementType(); - return nullptr; - }) - .Case( - [](auto *I) { return I->getSourceElementType(); }) - .Default([](auto *) { return nullptr; }); - } - /// Returns true if recipe \p Def can be safely handed for CSE. static bool canHandle(const VPSingleDefRecipe *Def) { // We can extend the list of handled recipes in the future, diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll index 7c9edb0a8e283..17bbae2b0282f 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll @@ -28,7 +28,7 @@ define void @reverse_unmasked_load_feeds_address(ptr noalias %src, i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<%n.minus.1>, ir<-1>, vp<[[VP0]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%src>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr double ir<%src>, ir<%iv> ; CHECK-NEXT: REPLICATE ir<%val> = load ir<%gep> ; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00> ; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b> @@ -82,13 +82,13 @@ define void @mixed_address_and_vector_uses(ptr noalias %src, ptr noalias %dst, i ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> -; CHECK-NEXT: EMIT ir<%gep.src> = getelementptr ir<%src>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.src> = getelementptr double ir<%src>, ir<%iv> ; CHECK-NEXT: REPLICATE ir<%val> = load ir<%gep.src> ; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00> ; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b> ; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel> ; CHECK-NEXT: EMIT ir<%doubled> = fmul ir<%val>, ir<2.000000e+00> -; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.dst> = getelementptr double ir<%dst>, ir<%iv> ; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1> ; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%doubled> ; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> @@ -143,9 +143,9 @@ define void @interleave_member_feeds_address(ptr noalias %arr, i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> -; CHECK-NEXT: EMIT ir<%gep.p> = getelementptr ir<%arr>, ir<%iv>, ir<0> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.p> = getelementptr %struct.pair = type { ptr, double } ir<%arr>, ir<%iv>, ir<0> ; CHECK-NEXT: REPLICATE ir<%p> = load ir<%gep.p> -; CHECK-NEXT: EMIT ir<%gep.v> = getelementptr ir<%arr>, ir<%iv>, ir<1> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.v> = getelementptr %struct.pair = type { ptr, double } ir<%arr>, ir<%iv>, ir<1> ; CHECK-NEXT: REPLICATE ir<%v> = load ir<%gep.v> ; CHECK-NEXT: REPLICATE store ir<%v>, ir<%p> ; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> @@ -196,11 +196,11 @@ define void @symbolic_stride_versioned_to_one(ptr noalias %src, ptr noalias %dst ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> -; CHECK-NEXT: EMIT ir<%gep.src> = getelementptr ir<%src>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.src> = getelementptr double ir<%src>, ir<%iv> ; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.src>, ir<1> ; CHECK-NEXT: WIDEN ir<%val> = load vp<[[VP4]]> ; CHECK-NEXT: EMIT ir<%doubled> = fmul ir<%val>, ir<2.000000e+00> -; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.dst> = getelementptr double ir<%dst>, ir<%iv> ; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1> ; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%doubled> ; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll index a2c71c4aa6a76..ec55e4deaaef0 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll @@ -26,9 +26,9 @@ define void @f1() { ; CHECK-NEXT: bb2: ; CHECK-NEXT: ir<%c.1.0> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VF]]> ; CHECK-NEXT: EMIT-SCALAR ir<%_tmp1> = zext ir<0> to i64 -; CHECK-NEXT: EMIT ir<%_tmp2> = getelementptr ir<@a>, ir<0>, ir<0> +; CHECK-NEXT: EMIT-SCALAR ir<%_tmp2> = getelementptr [1 x %rec8] ir<@a>, ir<0>, ir<0> ; CHECK-NEXT: EMIT-SCALAR ir<%_tmp6> = sext ir<%c.1.0> to i64 -; CHECK-NEXT: EMIT ir<%_tmp7> = getelementptr ir<@b>, ir<0>, ir<%_tmp6> +; CHECK-NEXT: EMIT-SCALAR ir<%_tmp7> = getelementptr [2 x ptr] ir<@b>, ir<0>, ir<%_tmp6> ; CHECK-NEXT: EMIT store ir<@a>, ir<%_tmp7> ; CHECK-NEXT: EMIT ir<%_tmp9> = add nsw ir<%c.1.0>, ir<1> ; CHECK-NEXT: EMIT ir<%_tmp11> = icmp sge ir<%_tmp9>, ir<2> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 2e955de703cdf..4305228fbcd14 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -8,7 +8,7 @@ define void @diamond_phi(ptr %a) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv> ; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0> ; CHECK-NEXT: Successor(s): bb2 ; CHECK-EMPTY: @@ -73,7 +73,7 @@ define void @mask_reuse(ptr %a) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv> ; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0> ; CHECK-NEXT: Successor(s): bb1 ; CHECK-EMPTY: @@ -158,7 +158,7 @@ define void @optimized_mask(ptr %a) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv> ; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0> ; CHECK-NEXT: Successor(s): bb6 ; CHECK-EMPTY: @@ -283,7 +283,7 @@ define void @switch(ptr %a) { ; CHECK-EMPTY: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv> ; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0> ; CHECK-NEXT: Successor(s): bb2 ; CHECK-EMPTY: @@ -417,7 +417,7 @@ define void @diamond_phi2(ptr %a, i1 %c1, i1 %c2) { ; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> ; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]> ; CHECK-NEXT: BLEND ir<%phi> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP6]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv> ; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP7]]> ; CHECK-NEXT: Successor(s): bb5 ; CHECK-EMPTY: @@ -518,7 +518,7 @@ define void @blend_masks(ptr noalias %p, i1 %c0, i1 %c1, i1 %c2, i1 %c3, i1 %c4) ; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP9]]>, ir<%c4> ; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = or vp<[[VP15]]>, vp<[[VP14]]> ; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP15]]> ir<0>/vp<[[VP14]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv> ; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP16]]> ; CHECK-NEXT: Successor(s): bb8 ; CHECK-EMPTY: @@ -604,7 +604,7 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) { ; CHECK-NEXT: bb3: ; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> ; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP7]]> ir<0>/vp<[[VP8]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv> ; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep> ; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> ; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll index 464bc967bc65d..7a457e9808363 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll @@ -27,7 +27,7 @@ define i32 @live_out(ptr noalias %p, i32 %n) { ; CHECK-NEXT: Successor(s): vector.body.split, vector.latch ; CHECK-EMPTY: ; CHECK-NEXT: vector.body.split: -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv> ; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep> ; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1> ; CHECK-NEXT: EMIT store ir<%y>, ir<%gep> @@ -118,7 +118,7 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) { ; CHECK-NEXT: Successor(s): if, latch ; CHECK-EMPTY: ; CHECK-NEXT: if: -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv> ; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep> ; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1> ; CHECK-NEXT: EMIT store ir<%y>, ir<%gep> @@ -284,7 +284,7 @@ define i32 @reduction(ptr noalias %p, i32 %n) { ; CHECK-NEXT: Successor(s): vector.body.split, vector.latch ; CHECK-EMPTY: ; CHECK-NEXT: vector.body.split: -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv> ; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep> ; CHECK-NEXT: EMIT ir<%rdx.next> = add ir<%rdx>, ir<%x> ; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll index 30d916283d2be..932795c9f8cbe 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll @@ -16,11 +16,11 @@ define void @cast_flags_mixed(ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: loop: ; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop ] ; CHECK-NEXT: EMIT-SCALAR ir<%zext.nneg> = zext nneg ir<3> to i64 -; CHECK-NEXT: EMIT ir<%gep.zext> = getelementptr ir<%A>, ir<%zext.nneg> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.zext> = getelementptr ptr ir<%A>, ir<%zext.nneg> ; CHECK-NEXT: EMIT-SCALAR ir<%sext.plain> = sext ir<%iv> to i64 -; CHECK-NEXT: EMIT ir<%gep.sext> = getelementptr ir<%A>, ir<%sext.plain> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.sext> = getelementptr [4 x ptr] ir<%A>, ir<%sext.plain> ; CHECK-NEXT: EMIT-SCALAR ir<%trunc.flags> = trunc nuw nsw ir<3> to i8 -; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%B>, ir<%trunc.flags> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.trunc> = getelementptr [4 x i8] ir<%B>, ir<%trunc.flags> ; CHECK-NEXT: EMIT store ir<%gep.sext>, ir<%gep.zext> ; CHECK-NEXT: EMIT store ir<0>, ir<%gep.trunc> ; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1> @@ -67,8 +67,8 @@ define void @cast_flags_single(ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: EMIT-SCALAR ir<%trunc.nuw.only> = trunc nuw ir<3> to i8 ; CHECK-NEXT: EMIT-SCALAR ir<%trunc.nsw.only> = trunc nsw ir<3> to i8 ; CHECK-NEXT: EMIT-SCALAR ir<%zext.plain> = zext ir<3> to i64 -; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr ir<%A>, ir<%iv> -; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr ir<%B>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.a> = getelementptr i8 ir<%A>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.b> = getelementptr i64 ir<%B>, ir<%iv> ; CHECK-NEXT: EMIT store ir<%trunc.nuw.only>, ir<%gep.a> ; CHECK-NEXT: EMIT store ir<%trunc.nsw.only>, ir<%gep.a> ; CHECK-NEXT: EMIT store ir<%zext.plain>, ir<%gep.b> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll index 6eff6d1df3523..def2b2d90888a 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll @@ -15,14 +15,14 @@ define void @foo(i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: outer.header: ; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.1> = getelementptr inbounds [8 x i64] ir<@arr2>, ir<0>, ir<%outer.iv> ; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1> ; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n> ; CHECK-NEXT: Successor(s): inner ; CHECK-EMPTY: ; CHECK-NEXT: inner: ; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ] -; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> +; CHECK-NEXT: EMIT-SCALAR ir<%gep.2> = getelementptr inbounds [8 x [8 x i64]] ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> ; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2> ; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1> ; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8> diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index d07c72c41de84..4bbfec0f330f9 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -140,7 +140,7 @@ compound=true N4 [label = "vector.body:\l" + " EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" + - " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%indvars.iv\>\l" + + " EMIT-SCALAR ir\<%arr.idx\> = getelementptr inbounds i32 ir\<%A\>, ir\<%indvars.iv\>\l" + " EMIT-SCALAR ir\<%l1\> = load ir\<%arr.idx\>\l" + " EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" + " EMIT store ir\<%res\>, ir\<%arr.idx\>\l" + @@ -307,7 +307,7 @@ compound=true N4 [label = "vector.body:\l" + " EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" + - " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%iv\>\l" + + " EMIT-SCALAR ir\<%arr.idx\> = getelementptr inbounds i32 ir\<%A\>, ir\<%iv\>\l" + " EMIT-SCALAR ir\<%l1\> = load ir\<%arr.idx\>\l" + " EMIT ir\<%c\> = icmp eq ir\<%l1\>, ir\<0\>\l" + "Successor(s): loop.latch\l"