diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 3bf3f599c9828..56e6c746db74d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7096,8 +7096,9 @@ DenseMap LoopVectorizationPlanner::executePlan( // 1. Set up the skeleton for vectorization, including vector pre-header and // middle block. The vector loop is created during VPlan execution. State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton(); - replaceVPBBWithIRVPBB(BestVPlan.getScalarPreheader(), - State.CFG.PrevBB->getSingleSuccessor(), &BestVPlan); + if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader()) + replaceVPBBWithIRVPBB(ScalarPH, State.CFG.PrevBB->getSingleSuccessor(), + &BestVPlan); VPlanTransforms::removeDeadRecipes(BestVPlan); assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid"); @@ -7923,6 +7924,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan)) return nullptr; + VPlanTransforms::removeBranchOnConst(*Plan); + // Create partial reduction recipes for scaled reductions and transform // recipes to abstract recipes if it is legal and beneficial and clamp the // range for better cost estimation. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 8048c3227a9a5..8d4654ade6453 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -945,7 +945,7 @@ void VPlan::execute(VPTransformState *State) { BasicBlock *ScalarPh = State->CFG.ExitBB; VPBasicBlock *ScalarPhVPBB = getScalarPreheader(); - if (ScalarPhVPBB->hasPredecessors()) { + if (ScalarPhVPBB) { // Disconnect scalar preheader and scalar header, as the dominator tree edge // will be updated as part of VPlan execution. This allows keeping the DTU // logic generic during VPlan execution. @@ -993,7 +993,7 @@ void VPlan::execute(VPTransformState *State) { } // If the original loop is unreachable, delete it and all its blocks. - if (!ScalarPhVPBB->hasPredecessors()) { + if (!ScalarPhVPBB) { // DeleteDeadBlocks will remove single-entry phis. Remove them from the exit // VPIRBBs in VPlan as well, otherwise we would retain references to deleted // IR instructions. @@ -1007,7 +1007,7 @@ void VPlan::execute(VPTransformState *State) { Loop *OrigLoop = State->LI->getLoopFor(getScalarHeader()->getIRBasicBlock()); auto Blocks = OrigLoop->getBlocksVector(); - Blocks.push_back(cast(ScalarPhVPBB)->getIRBasicBlock()); + Blocks.push_back(ScalarPh); while (!OrigLoop->isInnermost()) State->LI->erase(*OrigLoop->begin()); State->LI->erase(OrigLoop); @@ -1710,7 +1710,8 @@ void LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo( // Update the metadata of the scalar loop. Skip the update when vectorizing // the epilogue loop to ensure it is updated only once. Also skip the update // when the scalar loop became unreachable. - if (Plan.getScalarPreheader()->hasPredecessors() && !VectorizingEpilogue) { + auto *ScalarPH = Plan.getScalarPreheader(); + if (ScalarPH && !VectorizingEpilogue) { std::optional RemainderLoopID = makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll, LLVMLoopVectorizeFollowupEpilogue}); @@ -1772,7 +1773,7 @@ void LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo( AverageVectorTripCount = SE.getSmallConstantTripCount(VectorLoop); if (ProfcheckDisableMetadataFixes || !AverageVectorTripCount) return; - if (Plan.getScalarPreheader()->hasPredecessors()) + if (ScalarPH) RemainderAverageTripCount = SE.getSmallConstantTripCount(OrigLoop) % EstimatedVFxUF; // Setting to 1 should be sufficient to generate the correct branch weights. @@ -1788,7 +1789,7 @@ void LoopVectorizationPlanner::updateLoopMetadataAndProfileInfo( OrigLoopInvocationWeight); } - if (Plan.getScalarPreheader()->hasPredecessors()) { + if (ScalarPH) { setLoopEstimatedTripCount(OrigLoop, RemainderAverageTripCount, OrigLoopInvocationWeight); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b8994b77da5fe..64aa21deb9d18 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4700,7 +4700,8 @@ class VPlan { /// Return the VPBasicBlock for the preheader of the scalar loop. VPBasicBlock *getScalarPreheader() const { - return cast(getScalarHeader()->getSinglePredecessor()); + return dyn_cast_or_null( + getScalarHeader()->getSinglePredecessor()); } /// Return the VPIRBasicBlock wrapping the header of the scalar loop. @@ -4963,8 +4964,9 @@ class VPlan { /// if the middle block is a predecessor of the scalar preheader. Note that /// this relies on unneeded branches to the scalar tail loop being removed. bool hasScalarTail() const { - return is_contained(getScalarPreheader()->getPredecessors(), - getMiddleBlock()); + auto *ScalarPH = getScalarPreheader(); + return ScalarPH && + is_contained(ScalarPH->getPredecessors(), getMiddleBlock()); } }; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 52f299dd5fdca..badc7fd1ea84b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2870,8 +2870,11 @@ void VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) { if (OnlyLatches) VPDT.emplace(Plan); - for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( - vp_depth_first_shallow(Plan.getEntry()))) { + // Collect all blocks before modifying the CFG so we can identify unreachable + // ones after constant branch removal. + SmallVector AllBlocks(vp_depth_first_shallow(Plan.getEntry())); + + for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(AllBlocks)) { VPValue *Cond; // Skip blocks that are not terminated by BranchOnCond. if (VPBB->empty() || !match(&VPBB->back(), m_BranchOnCond(m_VPValue(Cond)))) @@ -2894,16 +2897,42 @@ void VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) { cast(VPBB->getSuccessors()[RemovedIdx]); assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 && "There must be a single edge between VPBB and its successor"); - // Values coming from VPBB into phi recipes of RemoveSucc are removed from + // Values coming from VPBB into phi recipes of RemovedSucc are removed from // these recipes. for (VPRecipeBase &R : RemovedSucc->phis()) cast(&R)->removeIncomingValueFor(VPBB); - // Disconnect blocks and remove the terminator. RemovedSucc will be deleted - // automatically on VPlan destruction if it becomes unreachable. + // Disconnect blocks and remove the terminator. VPBlockUtils::disconnectBlocks(VPBB, RemovedSucc); VPBB->back().eraseFromParent(); } + + // Compute which blocks are still reachable from the entry after constant + // branch removal. + SmallPtrSet Reachable( + llvm::from_range, vp_depth_first_shallow(Plan.getEntry())); + + // Detach all unreachable blocks from their successors, removing their recipes + // and incoming values from phi recipes. + VPSymbolicValue Tmp; + for (VPBlockBase *B : AllBlocks) { + if (Reachable.contains(B)) + continue; + for (VPBlockBase *Succ : to_vector(B->successors())) { + if (auto *SuccBB = dyn_cast(Succ)) + for (VPRecipeBase &R : SuccBB->phis()) + cast(&R)->removeIncomingValueFor(B); + VPBlockUtils::disconnectBlocks(B, Succ); + } + for (VPBasicBlock *DeadBB : + VPBlockUtils::blocksOnly(vp_depth_first_deep(B))) { + for (VPRecipeBase &R : make_early_inc_range(*DeadBB)) { + for (VPValue *Def : R.definedValues()) + Def->replaceAllUsesWith(&Tmp); + R.eraseFromParent(); + } + } + } } void VPlanTransforms::optimize(VPlan &Plan) { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll index d96a96836bd3a..eaa552ab5c616 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll @@ -127,10 +127,9 @@ define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8 ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: br label %[[EXIT:.*]] ; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] ; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[SRC]], align 1 ; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[L]] to i64 ; CHECK-NEXT: [[ADD:%.*]] = or i64 [[L_EXT]], 1 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll index fabab210fb850..07205354ac300 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll @@ -48,10 +48,9 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) { ; CHECK: middle.block: ; CHECK-NEXT: br label [[EXIT:%.*]] ; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] ; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]] ; CHECK: loop.then: ; CHECK-NEXT: br label [[LOOP_LATCH]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll index 8232aa3eb76df..2ca02df345060 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll @@ -73,10 +73,9 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV32: middle.block: ; RV32-NEXT: br label [[FOR_END:%.*]] ; RV32: scalar.ph: -; RV32-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_MEMCHECK1]] ] ; RV32-NEXT: br label [[FOR_BODY:%.*]] ; RV32: for.body: -; RV32-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ] +; RV32-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; RV32-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDVARS_IV]] ; RV32-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; RV32-NEXT: [[CMP1:%.*]] = icmp slt i32 [[TMP21]], 100 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index 0842175d5c39a..12302fefb4288 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -206,8 +206,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64: [[MIDDLE_BLOCK]]: ; RV64-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]] ; RV64: [[SCALAR_PH]]: -; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ] -; RV64-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] ; RV64-NEXT: br label %[[FOR_BODY:.*]] ; RV64: [[FOR_COND_CLEANUP_LOOPEXIT]]: ; RV64-NEXT: br label %[[FOR_COND_CLEANUP]] @@ -431,8 +429,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64: [[MIDDLE_BLOCK]]: ; RV64-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]] ; RV64: [[SCALAR_PH]]: -; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ] -; RV64-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] ; RV64-NEXT: br label %[[FOR_BODY:.*]] ; RV64: [[FOR_COND_CLEANUP_LOOPEXIT]]: ; RV64-NEXT: br label %[[FOR_COND_CLEANUP]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll index 3e7bfe7a860e7..a31bb408dcbc7 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll @@ -111,10 +111,9 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) { ; NO-UNIT-STRIDE-MV: [[MIDDLE_BLOCK]]: ; NO-UNIT-STRIDE-MV-NEXT: br label %[[EXIT:.*]] ; NO-UNIT-STRIDE-MV: [[SCALAR_PH]]: -; NO-UNIT-STRIDE-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, %[[VECTOR_SCEVCHECK]] ], [ 1, %[[VECTOR_MEMCHECK]] ] ; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER1:.*]] ; NO-UNIT-STRIDE-MV: [[HEADER1]]: -; NO-UNIT-STRIDE-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ] +; NO-UNIT-STRIDE-MV-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ] ; NO-UNIT-STRIDE-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1 ; NO-UNIT-STRIDE-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]] ; NO-UNIT-STRIDE-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll index 4af43e5fb0851..567f54cd3bf8f 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll @@ -636,10 +636,9 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED: middle.block: ; NOSTRIDED-NEXT: br label [[EXIT:%.*]] ; NOSTRIDED: scalar.ph: -; NOSTRIDED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ] ; NOSTRIDED-NEXT: br label [[LOOP:%.*]] ; NOSTRIDED: loop: -; NOSTRIDED-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] +; NOSTRIDED-NEXT: [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; NOSTRIDED-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]] ; NOSTRIDED-NEXT: [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]] ; NOSTRIDED-NEXT: [[X0:%.*]] = load i32, ptr [[Q0]], align 4 @@ -788,10 +787,9 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; STRIDED: middle.block: ; STRIDED-NEXT: br label [[EXIT:%.*]] ; STRIDED: scalar.ph: -; STRIDED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK1]] ] ; STRIDED-NEXT: br label [[LOOP:%.*]] ; STRIDED: loop: -; STRIDED-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] +; STRIDED-NEXT: [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; STRIDED-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]] ; STRIDED-NEXT: [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]] ; STRIDED-NEXT: [[X0:%.*]] = load i32, ptr [[Q0]], align 4 diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll index dc0570110f606..e29e0e52580d7 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll @@ -55,8 +55,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<%n>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ ir<%n>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll index cc5b21c19db29..3a6bbd2c6a757 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll @@ -120,13 +120,11 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) { ; CHECK-TF-NEXT: No successors ; CHECK-TF-EMPTY: ; CHECK-TF-NEXT: scalar.ph: -; CHECK-TF-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] -; CHECK-TF-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ ir<-1>, ir-bb ] ; CHECK-TF-NEXT: Successor(s): ir-bb ; CHECK-TF-EMPTY: ; CHECK-TF-NEXT: ir-bb: -; CHECK-TF-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; CHECK-TF-NEXT: IR %data.phi = phi i32 [ -1, %entry ], [ %select.data, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph) +; CHECK-TF-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-TF-NEXT: IR %data.phi = phi i32 [ -1, %entry ], [ %select.data, %loop ] (extra operand: ir<-1> from scalar.ph) ; CHECK-TF-NEXT: IR %ld.addr = getelementptr inbounds i32, ptr %data, i64 %iv ; CHECK-TF-NEXT: IR %ld = load i32, ptr %ld.addr, align 4 ; CHECK-TF-NEXT: IR %select.cmp = icmp slt i32 %a, %ld @@ -287,13 +285,11 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val ; CHECK-TF-NEXT: No successors ; CHECK-TF-EMPTY: ; CHECK-TF-NEXT: scalar.ph: -; CHECK-TF-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] -; CHECK-TF-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ ir<%default_val>, ir-bb ] ; CHECK-TF-NEXT: Successor(s): ir-bb ; CHECK-TF-EMPTY: ; CHECK-TF-NEXT: ir-bb: -; CHECK-TF-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; CHECK-TF-NEXT: IR %data.phi = phi i32 [ %default_val, %entry ], [ %select.data, %latch ] (extra operand: vp<%bc.merge.rdx> from scalar.ph) +; CHECK-TF-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: ir<0> from scalar.ph) +; CHECK-TF-NEXT: IR %data.phi = phi i32 [ %default_val, %entry ], [ %select.data, %latch ] (extra operand: ir<%default_val> from scalar.ph) ; CHECK-TF-NEXT: IR %a.addr = getelementptr inbounds nuw i32, ptr %a, i64 %iv ; CHECK-TF-NEXT: IR %ld.a = load i32, ptr %a.addr, align 4 ; CHECK-TF-NEXT: IR %if.cond = icmp sgt i32 %ld.a, %threshold diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll index 5616344699878..188ce1e33b810 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll @@ -82,13 +82,11 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %rem = srem i32 %0, %x ; CHECK-NEXT: IR %gep = getelementptr i8, ptr %ptr, i32 %iv ; CHECK-NEXT: IR %lv = load i8, ptr %gep, align 1 @@ -181,13 +179,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %latch ] (extra operand: vp<%scalar.recur.init> from scalar.ph) -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %latch ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %recur.next = sext i8 %y to i32 ; CHECK-NEXT: IR %cond = icmp eq i32 %iv, %z ; CHECK-NEXT: No successors @@ -279,15 +275,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ ir<1234>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph) +; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] (extra operand: ir<1234> from scalar.ph) ; CHECK-NEXT: IR %rem = srem i32 %recur, %x ; CHECK-NEXT: IR %recur.next = sext i8 %y to i32 ; CHECK-NEXT: IR %add = add i32 %rem, %recur.next @@ -416,13 +409,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %0 = phi i32 [ 0, %entry ], [ %conv, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep = getelementptr i8, ptr %ptr, i32 %iv ; CHECK-NEXT: IR %rem = srem i32 %0, %x ; CHECK-NEXT: IR %lv = load i8, ptr %gep, align 1 @@ -523,13 +514,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %recur = phi i32 [ 0, %entry ], [ %recur.next, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %rem = srem i32 %recur, %x ; CHECK-NEXT: IR %rem.div = sdiv i32 20, %rem ; CHECK-NEXT: IR %recur.next = sext i8 %y to i32 @@ -620,13 +609,11 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<2>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ] (extra operand: ir<2> from scalar.ph) +; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %val = sdiv i32 %.pn, %x ; CHECK-NEXT: IR %l = load i32, ptr %src, align 4 ; CHECK-NEXT: IR %gep.dst = getelementptr i32, ptr %dst, i64 %iv diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll index f325c5ac15dec..48300e8141f70 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll @@ -147,11 +147,10 @@ define void @test(ptr %ptr) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %cond0 = icmp ult i64 %iv, 13 ; CHECK-NEXT: IR %s = select i1 %cond0, i32 10, i32 20 ; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %ptr, i64 %iv diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll index 767c10f901a6a..c2e9eae8c4bbb 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll @@ -84,11 +84,10 @@ define void @sink1(i32 %k, i32 %x) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %cond = icmp eq i32 %iv, %x ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -195,11 +194,10 @@ define void @sink2(i32 %k) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv ; CHECK-NEXT: IR %lv.b = load i32, ptr %gep.b, align 4 ; CHECK-NEXT: IR %add = add i32 %lv.b, 10 @@ -308,11 +306,10 @@ define void @sink3(i32 %k) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv ; CHECK-NEXT: IR %lv.b = load i32, ptr %gep.b, align 4 ; CHECK-NEXT: IR %add = add i32 %lv.b, 10 @@ -405,11 +402,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<21>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i64 [ 21, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i64 [ 21, %entry ], [ %iv.next, %loop.latch ] (extra operand: ir<21> from scalar.ph) ; CHECK-NEXT: IR %gep.A.uniform = getelementptr inbounds i16, ptr %A, i64 0 ; CHECK-NEXT: IR %gep.B = getelementptr inbounds i16, ptr %B, i64 %iv ; CHECK-NEXT: IR %lv = load i16, ptr %gep.A.uniform, align 1 @@ -517,11 +513,10 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.0 ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.0 ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv ; CHECK-NEXT: IR %c.1 = icmp ult i32 %iv, %j ; CHECK-NEXT: IR %mul = mul i32 %iv, 10 @@ -637,11 +632,10 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv ; CHECK-NEXT: IR %mul = mul i32 %iv, 10 ; CHECK-NEXT: IR %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul @@ -766,11 +760,10 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv ; CHECK-NEXT: IR %mul = mul i32 %iv, 10 ; CHECK-NEXT: IR %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul @@ -896,11 +889,10 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv ; CHECK-NEXT: IR %lv.a = load i32, ptr %gep.a, align 4 ; CHECK-NEXT: IR %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv @@ -1000,11 +992,10 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv ; CHECK-NEXT: IR %lv.a = load i32, ptr %gep.a, align 4 ; CHECK-NEXT: IR %div = sdiv i32 %lv.a, %lv.a @@ -1109,13 +1100,11 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ ir<0>, ir-bb ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; CHECK-NEXT: IR %for = phi i32 [ 0, %entry ], [ %lv.a, %loop ] (extra operand: vp<%scalar.recur.init> from scalar.ph) +; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph) +; CHECK-NEXT: IR %for = phi i32 [ 0, %entry ], [ %lv.a, %loop ] (extra operand: ir<0> from scalar.ph) ; CHECK-NEXT: IR %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv ; CHECK-NEXT: IR %lv.a = load i32, ptr %gep.a, align 4 ; CHECK-NEXT: IR %div = sdiv i32 %for, %lv.a