[VPlan] Introduce refinement of vputils::isSingleScalar (NFCI)#196181
Open
artagnon wants to merge 1 commit into
Open
[VPlan] Introduce refinement of vputils::isSingleScalar (NFCI)#196181artagnon wants to merge 1 commit into
artagnon wants to merge 1 commit into
Conversation
|
@llvm/pr-subscribers-vectorizers @llvm/pr-subscribers-llvm-transforms Author: Ramkumar Ramachandra (artagnon) ChangesWe introduce VPWideningInfo, a distillation of widening semantics of recipes, to communicate more fine-grained information about widening information than vputils::isSingleScalar, and demonstrate its utility in a few places. -- 8< -- Full diff: https://github.com/llvm/llvm-project/pull/196181.diff 6 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..e28c050ec2068 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3438,8 +3438,8 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// effects.
bool mayHaveSideEffects() const;
- /// Returns true if the result of this VPExpressionRecipe is a single-scalar.
- bool isSingleScalar() const;
+ /// Returns true if this VPExpressionRecipe produces a scalar.
+ bool isVectorToScalar() const;
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d04b5edcfc212..aa8aa0351ceb7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -550,7 +550,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
}
bool VPInstruction::doesGeneratePerAllLanes() const {
- return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
+ return Opcode == VPInstruction::Unpack ||
+ (Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this));
}
bool VPInstruction::canGenerateScalarForFirstLane() const {
@@ -3119,9 +3120,7 @@ bool VPExpressionRecipe::mayHaveSideEffects() const {
return false;
}
-bool VPExpressionRecipe::isSingleScalar() const {
- // Cannot use vputils::isSingleScalar(), because all external operands
- // of the expression will be live-ins while bundled.
+bool VPExpressionRecipe::isVectorToScalar() const {
auto *RR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
return RR && !RR->isPartialReduction();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bcdb91a54e305..40abce8883ce2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4897,18 +4897,15 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
for (VPBasicBlock *VPBB :
concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPScalarIVStepsRecipe, VPReplicateRecipe, VPInstruction>(&R))
+ if (!vputils::getWideningInfo(R).couldReplicatePerPart())
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
VPRegionBlock *ParentRegion = cast<VPRecipeBase>(U)->getRegion();
return !U->usesScalars(DefR) || ParentRegion != LoopRegion;
};
- if ((isa<VPReplicateRecipe>(DefR) &&
- cast<VPReplicateRecipe>(DefR)->isSingleScalar()) ||
- (isa<VPInstruction>(DefR) &&
- (vputils::onlyFirstLaneUsed(DefR) ||
- !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes())) ||
+ if ((isa<VPInstruction>(DefR) &&
+ !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes()) ||
none_of(DefR->users(), UsesVectorOrInsideReplicateRegion))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f1b9efae08377..d31059d5de1d7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -940,12 +940,9 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
SmallVector<VPRecipeBase *> ToRemove;
for (VPBasicBlock *VPBB : VPBBsToUnroll) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPInstruction, VPReplicateRecipe, VPScalarIVStepsRecipe>(&R) ||
- (isa<VPReplicateRecipe>(&R) &&
- cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||
+ if (!vputils::getWideningInfo(R).couldReplicatePerPart() ||
(isa<VPInstruction>(&R) &&
- !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&
- cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))
+ !cast<VPInstruction>(&R)->doesGeneratePerAllLanes()))
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 3327c4b188bb3..15df61b84daf6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -357,11 +357,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
- if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
+static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
+ VPWideningInfo WideOrRep) {
switch (Opcode) {
case Instruction::Freeze:
case Instruction::GetElementPtr:
@@ -369,50 +366,95 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
- case VPInstruction::PtrAdd:
- return true;
+ return WideOrRep | VPWideningInfo::Narrow;
default:
- return false;
+ if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
+ return WideOrRep | VPWideningInfo::Narrow;
+ return WideOrRep;
}
}
-bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and region-values represent single-scalar values.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
- return true;
+VPWideningInfo vputils::getWideningInfo(const VPValue *VPV) {
+ if (!VPV->hasDefiningRecipe())
+ return VPWideningInfo::Narrow;
+ switch (VPV->getDefiningRecipe()->getVPRecipeID()) {
+ default:
+ return VPWideningInfo::Wide;
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPDerivedIVSC:
+ case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPBranchOnMaskSC:
+ return VPWideningInfo::Narrow;
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return VPWideningInfo::ReplicatePart;
+ case VPRecipeBase::VPWidenCastSC:
+ // FIXME: This should be Wide | Narrow.
+ return VPWideningInfo::Wide;
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPBlendSC:
+ return VPWideningInfo::Wide | VPWideningInfo::Narrow;
+ case VPRecipeBase::VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(VPV);
+ if (VPI->isVectorToScalar())
+ return VPWideningInfo::Narrow | VPWideningInfo::Agnostic;
+ if (VPI->getOpcode() == VPInstruction::Broadcast)
+ return VPWideningInfo::Wide | VPWideningInfo::Agnostic;
+ if (VPI->isSingleScalar())
+ return VPWideningInfo::Narrow;
+ if (VPI->getOpcode() == VPInstruction::Unpack)
+ return VPWideningInfo::ReplicatePart;
+ if (VPI->getOpcode() == VPInstruction::PtrAdd)
+ return VPWideningInfo::ReplicatePart | VPWideningInfo::Narrow;
+ return getNarrowableWideningInfo(VPI->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPExpressionSC: {
+ auto *Expr = cast<VPExpressionRecipe>(VPV);
+ if (Expr->isVectorToScalar())
+ return VPWideningInfo::Narrow | VPWideningInfo::Agnostic;
+ return VPWideningInfo::Wide;
+ }
+ case VPRecipeBase::VPReductionSC: {
+ auto *Red = cast<VPReductionRecipe>(VPV);
+ return Red->isPartialReduction()
+ ? VPWideningInfo::Wide
+ : (VPWideningInfo::Narrow | VPWideningInfo::Agnostic);
+ }
+ case VPRecipeBase::VPReplicateSC: {
+ auto *Rep = cast<VPReplicateRecipe>(VPV);
+ if (Rep->isSingleScalar())
+ return VPWideningInfo::Narrow;
+ return getNarrowableWideningInfo(Rep->getOpcode(),
+ VPWideningInfo::ReplicatePart);
+ }
+ case VPRecipeBase::VPWidenSC: {
+ auto *Wide = dyn_cast<VPWidenRecipe>(VPV);
+ return getNarrowableWideningInfo(Wide->getOpcode(), VPWideningInfo::Wide);
+ }
+ }
+}
+
+VPWideningInfo vputils::getWideningInfo(const VPRecipeBase &R) {
+ return R.getNumDefinedValues() == 1 ? getWideningInfo(R.getVPSingleValue())
+ : VPWideningInfo::Wide;
+}
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
+bool vputils::isSingleScalar(const VPValue *VPV) {
+ if (auto *Rep = dyn_cast_or_null<VPReplicateRecipe>(VPV)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
// lane cannot be accessed when executing the replicate region for other
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isSingleScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ VPWideningInfo Info = getWideningInfo(VPV);
+ return Info.producesNarrowResult() || Info.isScalarToVector() ||
+ (Info.couldProduceNarrowResult() &&
+ all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar));
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -443,14 +485,11 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
+ .Case<VPWidenRecipe, VPInstruction>([](const auto *R) {
+ VPWideningInfo Info = getWideningInfo(R);
+ return (Info.couldProduceNarrowResult() || Info.isScalarToVector()) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPInstruction *VPI) {
- return preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
.Case([](const VPWidenCastRecipe *R) {
// A cast is uniform according to its operand.
return isUniformAcrossVFsAndUFs(R->getOperand(0));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index ac3a1005c8f24..de8911b4c6e1b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -21,6 +21,38 @@ class PredicatedScalarEvolution;
} // namespace llvm
namespace llvm {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a scalar value if only the Narrow bit is set,
+/// a wide value if only the Wide bit is set, and scalar values for each unroll
+/// part if only the ReplicatePart bit is set. The Narrow bit can be set on Wide
+/// and ReplicatePart recipes, which indicates that the recipe could be
+/// considered narrow if profitable. For instructions not producing values, like
+/// an assume or store, the bits talk about the inherent widening of the recipe.
+/// Finally, there is a class of instructions that necessarily take vector
+/// operands and produce a scalar result, like (Insert|Extract)Element, or
+/// necessarily take a single scalar operand and produce a vector, like
+/// Broadcast: there is no widening decision to make on this class, and it is
+/// marked with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ Narrow = 1 << 0,
+ Wide = 1 << 1,
+ ReplicatePart = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool isVectorToScalar() const { return (Info & Narrow) && (Info & Agnostic); }
+ bool isScalarToVector() const { return (Info & Wide) && (Info & Agnostic); }
+ bool producesNarrowResult() const { return !(Info & (Wide | ReplicatePart)); }
+ bool couldProduceNarrowResult() const { return Info & Narrow; }
+ bool couldProduceWideResult() const { return Info & Wide; }
+ bool couldReplicatePerPart() const { return Info & ReplicatePart; }
+};
namespace vputils {
/// Returns true if only the first lane of \p Def is used.
@@ -51,6 +83,11 @@ const SCEV *getSCEVExprForVPValue(const VPValue *V,
/// sign-extended AddRec.
bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L);
+/// Get widening information for a given \p VPV, whether it is a live-in, or
+/// whether it's recipe with an opcode.
+VPWideningInfo getWideningInfo(const VPValue *VPV);
+VPWideningInfo getWideningInfo(const VPRecipeBase &R);
+
/// Returns true if \p VPV is a single scalar, either because it produces the
/// same value for all lanes or only has its first lane used.
bool isSingleScalar(const VPValue *VPV);
|
e98b067 to
8f2e143
Compare
fhahn
reviewed
May 20, 2026
fhahn
left a comment
Contributor
There was a problem hiding this comment.
In terms of staging, it looks like in the initial version there are 3 main helpers used to simplify/unify code. Could those be introduced gradually, while having every step used + tested?
AFAICT those are:
- producesNarrowResult (value is directly guaranteed to produce a single scalar, e.g. IsSingleScalar replicate recipe)
- couldReplicatePerPart (for the current uses it looks like should answer if the recipe is replicating)
- couldProduceNarrowResult (can the recipe be narrowed to a single scalar with additional analysis)
This was referenced May 25, 2026
8f2e143 to
b2ee6fa
Compare
🪟 Windows x64 Test Results
✅ The build succeeded and all tests passed. |
b2ee6fa to
8bbb088
Compare
We introduce VPWideningInfo, a distillation of widening semantics of recipes, to communicate more fine-grained information about widening information than vputils::isSingleScalar, and demonstrate its utility in a few places.
8bbb088 to
559f8fd
Compare
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
We introduce VPWideningInfo, a distillation of widening semantics of recipes, to communicate more fine-grained information about widening information than vputils::isSingleScalar, and demonstrate its utility in vputils.