1//===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements IR expansion for vector predication intrinsics, allowing
10// targets to enable vector predication until just before codegen.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/ExpandVectorPredication.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/TargetTransformInfo.h"
17#include "llvm/Analysis/ValueTracking.h"
18#include "llvm/Analysis/VectorUtils.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Compiler.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Transforms/Utils/LoopUtils.h"
29#include <optional>
30
31using namespace llvm;
32
33using VPLegalization = TargetTransformInfo::VPLegalization;
34using VPTransform = TargetTransformInfo::VPLegalization::VPTransform;
35
36// Keep this in sync with TargetTransformInfo::VPLegalization.
37#define VPINTERNAL_VPLEGAL_CASES \
38 VPINTERNAL_CASE(Legal) \
39 VPINTERNAL_CASE(Discard) \
40 VPINTERNAL_CASE(Convert)
41
42#define VPINTERNAL_CASE(X) "|" #X
43
44// Override options.
45static cl::opt<std::string> EVLTransformOverride(
46 "expandvp-override-evl-transform", cl::init(Val: ""), cl::Hidden,
47 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
48 ". If non-empty, ignore "
49 "TargetTransformInfo and "
50 "always use this transformation for the %evl parameter (Used in "
51 "testing)."));
52
53static cl::opt<std::string> MaskTransformOverride(
54 "expandvp-override-mask-transform", cl::init(Val: ""), cl::Hidden,
55 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
56 ". If non-empty, Ignore "
57 "TargetTransformInfo and "
58 "always use this transformation for the %mask parameter (Used in "
59 "testing)."));
60
61#undef VPINTERNAL_CASE
62#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
63
64static VPTransform parseOverrideOption(const std::string &TextOpt) {
65 return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES;
66}
67
68#undef VPINTERNAL_VPLEGAL_CASES
69
70// Whether any override options are set.
71static bool anyExpandVPOverridesSet() {
72 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
73}
74
75#define DEBUG_TYPE "expandvp"
76
77STATISTIC(NumFoldedVL, "Number of folded vector length params");
78STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
79
80///// Helpers {
81
82/// \returns Whether the vector mask \p MaskVal has all lane bits set.
83static bool isAllTrueMask(Value *MaskVal) {
84 if (Value *SplattedVal = getSplatValue(V: MaskVal))
85 if (auto *ConstValue = dyn_cast<Constant>(Val: SplattedVal))
86 return ConstValue->isAllOnesValue();
87
88 return false;
89}
90
91/// \returns A non-excepting divisor constant for this type.
92static Constant *getSafeDivisor(Type *DivTy) {
93 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
94 return ConstantInt::get(Ty: DivTy, V: 1u, IsSigned: false);
95}
96
97/// Transfer operation properties from \p OldVPI to \p NewVal.
98static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
99 auto *NewInst = dyn_cast<Instruction>(Val: &NewVal);
100 if (!NewInst || !isa<FPMathOperator>(Val: NewVal))
101 return;
102
103 auto *OldFMOp = dyn_cast<FPMathOperator>(Val: &VPI);
104 if (!OldFMOp)
105 return;
106
107 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
108}
109
110/// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
111/// OldVP gets erased.
112static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
113 transferDecorations(NewVal&: NewOp, VPI&: OldOp);
114
115 if (isa<Instruction>(Val: NewOp) && !NewOp.hasName() && OldOp.hasName())
116 NewOp.takeName(V: &OldOp);
117
118 OldOp.replaceAllUsesWith(V: &NewOp);
119 OldOp.eraseFromParent();
120}
121
122static bool maySpeculateLanes(VPIntrinsic &VPI) {
123 // The result of VP reductions depends on the mask and evl.
124 if (isa<VPReductionIntrinsic>(Val: VPI))
125 return false;
126 // Fallback to whether the intrinsic is speculatable.
127 if (auto IntrID = VPI.getFunctionalIntrinsicID())
128 return Intrinsic::getFnAttributes(C&: VPI.getContext(), id: *IntrID)
129 .hasAttribute(Kind: Attribute::AttrKind::Speculatable);
130 if (auto Opc = VPI.getFunctionalOpcode())
131 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: *Opc, Inst: &VPI);
132 return false;
133}
134
135//// } Helpers
136
137namespace {
138
139// Expansion pass state at function scope.
140struct CachingVPExpander {
141 const TargetTransformInfo &TTI;
142
143 /// \returns A bitmask that is true where the lane position is less-than \p
144 /// EVLParam
145 ///
146 /// \p Builder
147 /// Used for instruction creation.
148 /// \p VLParam
149 /// The explicit vector length parameter to test against the lane
150 /// positions.
151 /// \p ElemCount
152 /// Static (potentially scalable) number of vector elements.
153 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
154 ElementCount ElemCount);
155
156 /// If needed, folds the EVL in the mask operand and discards the EVL
157 /// parameter. Returns true if the mask was actually folded.
158 bool foldEVLIntoMask(VPIntrinsic &VPI);
159
160 /// "Remove" the %evl parameter of \p PI by setting it to the static vector
161 /// length of the operation. Returns true if the %evl (if any) was effectively
162 /// changed.
163 bool discardEVLParameter(VPIntrinsic &PI);
164
165 /// Lower this VP binary operator to a unpredicated binary operator.
166 bool expandPredicationInBinaryOperator(IRBuilder<> &Builder, VPIntrinsic &PI);
167
168 /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
169 bool expandPredicationInReduction(IRBuilder<> &Builder,
170 VPReductionIntrinsic &PI);
171
172 /// Lower this VP memory operation to a non-VP intrinsic.
173 bool expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
174 VPIntrinsic &VPI);
175
176 /// Query TTI and expand the vector predication in \p P accordingly.
177 bool expandPredication(VPIntrinsic &PI);
178
179 /// Determine how and whether the VPIntrinsic \p VPI shall be expanded. This
180 /// overrides TTI with the cl::opts listed at the top of this file.
181 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
182 bool UsingTTIOverrides;
183
184public:
185 CachingVPExpander(const TargetTransformInfo &TTI)
186 : TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
187
188 /// Expand llvm.vp.* intrinsics as requested by \p TTI.
189 /// Returns the details of the expansion.
190 VPExpansionDetails expandVectorPredication(VPIntrinsic &VPI);
191};
192
193//// CachingVPExpander {
194
195Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
196 Value *EVLParam,
197 ElementCount ElemCount) {
198 // TODO add caching
199 // Scalable vector %evl conversion.
200 if (ElemCount.isScalable()) {
201 Type *BoolVecTy = VectorType::get(ElementType: Builder.getInt1Ty(), EC: ElemCount);
202 // `get_active_lane_mask` performs an implicit less-than comparison.
203 Value *ConstZero = Builder.getInt32(C: 0);
204 return Builder.CreateIntrinsic(ID: Intrinsic::get_active_lane_mask,
205 OverloadTypes: {BoolVecTy, EVLParam->getType()},
206 Args: {ConstZero, EVLParam});
207 }
208
209 // Fixed vector %evl conversion.
210 Type *LaneTy = EVLParam->getType();
211 unsigned NumElems = ElemCount.getFixedValue();
212 Value *VLSplat = Builder.CreateVectorSplat(NumElts: NumElems, V: EVLParam);
213 Value *IdxVec = Builder.CreateStepVector(DstType: VectorType::get(ElementType: LaneTy, EC: ElemCount));
214 return Builder.CreateICmp(P: CmpInst::ICMP_ULT, LHS: IdxVec, RHS: VLSplat);
215}
216
217bool CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
218 VPIntrinsic &VPI) {
219 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
220 "Implicitly dropping %evl in non-speculatable operator!");
221
222 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
223 assert(Instruction::isBinaryOp(OC));
224
225 Value *Op0 = VPI.getOperand(i_nocapture: 0);
226 Value *Op1 = VPI.getOperand(i_nocapture: 1);
227 Value *Mask = VPI.getMaskParam();
228
229 // Blend in safe operands.
230 if (Mask && !isAllTrueMask(MaskVal: Mask)) {
231 switch (OC) {
232 default:
233 // Can safely ignore the predicate.
234 break;
235
236 // Division operators need a safe divisor on masked-off lanes (1).
237 case Instruction::UDiv:
238 case Instruction::SDiv:
239 case Instruction::URem:
240 case Instruction::SRem:
241 // 2nd operand must not be zero. The mask needs to be frozen as it may
242 // have poison lanes, and a poison divisor would introduce UB even if Op1
243 // didn't cause UB to begin with.
244 Value *SafeDivisor = getSafeDivisor(DivTy: VPI.getType());
245 Op1 = Builder.CreateSelect(C: Builder.CreateFreeze(V: Mask), True: Op1, False: SafeDivisor);
246 }
247 }
248
249 Value *NewBinOp = Builder.CreateBinOp(Opc: OC, LHS: Op0, RHS: Op1);
250
251 replaceOperation(NewOp&: *NewBinOp, OldOp&: VPI);
252 return true;
253}
254
255static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
256 Type *EltTy) {
257 Intrinsic::ID RdxID = *VPI.getFunctionalIntrinsicID();
258 return getReductionIdentity(RdxID, Ty: EltTy, FMF: VPI.getFastMathFlagsOrNone());
259}
260
261bool CachingVPExpander::expandPredicationInReduction(
262 IRBuilder<> &Builder, VPReductionIntrinsic &VPI) {
263 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
264 "Implicitly dropping %evl in non-speculatable operator!");
265
266 Value *Mask = VPI.getMaskParam();
267 Value *RedOp = VPI.getOperand(i_nocapture: VPI.getVectorParamPos());
268
269 // Insert neutral element in masked-out positions
270 if (Mask && !isAllTrueMask(MaskVal: Mask)) {
271 auto *NeutralElt = getNeutralReductionElement(VPI, EltTy: VPI.getType());
272 auto *NeutralVector = Builder.CreateVectorSplat(
273 EC: cast<VectorType>(Val: RedOp->getType())->getElementCount(), V: NeutralElt);
274 RedOp = Builder.CreateSelect(C: Mask, True: RedOp, False: NeutralVector);
275 }
276
277 Value *Reduction;
278 Value *Start = VPI.getOperand(i_nocapture: VPI.getStartParamPos());
279
280 switch (VPI.getIntrinsicID()) {
281 default:
282 llvm_unreachable("Impossible reduction kind");
283 case Intrinsic::vp_reduce_add:
284 case Intrinsic::vp_reduce_mul:
285 case Intrinsic::vp_reduce_and:
286 case Intrinsic::vp_reduce_or:
287 case Intrinsic::vp_reduce_xor: {
288 Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID();
289 unsigned Opc = getArithmeticReductionInstruction(RdxID: RedID);
290 assert(Instruction::isBinaryOp(Opc));
291 Reduction = Builder.CreateUnaryIntrinsic(ID: RedID, Op: RedOp);
292 Reduction =
293 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Reduction, RHS: Start);
294 break;
295 }
296 case Intrinsic::vp_reduce_smax:
297 case Intrinsic::vp_reduce_smin:
298 case Intrinsic::vp_reduce_umax:
299 case Intrinsic::vp_reduce_umin:
300 case Intrinsic::vp_reduce_fmax:
301 case Intrinsic::vp_reduce_fmin:
302 case Intrinsic::vp_reduce_fmaximum:
303 case Intrinsic::vp_reduce_fminimum: {
304 Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID();
305 Intrinsic::ID ScalarID = getMinMaxReductionIntrinsicOp(RdxID: RedID);
306 Reduction = Builder.CreateUnaryIntrinsic(ID: RedID, Op: RedOp);
307 transferDecorations(NewVal&: *Reduction, VPI);
308 Reduction = Builder.CreateBinaryIntrinsic(ID: ScalarID, LHS: Reduction, RHS: Start);
309 break;
310 }
311 case Intrinsic::vp_reduce_fadd:
312 Reduction = Builder.CreateFAddReduce(Acc: Start, Src: RedOp);
313 break;
314 case Intrinsic::vp_reduce_fmul:
315 Reduction = Builder.CreateFMulReduce(Acc: Start, Src: RedOp);
316 break;
317 }
318
319 replaceOperation(NewOp&: *Reduction, OldOp&: VPI);
320 return true;
321}
322
323bool CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
324 VPIntrinsic &VPI) {
325 assert(VPI.canIgnoreVectorLengthParam());
326
327 const auto &DL = VPI.getDataLayout();
328
329 Value *MaskParam = VPI.getMaskParam();
330 Value *PtrParam = VPI.getMemoryPointerParam();
331 Value *DataParam = VPI.getMemoryDataParam();
332 bool IsUnmasked = isAllTrueMask(MaskVal: MaskParam);
333
334 MaybeAlign AlignOpt = VPI.getPointerAlignment();
335
336 Value *NewMemoryInst = nullptr;
337 switch (VPI.getIntrinsicID()) {
338 default:
339 llvm_unreachable("Not a VP memory intrinsic");
340 case Intrinsic::vp_store:
341 if (IsUnmasked) {
342 StoreInst *NewStore =
343 Builder.CreateStore(Val: DataParam, Ptr: PtrParam, /*IsVolatile*/ isVolatile: false);
344 if (AlignOpt.has_value())
345 NewStore->setAlignment(*AlignOpt);
346 NewMemoryInst = NewStore;
347 } else
348 NewMemoryInst = Builder.CreateMaskedStore(
349 Val: DataParam, Ptr: PtrParam, Alignment: AlignOpt.valueOrOne(), Mask: MaskParam);
350
351 break;
352 case Intrinsic::vp_load:
353 if (IsUnmasked) {
354 LoadInst *NewLoad =
355 Builder.CreateLoad(Ty: VPI.getType(), Ptr: PtrParam, /*IsVolatile*/ isVolatile: false);
356 if (AlignOpt.has_value())
357 NewLoad->setAlignment(*AlignOpt);
358 NewMemoryInst = NewLoad;
359 } else
360 NewMemoryInst = Builder.CreateMaskedLoad(
361 Ty: VPI.getType(), Ptr: PtrParam, Alignment: AlignOpt.valueOrOne(), Mask: MaskParam);
362
363 break;
364 case Intrinsic::vp_scatter: {
365 auto *ElementType =
366 cast<VectorType>(Val: DataParam->getType())->getElementType();
367 NewMemoryInst = Builder.CreateMaskedScatter(
368 Val: DataParam, Ptrs: PtrParam,
369 Alignment: AlignOpt.value_or(u: DL.getPrefTypeAlign(Ty: ElementType)), Mask: MaskParam);
370 break;
371 }
372 case Intrinsic::vp_gather: {
373 auto *ElementType = cast<VectorType>(Val: VPI.getType())->getElementType();
374 NewMemoryInst = Builder.CreateMaskedGather(
375 Ty: VPI.getType(), Ptrs: PtrParam,
376 Alignment: AlignOpt.value_or(u: DL.getPrefTypeAlign(Ty: ElementType)), Mask: MaskParam,
377 PassThru: nullptr);
378 break;
379 }
380 }
381
382 assert(NewMemoryInst);
383 replaceOperation(NewOp&: *NewMemoryInst, OldOp&: VPI);
384 return true;
385}
386
387bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
388 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
389
390 if (VPI.canIgnoreVectorLengthParam())
391 return false;
392
393 Value *EVLParam = VPI.getVectorLengthParam();
394 if (!EVLParam)
395 return false;
396
397 ElementCount StaticElemCount = VPI.getStaticVectorLength();
398 Value *MaxEVL = nullptr;
399 Type *Int32Ty = Type::getInt32Ty(C&: VPI.getContext());
400 if (StaticElemCount.isScalable()) {
401 // TODO add caching
402 IRBuilder<> Builder(VPI.getIterator());
403 Value *FactorConst = Builder.getInt32(C: StaticElemCount.getKnownMinValue());
404 Value *VScale = Builder.CreateVScale(Ty: Int32Ty, Name: "vscale");
405 MaxEVL = Builder.CreateNUWMul(LHS: VScale, RHS: FactorConst, Name: "scalable_size");
406 } else {
407 MaxEVL = ConstantInt::get(Ty: Int32Ty, V: StaticElemCount.getFixedValue(), IsSigned: false);
408 }
409 VPI.setVectorLengthParam(MaxEVL);
410 return true;
411}
412
413bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
414 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
415
416 IRBuilder<> Builder(&VPI);
417
418 // Ineffective %evl parameter and so nothing to do here.
419 if (VPI.canIgnoreVectorLengthParam())
420 return false;
421
422 // Only VP intrinsics can have an %evl parameter.
423 Value *OldMaskParam = VPI.getMaskParam();
424 if (!OldMaskParam) {
425 assert((VPI.getIntrinsicID() == Intrinsic::vp_merge) &&
426 "Unexpected VP intrinsic without mask operand");
427 OldMaskParam = VPI.getArgOperand(i: 0);
428 }
429
430 Value *OldEVLParam = VPI.getVectorLengthParam();
431 assert(OldMaskParam && "no mask param to fold the vl param into");
432 assert(OldEVLParam && "no EVL param to fold away");
433
434 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
435 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
436
437 // Convert the %evl predication into vector mask predication.
438 ElementCount ElemCount = VPI.getStaticVectorLength();
439 Value *VLMask = convertEVLToMask(Builder, EVLParam: OldEVLParam, ElemCount);
440 Value *NewMaskParam = Builder.CreateLogicalAnd(Cond1: VLMask, Cond2: OldMaskParam);
441 if (VPI.getIntrinsicID() == Intrinsic::vp_merge)
442 VPI.setArgOperand(i: 0, v: NewMaskParam);
443 else
444 VPI.setMaskParam(NewMaskParam);
445
446 // Drop the %evl parameter.
447 discardEVLParameter(VPI);
448 assert(VPI.canIgnoreVectorLengthParam() &&
449 "transformation did not render the evl param ineffective!");
450
451 // Reassess the modified instruction.
452 return true;
453}
454
455bool CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
456 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
457
458 IRBuilder<> Builder(&VPI);
459
460 // Try lowering to a LLVM instruction first.
461 auto OC = VPI.getFunctionalOpcode();
462
463 if (OC && Instruction::isBinaryOp(Opcode: *OC))
464 return expandPredicationInBinaryOperator(Builder, VPI);
465
466 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(Val: &VPI))
467 return expandPredicationInReduction(Builder, VPI&: *VPRI);
468
469 switch (VPI.getIntrinsicID()) {
470 default:
471 break;
472 case Intrinsic::vp_merge: {
473 assert(maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam());
474 Value *NewSelectOp = Builder.CreateSelect(
475 C: VPI.getOperand(i_nocapture: 0), True: VPI.getOperand(i_nocapture: 1), False: VPI.getOperand(i_nocapture: 2));
476 replaceOperation(NewOp&: *NewSelectOp, OldOp&: VPI);
477 return NewSelectOp;
478 }
479 case Intrinsic::vp_load:
480 case Intrinsic::vp_store:
481 case Intrinsic::vp_gather:
482 case Intrinsic::vp_scatter:
483 return expandPredicationInMemoryIntrinsic(Builder, VPI);
484 }
485
486 return false;
487}
488
489//// } CachingVPExpander
490
491void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
492 // Operations with speculatable lanes do not strictly need predication.
493 if (maySpeculateLanes(VPI)) {
494 // Converting a speculatable VP intrinsic means dropping %mask and %evl.
495 // No need to expand %evl into the %mask only to ignore that code.
496 if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
497 LegalizeStrat.EVLParamStrategy = VPLegalization::Discard;
498 return;
499 }
500
501 // We have to preserve the predicating effect of %evl for this
502 // non-speculatable VP intrinsic.
503 // 1) Never discard %evl.
504 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
505 // %evl gets folded into %mask.
506 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
507 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
508 LegalizeStrat.EVLParamStrategy = VPLegalization::Convert;
509 }
510}
511
512VPLegalization
513CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
514 auto VPStrat = TTI.getVPLegalizationStrategy(PI: VPI);
515 if (LLVM_LIKELY(!UsingTTIOverrides)) {
516 // No overrides - we are in production.
517 return VPStrat;
518 }
519
520 // Overrides set - we are in testing, the following does not need to be
521 // efficient.
522 VPStrat.EVLParamStrategy = parseOverrideOption(TextOpt: EVLTransformOverride);
523 VPStrat.OpStrategy = parseOverrideOption(TextOpt: MaskTransformOverride);
524 return VPStrat;
525}
526
527VPExpansionDetails
528CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) {
529 auto Strategy = getVPLegalizationStrategy(VPI);
530 sanitizeStrategy(VPI, LegalizeStrat&: Strategy);
531
532 VPExpansionDetails Changed = VPExpansionDetails::IntrinsicUnchanged;
533
534 // Transform the EVL parameter.
535 switch (Strategy.EVLParamStrategy) {
536 case VPLegalization::Legal:
537 break;
538 case VPLegalization::Discard:
539 if (discardEVLParameter(VPI))
540 Changed = VPExpansionDetails::IntrinsicUpdated;
541 break;
542 case VPLegalization::Convert:
543 if (foldEVLIntoMask(VPI)) {
544 Changed = VPExpansionDetails::IntrinsicUpdated;
545 ++NumFoldedVL;
546 }
547 break;
548 }
549
550 // Replace with a non-predicated operation.
551 switch (Strategy.OpStrategy) {
552 case VPLegalization::Legal:
553 break;
554 case VPLegalization::Discard:
555 llvm_unreachable("Invalid strategy for operators.");
556 case VPLegalization::Convert:
557 if (expandPredication(VPI)) {
558 ++NumLoweredVPOps;
559 Changed = VPExpansionDetails::IntrinsicReplaced;
560 }
561 break;
562 }
563
564 return Changed;
565}
566} // namespace
567
568VPExpansionDetails
569llvm::expandVectorPredicationIntrinsic(VPIntrinsic &VPI,
570 const TargetTransformInfo &TTI) {
571 return CachingVPExpander(TTI).expandVectorPredication(VPI);
572}
573