1//===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements IR expansion for vector predication intrinsics, allowing
10// targets to enable vector predication until just before codegen.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/ExpandVectorPredication.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/TargetTransformInfo.h"
17#include "llvm/Analysis/ValueTracking.h"
18#include "llvm/Analysis/VectorUtils.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Compiler.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Transforms/Utils/LoopUtils.h"
29#include <optional>
30
31using namespace llvm;
32
33using VPLegalization = TargetTransformInfo::VPLegalization;
34using VPTransform = TargetTransformInfo::VPLegalization::VPTransform;
35
36// Keep this in sync with TargetTransformInfo::VPLegalization.
37#define VPINTERNAL_VPLEGAL_CASES \
38 VPINTERNAL_CASE(Legal) \
39 VPINTERNAL_CASE(Discard) \
40 VPINTERNAL_CASE(Convert)
41
42#define VPINTERNAL_CASE(X) "|" #X
43
44// Override options.
45static cl::opt<std::string> EVLTransformOverride(
46 "expandvp-override-evl-transform", cl::init(Val: ""), cl::Hidden,
47 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
48 ". If non-empty, ignore "
49 "TargetTransformInfo and "
50 "always use this transformation for the %evl parameter (Used in "
51 "testing)."));
52
53static cl::opt<std::string> MaskTransformOverride(
54 "expandvp-override-mask-transform", cl::init(Val: ""), cl::Hidden,
55 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
56 ". If non-empty, Ignore "
57 "TargetTransformInfo and "
58 "always use this transformation for the %mask parameter (Used in "
59 "testing)."));
60
61#undef VPINTERNAL_CASE
62#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
63
64static VPTransform parseOverrideOption(const std::string &TextOpt) {
65 return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES;
66}
67
68#undef VPINTERNAL_VPLEGAL_CASES
69
70// Whether any override options are set.
71static bool anyExpandVPOverridesSet() {
72 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
73}
74
75#define DEBUG_TYPE "expandvp"
76
77STATISTIC(NumFoldedVL, "Number of folded vector length params");
78STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
79
80///// Helpers {
81
82/// \returns Whether the vector mask \p MaskVal has all lane bits set.
83static bool isAllTrueMask(Value *MaskVal) {
84 if (Value *SplattedVal = getSplatValue(V: MaskVal))
85 if (auto *ConstValue = dyn_cast<Constant>(Val: SplattedVal))
86 return ConstValue->isAllOnesValue();
87
88 return false;
89}
90
91/// \returns A non-excepting divisor constant for this type.
92static Constant *getSafeDivisor(Type *DivTy) {
93 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
94 return ConstantInt::get(Ty: DivTy, V: 1u, IsSigned: false);
95}
96
97/// Transfer operation properties from \p OldVPI to \p NewVal.
98static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
99 auto *NewInst = dyn_cast<Instruction>(Val: &NewVal);
100 if (!NewInst || !isa<FPMathOperator>(Val: NewVal))
101 return;
102
103 auto *OldFMOp = dyn_cast<FPMathOperator>(Val: &VPI);
104 if (!OldFMOp)
105 return;
106
107 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
108}
109
110/// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
111/// OldVP gets erased.
112static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
113 transferDecorations(NewVal&: NewOp, VPI&: OldOp);
114
115 if (isa<Instruction>(Val: NewOp) && !NewOp.hasName() && OldOp.hasName())
116 NewOp.takeName(V: &OldOp);
117
118 OldOp.replaceAllUsesWith(V: &NewOp);
119 OldOp.eraseFromParent();
120}
121
122static bool maySpeculateLanes(VPIntrinsic &VPI) {
123 // The result of VP reductions depends on the mask and evl.
124 if (isa<VPReductionIntrinsic>(Val: VPI))
125 return false;
126 // Fallback to whether the intrinsic is speculatable.
127 if (auto IntrID = VPI.getFunctionalIntrinsicID())
128 return Intrinsic::getFnAttributes(C&: VPI.getContext(), id: *IntrID)
129 .hasAttribute(Kind: Attribute::AttrKind::Speculatable);
130 if (auto Opc = VPI.getFunctionalOpcode())
131 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: *Opc, Inst: &VPI);
132 return false;
133}
134
135//// } Helpers
136
137namespace {
138
139// Expansion pass state at function scope.
140struct CachingVPExpander {
141 const TargetTransformInfo &TTI;
142
143 /// \returns A bitmask that is true where the lane position is less-than \p
144 /// EVLParam
145 ///
146 /// \p Builder
147 /// Used for instruction creation.
148 /// \p VLParam
149 /// The explicit vector length parameter to test against the lane
150 /// positions.
151 /// \p ElemCount
152 /// Static (potentially scalable) number of vector elements.
153 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
154 ElementCount ElemCount);
155
156 /// If needed, folds the EVL in the mask operand and discards the EVL
157 /// parameter. Returns true if the mask was actually folded.
158 bool foldEVLIntoMask(VPIntrinsic &VPI);
159
160 /// "Remove" the %evl parameter of \p PI by setting it to the static vector
161 /// length of the operation. Returns true if the %evl (if any) was effectively
162 /// changed.
163 bool discardEVLParameter(VPIntrinsic &PI);
164
165 /// Lower this VP binary operator to a unpredicated binary operator.
166 bool expandPredicationInBinaryOperator(IRBuilder<> &Builder, VPIntrinsic &PI);
167
168 /// Lower this VP int call to a unpredicated int call.
169 bool expandPredicationToIntCall(IRBuilder<> &Builder, VPIntrinsic &PI);
170
171 /// Lower this VP fp call to a unpredicated fp call.
172 bool expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
173 unsigned UnpredicatedIntrinsicID);
174
175 /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
176 bool expandPredicationInReduction(IRBuilder<> &Builder,
177 VPReductionIntrinsic &PI);
178
179 /// Lower this VP cast operation to a non-VP intrinsic.
180 bool expandPredicationToCastIntrinsic(IRBuilder<> &Builder, VPIntrinsic &VPI);
181
182 /// Lower this VP memory operation to a non-VP intrinsic.
183 bool expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
184 VPIntrinsic &VPI);
185
186 /// Lower this VP comparison to a call to an unpredicated comparison.
187 bool expandPredicationInComparison(IRBuilder<> &Builder, VPCmpIntrinsic &PI);
188
189 /// Query TTI and expand the vector predication in \p P accordingly.
190 bool expandPredication(VPIntrinsic &PI);
191
192 /// Determine how and whether the VPIntrinsic \p VPI shall be expanded. This
193 /// overrides TTI with the cl::opts listed at the top of this file.
194 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
195 bool UsingTTIOverrides;
196
197public:
198 CachingVPExpander(const TargetTransformInfo &TTI)
199 : TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
200
201 /// Expand llvm.vp.* intrinsics as requested by \p TTI.
202 /// Returns the details of the expansion.
203 VPExpansionDetails expandVectorPredication(VPIntrinsic &VPI);
204};
205
206//// CachingVPExpander {
207
208Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
209 Value *EVLParam,
210 ElementCount ElemCount) {
211 // TODO add caching
212 // Scalable vector %evl conversion.
213 if (ElemCount.isScalable()) {
214 Type *BoolVecTy = VectorType::get(ElementType: Builder.getInt1Ty(), EC: ElemCount);
215 // `get_active_lane_mask` performs an implicit less-than comparison.
216 Value *ConstZero = Builder.getInt32(C: 0);
217 return Builder.CreateIntrinsic(ID: Intrinsic::get_active_lane_mask,
218 OverloadTypes: {BoolVecTy, EVLParam->getType()},
219 Args: {ConstZero, EVLParam});
220 }
221
222 // Fixed vector %evl conversion.
223 Type *LaneTy = EVLParam->getType();
224 unsigned NumElems = ElemCount.getFixedValue();
225 Value *VLSplat = Builder.CreateVectorSplat(NumElts: NumElems, V: EVLParam);
226 Value *IdxVec = Builder.CreateStepVector(DstType: VectorType::get(ElementType: LaneTy, EC: ElemCount));
227 return Builder.CreateICmp(P: CmpInst::ICMP_ULT, LHS: IdxVec, RHS: VLSplat);
228}
229
230bool CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
231 VPIntrinsic &VPI) {
232 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
233 "Implicitly dropping %evl in non-speculatable operator!");
234
235 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
236 assert(Instruction::isBinaryOp(OC));
237
238 Value *Op0 = VPI.getOperand(i_nocapture: 0);
239 Value *Op1 = VPI.getOperand(i_nocapture: 1);
240 Value *Mask = VPI.getMaskParam();
241
242 // Blend in safe operands.
243 if (Mask && !isAllTrueMask(MaskVal: Mask)) {
244 switch (OC) {
245 default:
246 // Can safely ignore the predicate.
247 break;
248
249 // Division operators need a safe divisor on masked-off lanes (1).
250 case Instruction::UDiv:
251 case Instruction::SDiv:
252 case Instruction::URem:
253 case Instruction::SRem:
254 // 2nd operand must not be zero.
255 Value *SafeDivisor = getSafeDivisor(DivTy: VPI.getType());
256 Op1 = Builder.CreateSelect(C: Mask, True: Op1, False: SafeDivisor);
257 }
258 }
259
260 Value *NewBinOp = Builder.CreateBinOp(Opc: OC, LHS: Op0, RHS: Op1);
261
262 replaceOperation(NewOp&: *NewBinOp, OldOp&: VPI);
263 return true;
264}
265
266bool CachingVPExpander::expandPredicationToIntCall(IRBuilder<> &Builder,
267 VPIntrinsic &VPI) {
268 std::optional<unsigned> FID = VPI.getFunctionalIntrinsicID();
269 if (!FID)
270 return false;
271 SmallVector<Value *, 2> Argument;
272 for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) {
273 Argument.push_back(Elt: VPI.getOperand(i_nocapture: i));
274 }
275 Value *NewOp =
276 Builder.CreateIntrinsic(ID: FID.value(), OverloadTypes: {VPI.getType()}, Args: Argument);
277 replaceOperation(NewOp&: *NewOp, OldOp&: VPI);
278 return true;
279}
280
281bool CachingVPExpander::expandPredicationToFPCall(
282 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
283 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
284 "Implicitly dropping %evl in non-speculatable operator!");
285
286 switch (UnpredicatedIntrinsicID) {
287 case Intrinsic::fabs:
288 case Intrinsic::copysign:
289 case Intrinsic::sqrt:
290 case Intrinsic::maxnum:
291 case Intrinsic::minnum:
292 case Intrinsic::maximum:
293 case Intrinsic::minimum:
294 case Intrinsic::ceil:
295 case Intrinsic::floor:
296 case Intrinsic::round:
297 case Intrinsic::roundeven:
298 case Intrinsic::trunc:
299 case Intrinsic::rint:
300 case Intrinsic::nearbyint:
301 case Intrinsic::lrint:
302 case Intrinsic::llrint:
303 case Intrinsic::is_fpclass: {
304 SmallVector<Value *, 2> Argument;
305 for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) {
306 Argument.push_back(Elt: VPI.getOperand(i_nocapture: i));
307 }
308 Value *NewOp = Builder.CreateIntrinsic(RetTy: VPI.getType(),
309 ID: UnpredicatedIntrinsicID, Args: Argument);
310 replaceOperation(NewOp&: *NewOp, OldOp&: VPI);
311 return true;
312 }
313 case Intrinsic::fma:
314 case Intrinsic::fmuladd:
315 case Intrinsic::experimental_constrained_fma:
316 case Intrinsic::experimental_constrained_fmuladd: {
317 Value *Op0 = VPI.getOperand(i_nocapture: 0);
318 Value *Op1 = VPI.getOperand(i_nocapture: 1);
319 Value *Op2 = VPI.getOperand(i_nocapture: 2);
320 Function *Fn = Intrinsic::getOrInsertDeclaration(
321 M: VPI.getModule(), id: UnpredicatedIntrinsicID, OverloadTys: {VPI.getType()});
322 Value *NewOp;
323 if (Intrinsic::isConstrainedFPIntrinsic(QID: UnpredicatedIntrinsicID))
324 NewOp = Builder.CreateConstrainedFPCall(Callee: Fn, Args: {Op0, Op1, Op2});
325 else
326 NewOp = Builder.CreateCall(Callee: Fn, Args: {Op0, Op1, Op2});
327 replaceOperation(NewOp&: *NewOp, OldOp&: VPI);
328 return true;
329 }
330 }
331
332 return false;
333}
334
335static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
336 Type *EltTy) {
337 Intrinsic::ID RdxID = *VPI.getFunctionalIntrinsicID();
338 return getReductionIdentity(RdxID, Ty: EltTy, FMF: VPI.getFastMathFlagsOrNone());
339}
340
341bool CachingVPExpander::expandPredicationInReduction(
342 IRBuilder<> &Builder, VPReductionIntrinsic &VPI) {
343 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
344 "Implicitly dropping %evl in non-speculatable operator!");
345
346 Value *Mask = VPI.getMaskParam();
347 Value *RedOp = VPI.getOperand(i_nocapture: VPI.getVectorParamPos());
348
349 // Insert neutral element in masked-out positions
350 if (Mask && !isAllTrueMask(MaskVal: Mask)) {
351 auto *NeutralElt = getNeutralReductionElement(VPI, EltTy: VPI.getType());
352 auto *NeutralVector = Builder.CreateVectorSplat(
353 EC: cast<VectorType>(Val: RedOp->getType())->getElementCount(), V: NeutralElt);
354 RedOp = Builder.CreateSelect(C: Mask, True: RedOp, False: NeutralVector);
355 }
356
357 Value *Reduction;
358 Value *Start = VPI.getOperand(i_nocapture: VPI.getStartParamPos());
359
360 switch (VPI.getIntrinsicID()) {
361 default:
362 llvm_unreachable("Impossible reduction kind");
363 case Intrinsic::vp_reduce_add:
364 case Intrinsic::vp_reduce_mul:
365 case Intrinsic::vp_reduce_and:
366 case Intrinsic::vp_reduce_or:
367 case Intrinsic::vp_reduce_xor: {
368 Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID();
369 unsigned Opc = getArithmeticReductionInstruction(RdxID: RedID);
370 assert(Instruction::isBinaryOp(Opc));
371 Reduction = Builder.CreateUnaryIntrinsic(ID: RedID, Op: RedOp);
372 Reduction =
373 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Reduction, RHS: Start);
374 break;
375 }
376 case Intrinsic::vp_reduce_smax:
377 case Intrinsic::vp_reduce_smin:
378 case Intrinsic::vp_reduce_umax:
379 case Intrinsic::vp_reduce_umin:
380 case Intrinsic::vp_reduce_fmax:
381 case Intrinsic::vp_reduce_fmin:
382 case Intrinsic::vp_reduce_fmaximum:
383 case Intrinsic::vp_reduce_fminimum: {
384 Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID();
385 Intrinsic::ID ScalarID = getMinMaxReductionIntrinsicOp(RdxID: RedID);
386 Reduction = Builder.CreateUnaryIntrinsic(ID: RedID, Op: RedOp);
387 transferDecorations(NewVal&: *Reduction, VPI);
388 Reduction = Builder.CreateBinaryIntrinsic(ID: ScalarID, LHS: Reduction, RHS: Start);
389 break;
390 }
391 case Intrinsic::vp_reduce_fadd:
392 Reduction = Builder.CreateFAddReduce(Acc: Start, Src: RedOp);
393 break;
394 case Intrinsic::vp_reduce_fmul:
395 Reduction = Builder.CreateFMulReduce(Acc: Start, Src: RedOp);
396 break;
397 }
398
399 replaceOperation(NewOp&: *Reduction, OldOp&: VPI);
400 return true;
401}
402
403bool CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
404 VPIntrinsic &VPI) {
405 Intrinsic::ID VPID = VPI.getIntrinsicID();
406 unsigned CastOpcode = VPIntrinsic::getFunctionalOpcodeForVP(ID: VPID).value();
407 assert(Instruction::isCast(CastOpcode));
408 Value *CastOp = Builder.CreateCast(Op: Instruction::CastOps(CastOpcode),
409 V: VPI.getOperand(i_nocapture: 0), DestTy: VPI.getType());
410
411 replaceOperation(NewOp&: *CastOp, OldOp&: VPI);
412 return true;
413}
414
415bool CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
416 VPIntrinsic &VPI) {
417 assert(VPI.canIgnoreVectorLengthParam());
418
419 const auto &DL = VPI.getDataLayout();
420
421 Value *MaskParam = VPI.getMaskParam();
422 Value *PtrParam = VPI.getMemoryPointerParam();
423 Value *DataParam = VPI.getMemoryDataParam();
424 bool IsUnmasked = isAllTrueMask(MaskVal: MaskParam);
425
426 MaybeAlign AlignOpt = VPI.getPointerAlignment();
427
428 Value *NewMemoryInst = nullptr;
429 switch (VPI.getIntrinsicID()) {
430 default:
431 llvm_unreachable("Not a VP memory intrinsic");
432 case Intrinsic::vp_store:
433 if (IsUnmasked) {
434 StoreInst *NewStore =
435 Builder.CreateStore(Val: DataParam, Ptr: PtrParam, /*IsVolatile*/ isVolatile: false);
436 if (AlignOpt.has_value())
437 NewStore->setAlignment(*AlignOpt);
438 NewMemoryInst = NewStore;
439 } else
440 NewMemoryInst = Builder.CreateMaskedStore(
441 Val: DataParam, Ptr: PtrParam, Alignment: AlignOpt.valueOrOne(), Mask: MaskParam);
442
443 break;
444 case Intrinsic::vp_load:
445 if (IsUnmasked) {
446 LoadInst *NewLoad =
447 Builder.CreateLoad(Ty: VPI.getType(), Ptr: PtrParam, /*IsVolatile*/ isVolatile: false);
448 if (AlignOpt.has_value())
449 NewLoad->setAlignment(*AlignOpt);
450 NewMemoryInst = NewLoad;
451 } else
452 NewMemoryInst = Builder.CreateMaskedLoad(
453 Ty: VPI.getType(), Ptr: PtrParam, Alignment: AlignOpt.valueOrOne(), Mask: MaskParam);
454
455 break;
456 case Intrinsic::vp_scatter: {
457 auto *ElementType =
458 cast<VectorType>(Val: DataParam->getType())->getElementType();
459 NewMemoryInst = Builder.CreateMaskedScatter(
460 Val: DataParam, Ptrs: PtrParam,
461 Alignment: AlignOpt.value_or(u: DL.getPrefTypeAlign(Ty: ElementType)), Mask: MaskParam);
462 break;
463 }
464 case Intrinsic::vp_gather: {
465 auto *ElementType = cast<VectorType>(Val: VPI.getType())->getElementType();
466 NewMemoryInst = Builder.CreateMaskedGather(
467 Ty: VPI.getType(), Ptrs: PtrParam,
468 Alignment: AlignOpt.value_or(u: DL.getPrefTypeAlign(Ty: ElementType)), Mask: MaskParam,
469 PassThru: nullptr);
470 break;
471 }
472 }
473
474 assert(NewMemoryInst);
475 replaceOperation(NewOp&: *NewMemoryInst, OldOp&: VPI);
476 return true;
477}
478
479bool CachingVPExpander::expandPredicationInComparison(IRBuilder<> &Builder,
480 VPCmpIntrinsic &VPI) {
481 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
482 "Implicitly dropping %evl in non-speculatable operator!");
483
484 assert(*VPI.getFunctionalOpcode() == Instruction::ICmp ||
485 *VPI.getFunctionalOpcode() == Instruction::FCmp);
486
487 Value *Op0 = VPI.getOperand(i_nocapture: 0);
488 Value *Op1 = VPI.getOperand(i_nocapture: 1);
489 auto Pred = VPI.getPredicate();
490
491 auto *NewCmp = Builder.CreateCmp(Pred, LHS: Op0, RHS: Op1);
492
493 replaceOperation(NewOp&: *NewCmp, OldOp&: VPI);
494 return true;
495}
496
497bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
498 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
499
500 if (VPI.canIgnoreVectorLengthParam())
501 return false;
502
503 Value *EVLParam = VPI.getVectorLengthParam();
504 if (!EVLParam)
505 return false;
506
507 ElementCount StaticElemCount = VPI.getStaticVectorLength();
508 Value *MaxEVL = nullptr;
509 Type *Int32Ty = Type::getInt32Ty(C&: VPI.getContext());
510 if (StaticElemCount.isScalable()) {
511 // TODO add caching
512 IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
513 Value *FactorConst = Builder.getInt32(C: StaticElemCount.getKnownMinValue());
514 Value *VScale = Builder.CreateVScale(Ty: Int32Ty, Name: "vscale");
515 MaxEVL = Builder.CreateNUWMul(LHS: VScale, RHS: FactorConst, Name: "scalable_size");
516 } else {
517 MaxEVL = ConstantInt::get(Ty: Int32Ty, V: StaticElemCount.getFixedValue(), IsSigned: false);
518 }
519 VPI.setVectorLengthParam(MaxEVL);
520 return true;
521}
522
523bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
524 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
525
526 IRBuilder<> Builder(&VPI);
527
528 // Ineffective %evl parameter and so nothing to do here.
529 if (VPI.canIgnoreVectorLengthParam())
530 return false;
531
532 // Only VP intrinsics can have an %evl parameter.
533 Value *OldMaskParam = VPI.getMaskParam();
534 if (!OldMaskParam) {
535 assert((VPI.getIntrinsicID() == Intrinsic::vp_merge ||
536 VPI.getIntrinsicID() == Intrinsic::vp_select) &&
537 "Unexpected VP intrinsic without mask operand");
538 OldMaskParam = VPI.getArgOperand(i: 0);
539 }
540
541 Value *OldEVLParam = VPI.getVectorLengthParam();
542 assert(OldMaskParam && "no mask param to fold the vl param into");
543 assert(OldEVLParam && "no EVL param to fold away");
544
545 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
546 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
547
548 // Convert the %evl predication into vector mask predication.
549 ElementCount ElemCount = VPI.getStaticVectorLength();
550 Value *VLMask = convertEVLToMask(Builder, EVLParam: OldEVLParam, ElemCount);
551 Value *NewMaskParam = Builder.CreateAnd(LHS: VLMask, RHS: OldMaskParam);
552 if (VPI.getIntrinsicID() == Intrinsic::vp_merge ||
553 VPI.getIntrinsicID() == Intrinsic::vp_select)
554 VPI.setArgOperand(i: 0, v: NewMaskParam);
555 else
556 VPI.setMaskParam(NewMaskParam);
557
558 // Drop the %evl parameter.
559 discardEVLParameter(VPI);
560 assert(VPI.canIgnoreVectorLengthParam() &&
561 "transformation did not render the evl param ineffective!");
562
563 // Reassess the modified instruction.
564 return true;
565}
566
567bool CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
568 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
569
570 IRBuilder<> Builder(&VPI);
571
572 // Try lowering to a LLVM instruction first.
573 auto OC = VPI.getFunctionalOpcode();
574
575 if (OC && Instruction::isBinaryOp(Opcode: *OC))
576 return expandPredicationInBinaryOperator(Builder, VPI);
577
578 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(Val: &VPI))
579 return expandPredicationInReduction(Builder, VPI&: *VPRI);
580
581 if (auto *VPCmp = dyn_cast<VPCmpIntrinsic>(Val: &VPI))
582 return expandPredicationInComparison(Builder, VPI&: *VPCmp);
583
584 if (VPCastIntrinsic::isVPCast(ID: VPI.getIntrinsicID()))
585 return expandPredicationToCastIntrinsic(Builder, VPI);
586
587 switch (VPI.getIntrinsicID()) {
588 default:
589 break;
590 case Intrinsic::vp_fneg: {
591 Value *NewNegOp = Builder.CreateFNeg(V: VPI.getOperand(i_nocapture: 0));
592 replaceOperation(NewOp&: *NewNegOp, OldOp&: VPI);
593 return NewNegOp;
594 }
595 case Intrinsic::vp_select:
596 case Intrinsic::vp_merge: {
597 assert(maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam());
598 Value *NewSelectOp = Builder.CreateSelect(
599 C: VPI.getOperand(i_nocapture: 0), True: VPI.getOperand(i_nocapture: 1), False: VPI.getOperand(i_nocapture: 2));
600 replaceOperation(NewOp&: *NewSelectOp, OldOp&: VPI);
601 return NewSelectOp;
602 }
603 case Intrinsic::vp_abs:
604 case Intrinsic::vp_smax:
605 case Intrinsic::vp_smin:
606 case Intrinsic::vp_umax:
607 case Intrinsic::vp_umin:
608 case Intrinsic::vp_bswap:
609 case Intrinsic::vp_bitreverse:
610 case Intrinsic::vp_ctpop:
611 case Intrinsic::vp_ctlz:
612 case Intrinsic::vp_cttz:
613 case Intrinsic::vp_sadd_sat:
614 case Intrinsic::vp_uadd_sat:
615 case Intrinsic::vp_ssub_sat:
616 case Intrinsic::vp_usub_sat:
617 case Intrinsic::vp_fshl:
618 case Intrinsic::vp_fshr:
619 return expandPredicationToIntCall(Builder, VPI);
620 case Intrinsic::vp_fabs:
621 case Intrinsic::vp_copysign:
622 case Intrinsic::vp_sqrt:
623 case Intrinsic::vp_maxnum:
624 case Intrinsic::vp_minnum:
625 case Intrinsic::vp_maximum:
626 case Intrinsic::vp_minimum:
627 case Intrinsic::vp_ceil:
628 case Intrinsic::vp_floor:
629 case Intrinsic::vp_round:
630 case Intrinsic::vp_roundeven:
631 case Intrinsic::vp_roundtozero:
632 case Intrinsic::vp_rint:
633 case Intrinsic::vp_nearbyint:
634 case Intrinsic::vp_lrint:
635 case Intrinsic::vp_llrint:
636 case Intrinsic::vp_fma:
637 case Intrinsic::vp_fmuladd:
638 case Intrinsic::vp_is_fpclass:
639 return expandPredicationToFPCall(Builder, VPI,
640 UnpredicatedIntrinsicID: VPI.getFunctionalIntrinsicID().value());
641 case Intrinsic::vp_load:
642 case Intrinsic::vp_store:
643 case Intrinsic::vp_gather:
644 case Intrinsic::vp_scatter:
645 return expandPredicationInMemoryIntrinsic(Builder, VPI);
646 }
647
648 if (auto CID = VPI.getConstrainedIntrinsicID())
649 if (expandPredicationToFPCall(Builder, VPI, UnpredicatedIntrinsicID: *CID))
650 return true;
651
652 return false;
653}
654
655//// } CachingVPExpander
656
657void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
658 // Operations with speculatable lanes do not strictly need predication.
659 if (maySpeculateLanes(VPI)) {
660 // Converting a speculatable VP intrinsic means dropping %mask and %evl.
661 // No need to expand %evl into the %mask only to ignore that code.
662 if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
663 LegalizeStrat.EVLParamStrategy = VPLegalization::Discard;
664 return;
665 }
666
667 // We have to preserve the predicating effect of %evl for this
668 // non-speculatable VP intrinsic.
669 // 1) Never discard %evl.
670 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
671 // %evl gets folded into %mask.
672 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
673 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
674 LegalizeStrat.EVLParamStrategy = VPLegalization::Convert;
675 }
676}
677
678VPLegalization
679CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
680 auto VPStrat = TTI.getVPLegalizationStrategy(PI: VPI);
681 if (LLVM_LIKELY(!UsingTTIOverrides)) {
682 // No overrides - we are in production.
683 return VPStrat;
684 }
685
686 // Overrides set - we are in testing, the following does not need to be
687 // efficient.
688 VPStrat.EVLParamStrategy = parseOverrideOption(TextOpt: EVLTransformOverride);
689 VPStrat.OpStrategy = parseOverrideOption(TextOpt: MaskTransformOverride);
690 return VPStrat;
691}
692
693VPExpansionDetails
694CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) {
695 auto Strategy = getVPLegalizationStrategy(VPI);
696 sanitizeStrategy(VPI, LegalizeStrat&: Strategy);
697
698 VPExpansionDetails Changed = VPExpansionDetails::IntrinsicUnchanged;
699
700 // Transform the EVL parameter.
701 switch (Strategy.EVLParamStrategy) {
702 case VPLegalization::Legal:
703 break;
704 case VPLegalization::Discard:
705 if (discardEVLParameter(VPI))
706 Changed = VPExpansionDetails::IntrinsicUpdated;
707 break;
708 case VPLegalization::Convert:
709 if (foldEVLIntoMask(VPI)) {
710 Changed = VPExpansionDetails::IntrinsicUpdated;
711 ++NumFoldedVL;
712 }
713 break;
714 }
715
716 // Replace with a non-predicated operation.
717 switch (Strategy.OpStrategy) {
718 case VPLegalization::Legal:
719 break;
720 case VPLegalization::Discard:
721 llvm_unreachable("Invalid strategy for operators.");
722 case VPLegalization::Convert:
723 if (expandPredication(VPI)) {
724 ++NumLoweredVPOps;
725 Changed = VPExpansionDetails::IntrinsicReplaced;
726 }
727 break;
728 }
729
730 return Changed;
731}
732} // namespace
733
734VPExpansionDetails
735llvm::expandVectorPredicationIntrinsic(VPIntrinsic &VPI,
736 const TargetTransformInfo &TTI) {
737 return CachingVPExpander(TTI).expandVectorPredication(VPI);
738}
739