1//===- RISCVGatherScatterLowering.cpp - Gather/Scatter lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass custom lowers llvm.gather and llvm.scatter instructions to
10// RISC-V intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCV.h"
15#include "RISCVTargetMachine.h"
16#include "llvm/Analysis/InstSimplifyFolder.h"
17#include "llvm/Analysis/LoopInfo.h"
18#include "llvm/Analysis/ValueTracking.h"
19#include "llvm/Analysis/VectorUtils.h"
20#include "llvm/CodeGen/TargetPassConfig.h"
21#include "llvm/IR/GetElementPtrTypeIterator.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/PatternMatch.h"
25#include "llvm/InitializePasses.h"
26#include "llvm/Pass.h"
27#include "llvm/Transforms/Utils/Local.h"
28#include <optional>
29
30using namespace llvm;
31using namespace PatternMatch;
32
33#define DEBUG_TYPE "riscv-gather-scatter-lowering"
34
35namespace {
36
37class RISCVGatherScatterLoweringImpl {
38 const RISCVSubtarget *ST;
39 const RISCVTargetLowering *TLI;
40 function_ref<LoopInfo &()> GetLI;
41 LoopInfo *LI = nullptr;
42 const DataLayout *DL;
43
44 SmallVector<WeakTrackingVH> MaybeDeadPHIs;
45
46 // Cache of the BasePtr and Stride determined from this GEP. When a GEP is
47 // used by multiple gathers/scatters, this allow us to reuse the scalar
48 // instructions we created for the first gather/scatter for the others.
49 DenseMap<GetElementPtrInst *, std::pair<Value *, Value *>> StridedAddrs;
50
51public:
52 RISCVGatherScatterLoweringImpl(const RISCVSubtarget *ST,
53 function_ref<LoopInfo &()> GetLI,
54 const DataLayout *DL)
55 : ST(ST), TLI(ST->getTargetLowering()), GetLI(GetLI), DL(DL) {}
56
57 bool run(Function &F);
58
59private:
60 bool tryCreateStridedLoadStore(IntrinsicInst *II);
61
62 std::pair<Value *, Value *> determineBaseAndStride(Instruction *Ptr,
63 IRBuilderBase &Builder);
64
65 bool matchStridedRecurrence(Value *Index, Loop *L, Value *&Stride,
66 PHINode *&BasePtr, BinaryOperator *&Inc,
67 IRBuilderBase &Builder);
68};
69
70} // end anonymous namespace
71
72namespace {
73class RISCVGatherScatterLoweringLegacy : public FunctionPass {
74public:
75 static char ID;
76
77 RISCVGatherScatterLoweringLegacy() : FunctionPass(ID) {}
78
79 bool runOnFunction(Function &F) override;
80
81 void getAnalysisUsage(AnalysisUsage &AU) const override {
82 AU.setPreservesCFG();
83 AU.addRequired<TargetPassConfig>();
84 AU.addRequired<LoopInfoWrapperPass>();
85 }
86
87 StringRef getPassName() const override {
88 return "RISC-V gather/scatter lowering";
89 }
90};
91} // namespace
92
93char RISCVGatherScatterLoweringLegacy::ID = 0;
94
95INITIALIZE_PASS_BEGIN(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
96 "RISC-V gather/scatter lowering pass", false, false)
97INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
98INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
99INITIALIZE_PASS_END(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
100 "RISC-V gather/scatter lowering pass", false, false)
101
102FunctionPass *llvm::createRISCVGatherScatterLoweringLegacyPass() {
103 return new RISCVGatherScatterLoweringLegacy();
104}
105
106// TODO: Should we consider the mask when looking for a stride?
107static std::pair<Value *, Value *> matchStridedConstant(Constant *StartC) {
108 if (!isa<FixedVectorType>(Val: StartC->getType()))
109 return std::make_pair(x: nullptr, y: nullptr);
110
111 unsigned NumElts = cast<FixedVectorType>(Val: StartC->getType())->getNumElements();
112
113 // Check that the start value is a strided constant.
114 auto *StartVal =
115 dyn_cast_or_null<ConstantInt>(Val: StartC->getAggregateElement(Elt: (unsigned)0));
116 if (!StartVal)
117 return std::make_pair(x: nullptr, y: nullptr);
118 APInt StrideVal(StartVal->getValue().getBitWidth(), 0);
119 ConstantInt *Prev = StartVal;
120 for (unsigned i = 1; i != NumElts; ++i) {
121 auto *C = dyn_cast_or_null<ConstantInt>(Val: StartC->getAggregateElement(Elt: i));
122 if (!C)
123 return std::make_pair(x: nullptr, y: nullptr);
124
125 APInt LocalStride = C->getValue() - Prev->getValue();
126 if (i == 1)
127 StrideVal = LocalStride;
128 else if (StrideVal != LocalStride)
129 return std::make_pair(x: nullptr, y: nullptr);
130
131 Prev = C;
132 }
133
134 Value *Stride = ConstantInt::get(Ty: StartVal->getType(), V: StrideVal);
135
136 return std::make_pair(x&: StartVal, y&: Stride);
137}
138
139static std::pair<Value *, Value *> matchStridedStart(Value *Start,
140 IRBuilderBase &Builder) {
141 // Base case, start is a strided constant.
142 auto *StartC = dyn_cast<Constant>(Val: Start);
143 if (StartC)
144 return matchStridedConstant(StartC);
145
146 // Base case, start is a stepvector
147 if (match(V: Start, P: m_Intrinsic<Intrinsic::stepvector>())) {
148 auto *Ty = Start->getType()->getScalarType();
149 return std::make_pair(x: ConstantInt::get(Ty, V: 0), y: ConstantInt::get(Ty, V: 1));
150 }
151
152 // Not a constant, maybe it's a strided constant with a splat added or
153 // multiplied.
154 auto *BO = dyn_cast<BinaryOperator>(Val: Start);
155 if (!BO || (BO->getOpcode() != Instruction::Add &&
156 BO->getOpcode() != Instruction::Or &&
157 BO->getOpcode() != Instruction::Shl &&
158 BO->getOpcode() != Instruction::Mul))
159 return std::make_pair(x: nullptr, y: nullptr);
160
161 if (BO->getOpcode() == Instruction::Or &&
162 !cast<PossiblyDisjointInst>(Val: BO)->isDisjoint())
163 return std::make_pair(x: nullptr, y: nullptr);
164
165 // Look for an operand that is splatted.
166 unsigned OtherIndex = 0;
167 Value *Splat = getSplatValue(V: BO->getOperand(i_nocapture: 1));
168 if (!Splat && Instruction::isCommutative(Opcode: BO->getOpcode())) {
169 Splat = getSplatValue(V: BO->getOperand(i_nocapture: 0));
170 OtherIndex = 1;
171 }
172 if (!Splat)
173 return std::make_pair(x: nullptr, y: nullptr);
174
175 Value *Stride;
176 std::tie(args&: Start, args&: Stride) = matchStridedStart(Start: BO->getOperand(i_nocapture: OtherIndex),
177 Builder);
178 if (!Start)
179 return std::make_pair(x: nullptr, y: nullptr);
180
181 Builder.SetInsertPoint(BO);
182 Builder.SetCurrentDebugLocation(DebugLoc());
183 // Add the splat value to the start or multiply the start and stride by the
184 // splat.
185 switch (BO->getOpcode()) {
186 default:
187 llvm_unreachable("Unexpected opcode");
188 case Instruction::Or:
189 Start = Builder.CreateDisjointOr(LHS: Start, RHS: Splat);
190 break;
191 case Instruction::Add:
192 Start = Builder.CreateAdd(LHS: Start, RHS: Splat);
193 break;
194 case Instruction::Mul:
195 Start = Builder.CreateMul(LHS: Start, RHS: Splat);
196 Stride = Builder.CreateMul(LHS: Stride, RHS: Splat);
197 break;
198 case Instruction::Shl:
199 Start = Builder.CreateShl(LHS: Start, RHS: Splat);
200 Stride = Builder.CreateShl(LHS: Stride, RHS: Splat);
201 break;
202 }
203
204 return std::make_pair(x&: Start, y&: Stride);
205}
206
207// Recursively, walk about the use-def chain until we find a Phi with a strided
208// start value. Build and update a scalar recurrence as we unwind the recursion.
209// We also update the Stride as we unwind. Our goal is to move all of the
210// arithmetic out of the loop.
211bool RISCVGatherScatterLoweringImpl::matchStridedRecurrence(
212 Value *Index, Loop *L, Value *&Stride, PHINode *&BasePtr,
213 BinaryOperator *&Inc, IRBuilderBase &Builder) {
214 // Our base case is a Phi.
215 if (auto *Phi = dyn_cast<PHINode>(Val: Index)) {
216 // A phi node we want to perform this function on should be from the
217 // loop header.
218 if (Phi->getParent() != L->getHeader())
219 return false;
220
221 Value *Step, *Start;
222 if (!matchSimpleRecurrence(P: Phi, BO&: Inc, Start, Step) ||
223 Inc->getOpcode() != Instruction::Add)
224 return false;
225 assert(Phi->getNumIncomingValues() == 2 && "Expected 2 operand phi.");
226 unsigned IncrementingBlock = Phi->getIncomingValue(i: 0) == Inc ? 0 : 1;
227 assert(Phi->getIncomingValue(IncrementingBlock) == Inc &&
228 "Expected one operand of phi to be Inc");
229
230 // Step should be a splat.
231 Step = getSplatValue(V: Step);
232 if (!Step)
233 return false;
234
235 std::tie(args&: Start, args&: Stride) = matchStridedStart(Start, Builder);
236 if (!Start)
237 return false;
238 assert(Stride != nullptr);
239
240 // Build scalar phi and increment.
241 BasePtr =
242 PHINode::Create(Ty: Start->getType(), NumReservedValues: 2, NameStr: Phi->getName() + ".scalar", InsertBefore: Phi->getIterator());
243 Inc = BinaryOperator::CreateAdd(V1: BasePtr, V2: Step, Name: Inc->getName() + ".scalar",
244 InsertBefore: Inc->getIterator());
245 BasePtr->addIncoming(V: Start, BB: Phi->getIncomingBlock(i: 1 - IncrementingBlock));
246 BasePtr->addIncoming(V: Inc, BB: Phi->getIncomingBlock(i: IncrementingBlock));
247
248 // Note that this Phi might be eligible for removal.
249 MaybeDeadPHIs.push_back(Elt: Phi);
250 return true;
251 }
252
253 // Otherwise look for binary operator.
254 auto *BO = dyn_cast<BinaryOperator>(Val: Index);
255 if (!BO)
256 return false;
257
258 switch (BO->getOpcode()) {
259 default:
260 return false;
261 case Instruction::Or:
262 // We need to be able to treat Or as Add.
263 if (!cast<PossiblyDisjointInst>(Val: BO)->isDisjoint())
264 return false;
265 break;
266 case Instruction::Add:
267 break;
268 case Instruction::Shl:
269 break;
270 case Instruction::Mul:
271 break;
272 }
273
274 // We should have one operand in the loop and one splat.
275 Value *OtherOp;
276 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 0)) &&
277 L->contains(Inst: cast<Instruction>(Val: BO->getOperand(i_nocapture: 0)))) {
278 Index = cast<Instruction>(Val: BO->getOperand(i_nocapture: 0));
279 OtherOp = BO->getOperand(i_nocapture: 1);
280 } else if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 1)) &&
281 L->contains(Inst: cast<Instruction>(Val: BO->getOperand(i_nocapture: 1))) &&
282 Instruction::isCommutative(Opcode: BO->getOpcode())) {
283 Index = cast<Instruction>(Val: BO->getOperand(i_nocapture: 1));
284 OtherOp = BO->getOperand(i_nocapture: 0);
285 } else {
286 return false;
287 }
288
289 // Make sure other op is loop invariant.
290 if (!L->isLoopInvariant(V: OtherOp))
291 return false;
292
293 // Make sure we have a splat.
294 Value *SplatOp = getSplatValue(V: OtherOp);
295 if (!SplatOp)
296 return false;
297
298 // Recurse up the use-def chain.
299 if (!matchStridedRecurrence(Index, L, Stride, BasePtr, Inc, Builder))
300 return false;
301
302 // Locate the Step and Start values from the recurrence.
303 unsigned StepIndex = Inc->getOperand(i_nocapture: 0) == BasePtr ? 1 : 0;
304 unsigned StartBlock = BasePtr->getOperand(i_nocapture: 0) == Inc ? 1 : 0;
305 Value *Step = Inc->getOperand(i_nocapture: StepIndex);
306 Value *Start = BasePtr->getOperand(i_nocapture: StartBlock);
307
308 // We need to adjust the start value in the preheader.
309 Builder.SetInsertPoint(
310 BasePtr->getIncomingBlock(i: StartBlock)->getTerminator());
311 Builder.SetCurrentDebugLocation(DebugLoc());
312
313 // TODO: Share this switch with matchStridedStart?
314 switch (BO->getOpcode()) {
315 default:
316 llvm_unreachable("Unexpected opcode!");
317 case Instruction::Add:
318 case Instruction::Or: {
319 // An add only affects the start value. It's ok to do this for Or because
320 // we already checked that there are no common set bits.
321 Start = Builder.CreateAdd(LHS: Start, RHS: SplatOp, Name: "start");
322 break;
323 }
324 case Instruction::Mul: {
325 Start = Builder.CreateMul(LHS: Start, RHS: SplatOp, Name: "start");
326 Stride = Builder.CreateMul(LHS: Stride, RHS: SplatOp, Name: "stride");
327 break;
328 }
329 case Instruction::Shl: {
330 Start = Builder.CreateShl(LHS: Start, RHS: SplatOp, Name: "start");
331 Stride = Builder.CreateShl(LHS: Stride, RHS: SplatOp, Name: "stride");
332 break;
333 }
334 }
335
336 // If the Step was defined inside the loop, adjust it before its definition
337 // instead of in the preheader.
338 if (auto *StepI = dyn_cast<Instruction>(Val: Step); StepI && L->contains(Inst: StepI))
339 Builder.SetInsertPoint(*StepI->getInsertionPointAfterDef());
340
341 switch (BO->getOpcode()) {
342 default:
343 break;
344 case Instruction::Mul:
345 Step = Builder.CreateMul(LHS: Step, RHS: SplatOp, Name: "step");
346 break;
347 case Instruction::Shl:
348 Step = Builder.CreateShl(LHS: Step, RHS: SplatOp, Name: "step");
349 break;
350 }
351
352 Inc->setOperand(i_nocapture: StepIndex, Val_nocapture: Step);
353 BasePtr->setIncomingValue(i: StartBlock, V: Start);
354 return true;
355}
356
357std::pair<Value *, Value *>
358RISCVGatherScatterLoweringImpl::determineBaseAndStride(Instruction *Ptr,
359 IRBuilderBase &Builder) {
360
361 // A gather/scatter of a splat is a zero strided load/store.
362 if (auto *BasePtr = getSplatValue(V: Ptr)) {
363 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
364 return std::make_pair(x&: BasePtr, y: ConstantInt::get(Ty: IntPtrTy, V: 0));
365 }
366
367 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
368 if (!GEP)
369 return std::make_pair(x: nullptr, y: nullptr);
370
371 auto I = StridedAddrs.find(Val: GEP);
372 if (I != StridedAddrs.end())
373 return I->second;
374
375 SmallVector<Value *, 2> Ops(GEP->operands());
376
377 // If the base pointer is a vector, check if it's strided.
378 Value *Base = GEP->getPointerOperand();
379 if (auto *BaseInst = dyn_cast<Instruction>(Val: Base);
380 BaseInst && BaseInst->getType()->isVectorTy()) {
381 // If GEP's offset is scalar then we can add it to the base pointer's base.
382 auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); };
383 if (all_of(Range: GEP->indices(), P: IsScalar)) {
384 auto [BaseBase, Stride] = determineBaseAndStride(Ptr: BaseInst, Builder);
385 if (BaseBase) {
386 Builder.SetInsertPoint(GEP);
387 SmallVector<Value *> Indices(GEP->indices());
388 Value *OffsetBase =
389 Builder.CreateGEP(Ty: GEP->getSourceElementType(), Ptr: BaseBase, IdxList: Indices,
390 Name: GEP->getName() + "offset", NW: GEP->isInBounds());
391 return {OffsetBase, Stride};
392 }
393 }
394 }
395
396 // Base pointer needs to be a scalar.
397 Value *ScalarBase = Base;
398 if (ScalarBase->getType()->isVectorTy()) {
399 ScalarBase = getSplatValue(V: ScalarBase);
400 if (!ScalarBase)
401 return std::make_pair(x: nullptr, y: nullptr);
402 }
403
404 std::optional<unsigned> VecOperand;
405 unsigned TypeScale = 0;
406
407 // Look for a vector operand and scale.
408 gep_type_iterator GTI = gep_type_begin(GEP);
409 for (unsigned i = 1, e = GEP->getNumOperands(); i != e; ++i, ++GTI) {
410 if (!Ops[i]->getType()->isVectorTy())
411 continue;
412
413 if (VecOperand)
414 return std::make_pair(x: nullptr, y: nullptr);
415
416 VecOperand = i;
417
418 TypeSize TS = GTI.getSequentialElementStride(DL: *DL);
419 if (TS.isScalable())
420 return std::make_pair(x: nullptr, y: nullptr);
421
422 TypeScale = TS.getFixedValue();
423 }
424
425 // We need to find a vector index to simplify.
426 if (!VecOperand)
427 return std::make_pair(x: nullptr, y: nullptr);
428
429 // We can't extract the stride if the arithmetic is done at a different size
430 // than the pointer type. Adding the stride later may not wrap correctly.
431 // Technically we could handle wider indices, but I don't expect that in
432 // practice. Handle one special case here - constants. This simplifies
433 // writing test cases.
434 Value *VecIndex = Ops[*VecOperand];
435 Type *VecIntPtrTy = DL->getIntPtrType(GEP->getType());
436 if (VecIndex->getType() != VecIntPtrTy) {
437 auto *VecIndexC = dyn_cast<Constant>(Val: VecIndex);
438 if (!VecIndexC)
439 return std::make_pair(x: nullptr, y: nullptr);
440 if (VecIndex->getType()->getScalarSizeInBits() > VecIntPtrTy->getScalarSizeInBits())
441 VecIndex = ConstantFoldCastInstruction(opcode: Instruction::Trunc, V: VecIndexC, DestTy: VecIntPtrTy);
442 else
443 VecIndex = ConstantFoldCastInstruction(opcode: Instruction::SExt, V: VecIndexC, DestTy: VecIntPtrTy);
444 }
445
446 // Handle the non-recursive case. This is what we see if the vectorizer
447 // decides to use a scalar IV + vid on demand instead of a vector IV.
448 auto [Start, Stride] = matchStridedStart(Start: VecIndex, Builder);
449 if (Start) {
450 assert(Stride);
451 Builder.SetInsertPoint(GEP);
452
453 // Replace the vector index with the scalar start and build a scalar GEP.
454 Ops[*VecOperand] = Start;
455 Type *SourceTy = GEP->getSourceElementType();
456 Value *BasePtr =
457 Builder.CreateGEP(Ty: SourceTy, Ptr: ScalarBase, IdxList: ArrayRef(Ops).drop_front());
458
459 // Convert stride to pointer size if needed.
460 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
461 assert(Stride->getType() == IntPtrTy && "Unexpected type");
462
463 // Scale the stride by the size of the indexed type.
464 if (TypeScale != 1)
465 Stride = Builder.CreateMul(LHS: Stride, RHS: ConstantInt::get(Ty: IntPtrTy, V: TypeScale));
466
467 auto P = std::make_pair(x&: BasePtr, y&: Stride);
468 StridedAddrs[GEP] = P;
469 return P;
470 }
471
472 if (!LI)
473 LI = &GetLI();
474
475 // Make sure we're in a loop and that has a pre-header and a single latch.
476 Loop *L = LI->getLoopFor(BB: GEP->getParent());
477 if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
478 return std::make_pair(x: nullptr, y: nullptr);
479
480 BinaryOperator *Inc;
481 PHINode *BasePhi;
482 if (!matchStridedRecurrence(Index: VecIndex, L, Stride, BasePtr&: BasePhi, Inc, Builder))
483 return std::make_pair(x: nullptr, y: nullptr);
484
485 assert(BasePhi->getNumIncomingValues() == 2 && "Expected 2 operand phi.");
486 unsigned IncrementingBlock = BasePhi->getOperand(i_nocapture: 0) == Inc ? 0 : 1;
487 assert(BasePhi->getIncomingValue(IncrementingBlock) == Inc &&
488 "Expected one operand of phi to be Inc");
489
490 Builder.SetInsertPoint(GEP);
491
492 // Replace the vector index with the scalar phi and build a scalar GEP.
493 Ops[*VecOperand] = BasePhi;
494 Type *SourceTy = GEP->getSourceElementType();
495 Value *BasePtr =
496 Builder.CreateGEP(Ty: SourceTy, Ptr: ScalarBase, IdxList: ArrayRef(Ops).drop_front());
497
498 // Final adjustments to stride should go in the start block.
499 Builder.SetInsertPoint(
500 BasePhi->getIncomingBlock(i: 1 - IncrementingBlock)->getTerminator());
501
502 // Convert stride to pointer size if needed.
503 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
504 assert(Stride->getType() == IntPtrTy && "Unexpected type");
505
506 // Scale the stride by the size of the indexed type.
507 if (TypeScale != 1)
508 Stride = Builder.CreateMul(LHS: Stride, RHS: ConstantInt::get(Ty: IntPtrTy, V: TypeScale));
509
510 auto P = std::make_pair(x&: BasePtr, y&: Stride);
511 StridedAddrs[GEP] = P;
512 return P;
513}
514
515bool RISCVGatherScatterLoweringImpl::tryCreateStridedLoadStore(
516 IntrinsicInst *II) {
517 VectorType *DataType;
518 Value *StoreVal = nullptr, *Ptr, *Mask, *EVL = nullptr;
519 Align Alignment;
520 switch (II->getIntrinsicID()) {
521 case Intrinsic::masked_gather:
522 DataType = cast<VectorType>(Val: II->getType());
523 Ptr = II->getArgOperand(i: 0);
524 Alignment = II->getParamAlign(ArgNo: 0).valueOrOne();
525 Mask = II->getArgOperand(i: 1);
526 break;
527 case Intrinsic::vp_gather:
528 DataType = cast<VectorType>(Val: II->getType());
529 Ptr = II->getArgOperand(i: 0);
530 // FIXME: Falling back to ABI alignment is incorrect.
531 Alignment = II->getParamAlign(ArgNo: 0).value_or(
532 u: DL->getABITypeAlign(Ty: DataType->getElementType()));
533 Mask = II->getArgOperand(i: 1);
534 EVL = II->getArgOperand(i: 2);
535 break;
536 case Intrinsic::masked_scatter:
537 DataType = cast<VectorType>(Val: II->getArgOperand(i: 0)->getType());
538 StoreVal = II->getArgOperand(i: 0);
539 Ptr = II->getArgOperand(i: 1);
540 Alignment = II->getParamAlign(ArgNo: 1).valueOrOne();
541 Mask = II->getArgOperand(i: 2);
542 break;
543 case Intrinsic::vp_scatter:
544 DataType = cast<VectorType>(Val: II->getArgOperand(i: 0)->getType());
545 StoreVal = II->getArgOperand(i: 0);
546 Ptr = II->getArgOperand(i: 1);
547 // FIXME: Falling back to ABI alignment is incorrect.
548 Alignment = II->getParamAlign(ArgNo: 1).value_or(
549 u: DL->getABITypeAlign(Ty: DataType->getElementType()));
550 Mask = II->getArgOperand(i: 2);
551 EVL = II->getArgOperand(i: 3);
552 break;
553 default:
554 llvm_unreachable("Unexpected intrinsic");
555 }
556
557 // Make sure the operation will be supported by the backend.
558 EVT DataTypeVT = TLI->getValueType(DL: *DL, Ty: DataType);
559 if (!TLI->isLegalStridedLoadStore(DataType: DataTypeVT, Alignment))
560 return false;
561
562 // FIXME: Let the backend type legalize by splitting/widening?
563 if (!TLI->isTypeLegal(VT: DataTypeVT))
564 return false;
565
566 // Pointer should be an instruction.
567 auto *PtrI = dyn_cast<Instruction>(Val: Ptr);
568 if (!PtrI)
569 return false;
570
571 IRBuilder Builder(PtrI->getIterator(), InstSimplifyFolder(*DL));
572 Builder.disableGEPCanonicalization();
573
574 Value *BasePtr, *Stride;
575 std::tie(args&: BasePtr, args&: Stride) = determineBaseAndStride(Ptr: PtrI, Builder);
576 if (!BasePtr)
577 return false;
578 assert(Stride != nullptr);
579
580 Builder.SetInsertPoint(II);
581
582 if (!EVL)
583 EVL = Builder.CreateElementCount(
584 Ty: Builder.getInt32Ty(), EC: cast<VectorType>(Val: DataType)->getElementCount());
585
586 Value *Call;
587
588 if (!StoreVal) {
589 Call = Builder.CreateIntrinsic(
590 ID: Intrinsic::experimental_vp_strided_load,
591 OverloadTypes: {DataType, BasePtr->getType(), Stride->getType()},
592 Args: {BasePtr, Stride, Mask, EVL});
593
594 // Merge llvm.masked.gather's passthru
595 if (II->getIntrinsicID() == Intrinsic::masked_gather)
596 Call = Builder.CreateSelect(C: Mask, True: Call, False: II->getArgOperand(i: 2));
597 } else
598 Call = Builder.CreateIntrinsic(
599 ID: Intrinsic::experimental_vp_strided_store,
600 OverloadTypes: {DataType, BasePtr->getType(), Stride->getType()},
601 Args: {StoreVal, BasePtr, Stride, Mask, EVL});
602
603 Call->takeName(V: II);
604 II->replaceAllUsesWith(V: Call);
605 II->eraseFromParent();
606
607 if (PtrI->use_empty())
608 RecursivelyDeleteTriviallyDeadInstructions(V: PtrI);
609
610 return true;
611}
612
613bool RISCVGatherScatterLoweringImpl::run(Function &F) {
614 if (!ST->hasVInstructions() || !ST->useRVVForFixedLengthVectors())
615 return false;
616
617 SmallVector<IntrinsicInst *, 4> Worklist;
618
619 bool Changed = false;
620
621 for (BasicBlock &BB : F) {
622 for (Instruction &I : BB) {
623 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I);
624 if (!II)
625 continue;
626 switch (II->getIntrinsicID()) {
627 case Intrinsic::masked_gather:
628 case Intrinsic::masked_scatter:
629 case Intrinsic::vp_gather:
630 case Intrinsic::vp_scatter:
631 Worklist.push_back(Elt: II);
632 break;
633 default:
634 break;
635 }
636 }
637 }
638
639 // Rewrite gather/scatter to form strided load/store if possible.
640 for (auto *II : Worklist)
641 Changed |= tryCreateStridedLoadStore(II);
642
643 // Remove any dead phis.
644 while (!MaybeDeadPHIs.empty()) {
645 if (auto *Phi = dyn_cast_or_null<PHINode>(Val: MaybeDeadPHIs.pop_back_val()))
646 RecursivelyDeleteDeadPHINode(PN: Phi);
647 }
648
649 return Changed;
650}
651
652bool RISCVGatherScatterLoweringLegacy::runOnFunction(Function &F) {
653 if (skipFunction(F))
654 return false;
655
656 auto &TPC = getAnalysis<TargetPassConfig>();
657 auto &TM = TPC.getTM<RISCVTargetMachine>();
658 auto *ST = &TM.getSubtarget<RISCVSubtarget>(F);
659 auto GetLI = [&]() -> LoopInfo & {
660 return getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
661 };
662 return RISCVGatherScatterLoweringImpl(ST, GetLI, &F.getDataLayout()).run(F);
663}
664
665PreservedAnalyses
666RISCVGatherScatterLoweringPass::run(Function &F, FunctionAnalysisManager &FAM) {
667 auto *ST = &TM->getSubtarget<RISCVSubtarget>(F);
668 auto GetLI = [&]() -> LoopInfo & { return FAM.getResult<LoopAnalysis>(IR&: F); };
669 bool Changed =
670 RISCVGatherScatterLoweringImpl(ST, GetLI, &F.getDataLayout()).run(F);
671 if (!Changed)
672 return PreservedAnalyses::all();
673
674 return PreservedAnalyses::allInSet<CFGAnalyses>();
675}
676