1//===- RISCVGatherScatterLowering.cpp - Gather/Scatter lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass custom lowers llvm.gather and llvm.scatter instructions to
10// RISC-V intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCV.h"
15#include "RISCVTargetMachine.h"
16#include "llvm/Analysis/InstSimplifyFolder.h"
17#include "llvm/Analysis/LoopInfo.h"
18#include "llvm/Analysis/ValueTracking.h"
19#include "llvm/Analysis/VectorUtils.h"
20#include "llvm/CodeGen/TargetPassConfig.h"
21#include "llvm/IR/GetElementPtrTypeIterator.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/PatternMatch.h"
25#include "llvm/InitializePasses.h"
26#include "llvm/Pass.h"
27#include "llvm/Transforms/Utils/Local.h"
28#include <optional>
29
30using namespace llvm;
31using namespace PatternMatch;
32
33#define DEBUG_TYPE "riscv-gather-scatter-lowering"
34
35namespace {
36
37class RISCVGatherScatterLoweringImpl {
38 const RISCVSubtarget *ST;
39 const RISCVTargetLowering *TLI;
40 LoopInfo *LI;
41 const DataLayout *DL;
42
43 SmallVector<WeakTrackingVH> MaybeDeadPHIs;
44
45 // Cache of the BasePtr and Stride determined from this GEP. When a GEP is
46 // used by multiple gathers/scatters, this allow us to reuse the scalar
47 // instructions we created for the first gather/scatter for the others.
48 DenseMap<GetElementPtrInst *, std::pair<Value *, Value *>> StridedAddrs;
49
50public:
51 RISCVGatherScatterLoweringImpl(const RISCVSubtarget *ST, LoopInfo *LI,
52 const DataLayout *DL)
53 : ST(ST), TLI(ST->getTargetLowering()), LI(LI), DL(DL) {}
54
55 bool run(Function &F);
56
57private:
58 bool tryCreateStridedLoadStore(IntrinsicInst *II);
59
60 std::pair<Value *, Value *> determineBaseAndStride(Instruction *Ptr,
61 IRBuilderBase &Builder);
62
63 bool matchStridedRecurrence(Value *Index, Loop *L, Value *&Stride,
64 PHINode *&BasePtr, BinaryOperator *&Inc,
65 IRBuilderBase &Builder);
66};
67
68} // end anonymous namespace
69
70namespace {
71class RISCVGatherScatterLoweringLegacy : public FunctionPass {
72public:
73 static char ID;
74
75 RISCVGatherScatterLoweringLegacy() : FunctionPass(ID) {}
76
77 bool runOnFunction(Function &F) override;
78
79 void getAnalysisUsage(AnalysisUsage &AU) const override {
80 AU.setPreservesCFG();
81 AU.addRequired<TargetPassConfig>();
82 AU.addRequired<LoopInfoWrapperPass>();
83 }
84
85 StringRef getPassName() const override {
86 return "RISC-V gather/scatter lowering";
87 }
88};
89} // namespace
90
91char RISCVGatherScatterLoweringLegacy::ID = 0;
92
93INITIALIZE_PASS_BEGIN(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
94 "RISC-V gather/scatter lowering pass", false, false)
95INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
96INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
97INITIALIZE_PASS_END(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
98 "RISC-V gather/scatter lowering pass", false, false)
99
100FunctionPass *llvm::createRISCVGatherScatterLoweringLegacyPass() {
101 return new RISCVGatherScatterLoweringLegacy();
102}
103
104// TODO: Should we consider the mask when looking for a stride?
105static std::pair<Value *, Value *> matchStridedConstant(Constant *StartC) {
106 if (!isa<FixedVectorType>(Val: StartC->getType()))
107 return std::make_pair(x: nullptr, y: nullptr);
108
109 unsigned NumElts = cast<FixedVectorType>(Val: StartC->getType())->getNumElements();
110
111 // Check that the start value is a strided constant.
112 auto *StartVal =
113 dyn_cast_or_null<ConstantInt>(Val: StartC->getAggregateElement(Elt: (unsigned)0));
114 if (!StartVal)
115 return std::make_pair(x: nullptr, y: nullptr);
116 APInt StrideVal(StartVal->getValue().getBitWidth(), 0);
117 ConstantInt *Prev = StartVal;
118 for (unsigned i = 1; i != NumElts; ++i) {
119 auto *C = dyn_cast_or_null<ConstantInt>(Val: StartC->getAggregateElement(Elt: i));
120 if (!C)
121 return std::make_pair(x: nullptr, y: nullptr);
122
123 APInt LocalStride = C->getValue() - Prev->getValue();
124 if (i == 1)
125 StrideVal = LocalStride;
126 else if (StrideVal != LocalStride)
127 return std::make_pair(x: nullptr, y: nullptr);
128
129 Prev = C;
130 }
131
132 Value *Stride = ConstantInt::get(Ty: StartVal->getType(), V: StrideVal);
133
134 return std::make_pair(x&: StartVal, y&: Stride);
135}
136
137static std::pair<Value *, Value *> matchStridedStart(Value *Start,
138 IRBuilderBase &Builder) {
139 // Base case, start is a strided constant.
140 auto *StartC = dyn_cast<Constant>(Val: Start);
141 if (StartC)
142 return matchStridedConstant(StartC);
143
144 // Base case, start is a stepvector
145 if (match(V: Start, P: m_Intrinsic<Intrinsic::stepvector>())) {
146 auto *Ty = Start->getType()->getScalarType();
147 return std::make_pair(x: ConstantInt::get(Ty, V: 0), y: ConstantInt::get(Ty, V: 1));
148 }
149
150 // Not a constant, maybe it's a strided constant with a splat added or
151 // multiplied.
152 auto *BO = dyn_cast<BinaryOperator>(Val: Start);
153 if (!BO || (BO->getOpcode() != Instruction::Add &&
154 BO->getOpcode() != Instruction::Or &&
155 BO->getOpcode() != Instruction::Shl &&
156 BO->getOpcode() != Instruction::Mul))
157 return std::make_pair(x: nullptr, y: nullptr);
158
159 if (BO->getOpcode() == Instruction::Or &&
160 !cast<PossiblyDisjointInst>(Val: BO)->isDisjoint())
161 return std::make_pair(x: nullptr, y: nullptr);
162
163 // Look for an operand that is splatted.
164 unsigned OtherIndex = 0;
165 Value *Splat = getSplatValue(V: BO->getOperand(i_nocapture: 1));
166 if (!Splat && Instruction::isCommutative(Opcode: BO->getOpcode())) {
167 Splat = getSplatValue(V: BO->getOperand(i_nocapture: 0));
168 OtherIndex = 1;
169 }
170 if (!Splat)
171 return std::make_pair(x: nullptr, y: nullptr);
172
173 Value *Stride;
174 std::tie(args&: Start, args&: Stride) = matchStridedStart(Start: BO->getOperand(i_nocapture: OtherIndex),
175 Builder);
176 if (!Start)
177 return std::make_pair(x: nullptr, y: nullptr);
178
179 Builder.SetInsertPoint(BO);
180 Builder.SetCurrentDebugLocation(DebugLoc());
181 // Add the splat value to the start or multiply the start and stride by the
182 // splat.
183 switch (BO->getOpcode()) {
184 default:
185 llvm_unreachable("Unexpected opcode");
186 case Instruction::Or:
187 Start = Builder.CreateDisjointOr(LHS: Start, RHS: Splat);
188 break;
189 case Instruction::Add:
190 Start = Builder.CreateAdd(LHS: Start, RHS: Splat);
191 break;
192 case Instruction::Mul:
193 Start = Builder.CreateMul(LHS: Start, RHS: Splat);
194 Stride = Builder.CreateMul(LHS: Stride, RHS: Splat);
195 break;
196 case Instruction::Shl:
197 Start = Builder.CreateShl(LHS: Start, RHS: Splat);
198 Stride = Builder.CreateShl(LHS: Stride, RHS: Splat);
199 break;
200 }
201
202 return std::make_pair(x&: Start, y&: Stride);
203}
204
205// Recursively, walk about the use-def chain until we find a Phi with a strided
206// start value. Build and update a scalar recurrence as we unwind the recursion.
207// We also update the Stride as we unwind. Our goal is to move all of the
208// arithmetic out of the loop.
209bool RISCVGatherScatterLoweringImpl::matchStridedRecurrence(
210 Value *Index, Loop *L, Value *&Stride, PHINode *&BasePtr,
211 BinaryOperator *&Inc, IRBuilderBase &Builder) {
212 // Our base case is a Phi.
213 if (auto *Phi = dyn_cast<PHINode>(Val: Index)) {
214 // A phi node we want to perform this function on should be from the
215 // loop header.
216 if (Phi->getParent() != L->getHeader())
217 return false;
218
219 Value *Step, *Start;
220 if (!matchSimpleRecurrence(P: Phi, BO&: Inc, Start, Step) ||
221 Inc->getOpcode() != Instruction::Add)
222 return false;
223 assert(Phi->getNumIncomingValues() == 2 && "Expected 2 operand phi.");
224 unsigned IncrementingBlock = Phi->getIncomingValue(i: 0) == Inc ? 0 : 1;
225 assert(Phi->getIncomingValue(IncrementingBlock) == Inc &&
226 "Expected one operand of phi to be Inc");
227
228 // Step should be a splat.
229 Step = getSplatValue(V: Step);
230 if (!Step)
231 return false;
232
233 std::tie(args&: Start, args&: Stride) = matchStridedStart(Start, Builder);
234 if (!Start)
235 return false;
236 assert(Stride != nullptr);
237
238 // Build scalar phi and increment.
239 BasePtr =
240 PHINode::Create(Ty: Start->getType(), NumReservedValues: 2, NameStr: Phi->getName() + ".scalar", InsertBefore: Phi->getIterator());
241 Inc = BinaryOperator::CreateAdd(V1: BasePtr, V2: Step, Name: Inc->getName() + ".scalar",
242 InsertBefore: Inc->getIterator());
243 BasePtr->addIncoming(V: Start, BB: Phi->getIncomingBlock(i: 1 - IncrementingBlock));
244 BasePtr->addIncoming(V: Inc, BB: Phi->getIncomingBlock(i: IncrementingBlock));
245
246 // Note that this Phi might be eligible for removal.
247 MaybeDeadPHIs.push_back(Elt: Phi);
248 return true;
249 }
250
251 // Otherwise look for binary operator.
252 auto *BO = dyn_cast<BinaryOperator>(Val: Index);
253 if (!BO)
254 return false;
255
256 switch (BO->getOpcode()) {
257 default:
258 return false;
259 case Instruction::Or:
260 // We need to be able to treat Or as Add.
261 if (!cast<PossiblyDisjointInst>(Val: BO)->isDisjoint())
262 return false;
263 break;
264 case Instruction::Add:
265 break;
266 case Instruction::Shl:
267 break;
268 case Instruction::Mul:
269 break;
270 }
271
272 // We should have one operand in the loop and one splat.
273 Value *OtherOp;
274 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 0)) &&
275 L->contains(Inst: cast<Instruction>(Val: BO->getOperand(i_nocapture: 0)))) {
276 Index = cast<Instruction>(Val: BO->getOperand(i_nocapture: 0));
277 OtherOp = BO->getOperand(i_nocapture: 1);
278 } else if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 1)) &&
279 L->contains(Inst: cast<Instruction>(Val: BO->getOperand(i_nocapture: 1))) &&
280 Instruction::isCommutative(Opcode: BO->getOpcode())) {
281 Index = cast<Instruction>(Val: BO->getOperand(i_nocapture: 1));
282 OtherOp = BO->getOperand(i_nocapture: 0);
283 } else {
284 return false;
285 }
286
287 // Make sure other op is loop invariant.
288 if (!L->isLoopInvariant(V: OtherOp))
289 return false;
290
291 // Make sure we have a splat.
292 Value *SplatOp = getSplatValue(V: OtherOp);
293 if (!SplatOp)
294 return false;
295
296 // Recurse up the use-def chain.
297 if (!matchStridedRecurrence(Index, L, Stride, BasePtr, Inc, Builder))
298 return false;
299
300 // Locate the Step and Start values from the recurrence.
301 unsigned StepIndex = Inc->getOperand(i_nocapture: 0) == BasePtr ? 1 : 0;
302 unsigned StartBlock = BasePtr->getOperand(i_nocapture: 0) == Inc ? 1 : 0;
303 Value *Step = Inc->getOperand(i_nocapture: StepIndex);
304 Value *Start = BasePtr->getOperand(i_nocapture: StartBlock);
305
306 // We need to adjust the start value in the preheader.
307 Builder.SetInsertPoint(
308 BasePtr->getIncomingBlock(i: StartBlock)->getTerminator());
309 Builder.SetCurrentDebugLocation(DebugLoc());
310
311 // TODO: Share this switch with matchStridedStart?
312 switch (BO->getOpcode()) {
313 default:
314 llvm_unreachable("Unexpected opcode!");
315 case Instruction::Add:
316 case Instruction::Or: {
317 // An add only affects the start value. It's ok to do this for Or because
318 // we already checked that there are no common set bits.
319 Start = Builder.CreateAdd(LHS: Start, RHS: SplatOp, Name: "start");
320 break;
321 }
322 case Instruction::Mul: {
323 Start = Builder.CreateMul(LHS: Start, RHS: SplatOp, Name: "start");
324 Stride = Builder.CreateMul(LHS: Stride, RHS: SplatOp, Name: "stride");
325 break;
326 }
327 case Instruction::Shl: {
328 Start = Builder.CreateShl(LHS: Start, RHS: SplatOp, Name: "start");
329 Stride = Builder.CreateShl(LHS: Stride, RHS: SplatOp, Name: "stride");
330 break;
331 }
332 }
333
334 // If the Step was defined inside the loop, adjust it before its definition
335 // instead of in the preheader.
336 if (auto *StepI = dyn_cast<Instruction>(Val: Step); StepI && L->contains(Inst: StepI))
337 Builder.SetInsertPoint(*StepI->getInsertionPointAfterDef());
338
339 switch (BO->getOpcode()) {
340 default:
341 break;
342 case Instruction::Mul:
343 Step = Builder.CreateMul(LHS: Step, RHS: SplatOp, Name: "step");
344 break;
345 case Instruction::Shl:
346 Step = Builder.CreateShl(LHS: Step, RHS: SplatOp, Name: "step");
347 break;
348 }
349
350 Inc->setOperand(i_nocapture: StepIndex, Val_nocapture: Step);
351 BasePtr->setIncomingValue(i: StartBlock, V: Start);
352 return true;
353}
354
355std::pair<Value *, Value *>
356RISCVGatherScatterLoweringImpl::determineBaseAndStride(Instruction *Ptr,
357 IRBuilderBase &Builder) {
358
359 // A gather/scatter of a splat is a zero strided load/store.
360 if (auto *BasePtr = getSplatValue(V: Ptr)) {
361 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
362 return std::make_pair(x&: BasePtr, y: ConstantInt::get(Ty: IntPtrTy, V: 0));
363 }
364
365 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
366 if (!GEP)
367 return std::make_pair(x: nullptr, y: nullptr);
368
369 auto I = StridedAddrs.find(Val: GEP);
370 if (I != StridedAddrs.end())
371 return I->second;
372
373 SmallVector<Value *, 2> Ops(GEP->operands());
374
375 // If the base pointer is a vector, check if it's strided.
376 Value *Base = GEP->getPointerOperand();
377 if (auto *BaseInst = dyn_cast<Instruction>(Val: Base);
378 BaseInst && BaseInst->getType()->isVectorTy()) {
379 // If GEP's offset is scalar then we can add it to the base pointer's base.
380 auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); };
381 if (all_of(Range: GEP->indices(), P: IsScalar)) {
382 auto [BaseBase, Stride] = determineBaseAndStride(Ptr: BaseInst, Builder);
383 if (BaseBase) {
384 Builder.SetInsertPoint(GEP);
385 SmallVector<Value *> Indices(GEP->indices());
386 Value *OffsetBase =
387 Builder.CreateGEP(Ty: GEP->getSourceElementType(), Ptr: BaseBase, IdxList: Indices,
388 Name: GEP->getName() + "offset", NW: GEP->isInBounds());
389 return {OffsetBase, Stride};
390 }
391 }
392 }
393
394 // Base pointer needs to be a scalar.
395 Value *ScalarBase = Base;
396 if (ScalarBase->getType()->isVectorTy()) {
397 ScalarBase = getSplatValue(V: ScalarBase);
398 if (!ScalarBase)
399 return std::make_pair(x: nullptr, y: nullptr);
400 }
401
402 std::optional<unsigned> VecOperand;
403 unsigned TypeScale = 0;
404
405 // Look for a vector operand and scale.
406 gep_type_iterator GTI = gep_type_begin(GEP);
407 for (unsigned i = 1, e = GEP->getNumOperands(); i != e; ++i, ++GTI) {
408 if (!Ops[i]->getType()->isVectorTy())
409 continue;
410
411 if (VecOperand)
412 return std::make_pair(x: nullptr, y: nullptr);
413
414 VecOperand = i;
415
416 TypeSize TS = GTI.getSequentialElementStride(DL: *DL);
417 if (TS.isScalable())
418 return std::make_pair(x: nullptr, y: nullptr);
419
420 TypeScale = TS.getFixedValue();
421 }
422
423 // We need to find a vector index to simplify.
424 if (!VecOperand)
425 return std::make_pair(x: nullptr, y: nullptr);
426
427 // We can't extract the stride if the arithmetic is done at a different size
428 // than the pointer type. Adding the stride later may not wrap correctly.
429 // Technically we could handle wider indices, but I don't expect that in
430 // practice. Handle one special case here - constants. This simplifies
431 // writing test cases.
432 Value *VecIndex = Ops[*VecOperand];
433 Type *VecIntPtrTy = DL->getIntPtrType(GEP->getType());
434 if (VecIndex->getType() != VecIntPtrTy) {
435 auto *VecIndexC = dyn_cast<Constant>(Val: VecIndex);
436 if (!VecIndexC)
437 return std::make_pair(x: nullptr, y: nullptr);
438 if (VecIndex->getType()->getScalarSizeInBits() > VecIntPtrTy->getScalarSizeInBits())
439 VecIndex = ConstantFoldCastInstruction(opcode: Instruction::Trunc, V: VecIndexC, DestTy: VecIntPtrTy);
440 else
441 VecIndex = ConstantFoldCastInstruction(opcode: Instruction::SExt, V: VecIndexC, DestTy: VecIntPtrTy);
442 }
443
444 // Handle the non-recursive case. This is what we see if the vectorizer
445 // decides to use a scalar IV + vid on demand instead of a vector IV.
446 auto [Start, Stride] = matchStridedStart(Start: VecIndex, Builder);
447 if (Start) {
448 assert(Stride);
449 Builder.SetInsertPoint(GEP);
450
451 // Replace the vector index with the scalar start and build a scalar GEP.
452 Ops[*VecOperand] = Start;
453 Type *SourceTy = GEP->getSourceElementType();
454 Value *BasePtr =
455 Builder.CreateGEP(Ty: SourceTy, Ptr: ScalarBase, IdxList: ArrayRef(Ops).drop_front());
456
457 // Convert stride to pointer size if needed.
458 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
459 assert(Stride->getType() == IntPtrTy && "Unexpected type");
460
461 // Scale the stride by the size of the indexed type.
462 if (TypeScale != 1)
463 Stride = Builder.CreateMul(LHS: Stride, RHS: ConstantInt::get(Ty: IntPtrTy, V: TypeScale));
464
465 auto P = std::make_pair(x&: BasePtr, y&: Stride);
466 StridedAddrs[GEP] = P;
467 return P;
468 }
469
470 // Make sure we're in a loop and that has a pre-header and a single latch.
471 Loop *L = LI->getLoopFor(BB: GEP->getParent());
472 if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
473 return std::make_pair(x: nullptr, y: nullptr);
474
475 BinaryOperator *Inc;
476 PHINode *BasePhi;
477 if (!matchStridedRecurrence(Index: VecIndex, L, Stride, BasePtr&: BasePhi, Inc, Builder))
478 return std::make_pair(x: nullptr, y: nullptr);
479
480 assert(BasePhi->getNumIncomingValues() == 2 && "Expected 2 operand phi.");
481 unsigned IncrementingBlock = BasePhi->getOperand(i_nocapture: 0) == Inc ? 0 : 1;
482 assert(BasePhi->getIncomingValue(IncrementingBlock) == Inc &&
483 "Expected one operand of phi to be Inc");
484
485 Builder.SetInsertPoint(GEP);
486
487 // Replace the vector index with the scalar phi and build a scalar GEP.
488 Ops[*VecOperand] = BasePhi;
489 Type *SourceTy = GEP->getSourceElementType();
490 Value *BasePtr =
491 Builder.CreateGEP(Ty: SourceTy, Ptr: ScalarBase, IdxList: ArrayRef(Ops).drop_front());
492
493 // Final adjustments to stride should go in the start block.
494 Builder.SetInsertPoint(
495 BasePhi->getIncomingBlock(i: 1 - IncrementingBlock)->getTerminator());
496
497 // Convert stride to pointer size if needed.
498 Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
499 assert(Stride->getType() == IntPtrTy && "Unexpected type");
500
501 // Scale the stride by the size of the indexed type.
502 if (TypeScale != 1)
503 Stride = Builder.CreateMul(LHS: Stride, RHS: ConstantInt::get(Ty: IntPtrTy, V: TypeScale));
504
505 auto P = std::make_pair(x&: BasePtr, y&: Stride);
506 StridedAddrs[GEP] = P;
507 return P;
508}
509
510bool RISCVGatherScatterLoweringImpl::tryCreateStridedLoadStore(
511 IntrinsicInst *II) {
512 VectorType *DataType;
513 Value *StoreVal = nullptr, *Ptr, *Mask, *EVL = nullptr;
514 Align Alignment;
515 switch (II->getIntrinsicID()) {
516 case Intrinsic::masked_gather:
517 DataType = cast<VectorType>(Val: II->getType());
518 Ptr = II->getArgOperand(i: 0);
519 Alignment = II->getParamAlign(ArgNo: 0).valueOrOne();
520 Mask = II->getArgOperand(i: 1);
521 break;
522 case Intrinsic::vp_gather:
523 DataType = cast<VectorType>(Val: II->getType());
524 Ptr = II->getArgOperand(i: 0);
525 // FIXME: Falling back to ABI alignment is incorrect.
526 Alignment = II->getParamAlign(ArgNo: 0).value_or(
527 u: DL->getABITypeAlign(Ty: DataType->getElementType()));
528 Mask = II->getArgOperand(i: 1);
529 EVL = II->getArgOperand(i: 2);
530 break;
531 case Intrinsic::masked_scatter:
532 DataType = cast<VectorType>(Val: II->getArgOperand(i: 0)->getType());
533 StoreVal = II->getArgOperand(i: 0);
534 Ptr = II->getArgOperand(i: 1);
535 Alignment = II->getParamAlign(ArgNo: 1).valueOrOne();
536 Mask = II->getArgOperand(i: 2);
537 break;
538 case Intrinsic::vp_scatter:
539 DataType = cast<VectorType>(Val: II->getArgOperand(i: 0)->getType());
540 StoreVal = II->getArgOperand(i: 0);
541 Ptr = II->getArgOperand(i: 1);
542 // FIXME: Falling back to ABI alignment is incorrect.
543 Alignment = II->getParamAlign(ArgNo: 1).value_or(
544 u: DL->getABITypeAlign(Ty: DataType->getElementType()));
545 Mask = II->getArgOperand(i: 2);
546 EVL = II->getArgOperand(i: 3);
547 break;
548 default:
549 llvm_unreachable("Unexpected intrinsic");
550 }
551
552 // Make sure the operation will be supported by the backend.
553 EVT DataTypeVT = TLI->getValueType(DL: *DL, Ty: DataType);
554 if (!TLI->isLegalStridedLoadStore(DataType: DataTypeVT, Alignment))
555 return false;
556
557 // FIXME: Let the backend type legalize by splitting/widening?
558 if (!TLI->isTypeLegal(VT: DataTypeVT))
559 return false;
560
561 // Pointer should be an instruction.
562 auto *PtrI = dyn_cast<Instruction>(Val: Ptr);
563 if (!PtrI)
564 return false;
565
566 LLVMContext &Ctx = PtrI->getContext();
567 IRBuilder Builder(Ctx, InstSimplifyFolder(*DL));
568 Builder.SetInsertPoint(PtrI);
569
570 Value *BasePtr, *Stride;
571 std::tie(args&: BasePtr, args&: Stride) = determineBaseAndStride(Ptr: PtrI, Builder);
572 if (!BasePtr)
573 return false;
574 assert(Stride != nullptr);
575
576 Builder.SetInsertPoint(II);
577
578 if (!EVL)
579 EVL = Builder.CreateElementCount(
580 Ty: Builder.getInt32Ty(), EC: cast<VectorType>(Val: DataType)->getElementCount());
581
582 Value *Call;
583
584 if (!StoreVal) {
585 Call = Builder.CreateIntrinsic(
586 ID: Intrinsic::experimental_vp_strided_load,
587 OverloadTypes: {DataType, BasePtr->getType(), Stride->getType()},
588 Args: {BasePtr, Stride, Mask, EVL});
589
590 // Merge llvm.masked.gather's passthru
591 if (II->getIntrinsicID() == Intrinsic::masked_gather)
592 Call = Builder.CreateSelect(C: Mask, True: Call, False: II->getArgOperand(i: 2));
593 } else
594 Call = Builder.CreateIntrinsic(
595 ID: Intrinsic::experimental_vp_strided_store,
596 OverloadTypes: {DataType, BasePtr->getType(), Stride->getType()},
597 Args: {StoreVal, BasePtr, Stride, Mask, EVL});
598
599 Call->takeName(V: II);
600 II->replaceAllUsesWith(V: Call);
601 II->eraseFromParent();
602
603 if (PtrI->use_empty())
604 RecursivelyDeleteTriviallyDeadInstructions(V: PtrI);
605
606 return true;
607}
608
609bool RISCVGatherScatterLoweringImpl::run(Function &F) {
610 if (!ST->hasVInstructions() || !ST->useRVVForFixedLengthVectors())
611 return false;
612
613 SmallVector<IntrinsicInst *, 4> Worklist;
614
615 bool Changed = false;
616
617 for (BasicBlock &BB : F) {
618 for (Instruction &I : BB) {
619 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I);
620 if (!II)
621 continue;
622 switch (II->getIntrinsicID()) {
623 case Intrinsic::masked_gather:
624 case Intrinsic::masked_scatter:
625 case Intrinsic::vp_gather:
626 case Intrinsic::vp_scatter:
627 Worklist.push_back(Elt: II);
628 break;
629 default:
630 break;
631 }
632 }
633 }
634
635 // Rewrite gather/scatter to form strided load/store if possible.
636 for (auto *II : Worklist)
637 Changed |= tryCreateStridedLoadStore(II);
638
639 // Remove any dead phis.
640 while (!MaybeDeadPHIs.empty()) {
641 if (auto *Phi = dyn_cast_or_null<PHINode>(Val: MaybeDeadPHIs.pop_back_val()))
642 RecursivelyDeleteDeadPHINode(PN: Phi);
643 }
644
645 return Changed;
646}
647
648bool RISCVGatherScatterLoweringLegacy::runOnFunction(Function &F) {
649 if (skipFunction(F))
650 return false;
651
652 auto &TPC = getAnalysis<TargetPassConfig>();
653 auto &TM = TPC.getTM<RISCVTargetMachine>();
654 auto *ST = &TM.getSubtarget<RISCVSubtarget>(F);
655 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
656 return RISCVGatherScatterLoweringImpl(ST, LI, &F.getDataLayout()).run(F);
657}
658
659PreservedAnalyses
660RISCVGatherScatterLoweringPass::run(Function &F, FunctionAnalysisManager &FAM) {
661 auto *ST = &TM->getSubtarget<RISCVSubtarget>(F);
662 auto *LI = &FAM.getResult<LoopAnalysis>(IR&: F);
663 bool Changed =
664 RISCVGatherScatterLoweringImpl(ST, LI, &F.getDataLayout()).run(F);
665 if (!Changed)
666 return PreservedAnalyses::all();
667
668 return PreservedAnalyses::allInSet<CFGAnalyses>();
669}
670