1//===-- RISCVInterleavedAccess.cpp - RISC-V Interleaved Access Transform --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Functions and callbacks related to the InterleavedAccessPass.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCV.h"
14#include "RISCVISelLowering.h"
15#include "RISCVSubtarget.h"
16#include "llvm/Analysis/ValueTracking.h"
17#include "llvm/Analysis/VectorUtils.h"
18#include "llvm/CodeGen/ValueTypes.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/Instructions.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/IR/Module.h"
23#include "llvm/IR/PatternMatch.h"
24
25using namespace llvm;
26
27bool RISCVTargetLowering::isLegalInterleavedAccessType(
28 VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace,
29 const DataLayout &DL) const {
30 EVT VT = getValueType(DL, Ty: VTy);
31 // Don't lower vlseg/vsseg for vector types that can't be split.
32 if (!isTypeLegal(VT))
33 return false;
34
35 if (!isLegalElementTypeForRVV(ScalarTy: VT.getScalarType()) ||
36 !allowsMemoryAccessForAlignment(Context&: VTy->getContext(), DL, VT, AddrSpace,
37 Alignment))
38 return false;
39
40 MVT ContainerVT = VT.getSimpleVT();
41
42 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: VTy)) {
43 if (!Subtarget.useRVVForFixedLengthVectors())
44 return false;
45 // Sometimes the interleaved access pass picks up splats as interleaves of
46 // one element. Don't lower these.
47 if (FVTy->getNumElements() < 2)
48 return false;
49
50 ContainerVT = getContainerForFixedLengthVector(VT: VT.getSimpleVT());
51 }
52
53 // Need to make sure that EMUL * NFIELDS ≤ 8
54 auto [LMUL, Fractional] = RISCVVType::decodeVLMUL(VLMul: getLMUL(VT: ContainerVT));
55 if (Fractional)
56 return true;
57 return Factor * LMUL <= 8;
58}
59
60static const Intrinsic::ID FixedVlsegIntrIds[] = {
61 Intrinsic::riscv_seg2_load_mask, Intrinsic::riscv_seg3_load_mask,
62 Intrinsic::riscv_seg4_load_mask, Intrinsic::riscv_seg5_load_mask,
63 Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
64 Intrinsic::riscv_seg8_load_mask};
65
66static const Intrinsic::ID FixedVlssegIntrIds[] = {
67 Intrinsic::riscv_sseg2_load_mask, Intrinsic::riscv_sseg3_load_mask,
68 Intrinsic::riscv_sseg4_load_mask, Intrinsic::riscv_sseg5_load_mask,
69 Intrinsic::riscv_sseg6_load_mask, Intrinsic::riscv_sseg7_load_mask,
70 Intrinsic::riscv_sseg8_load_mask};
71
72static const Intrinsic::ID ScalableVlsegIntrIds[] = {
73 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
74 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
75 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
76 Intrinsic::riscv_vlseg8_mask};
77
78static const Intrinsic::ID FixedVssegIntrIds[] = {
79 Intrinsic::riscv_seg2_store_mask, Intrinsic::riscv_seg3_store_mask,
80 Intrinsic::riscv_seg4_store_mask, Intrinsic::riscv_seg5_store_mask,
81 Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,
82 Intrinsic::riscv_seg8_store_mask};
83
84static const Intrinsic::ID FixedVsssegIntrIds[] = {
85 Intrinsic::riscv_sseg2_store_mask, Intrinsic::riscv_sseg3_store_mask,
86 Intrinsic::riscv_sseg4_store_mask, Intrinsic::riscv_sseg5_store_mask,
87 Intrinsic::riscv_sseg6_store_mask, Intrinsic::riscv_sseg7_store_mask,
88 Intrinsic::riscv_sseg8_store_mask};
89
90static const Intrinsic::ID ScalableVssegIntrIds[] = {
91 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
92 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
93 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
94 Intrinsic::riscv_vsseg8_mask};
95
96static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
97 assert(N);
98 if (N == 1)
99 return true;
100
101 using namespace PatternMatch;
102 // Right now we're only recognizing the simplest pattern.
103 uint64_t C;
104 if (match(V, P: m_CombineOr(L: m_ConstantInt(V&: C),
105 R: m_NUWMul(L: m_Value(), R: m_ConstantInt(V&: C)))) &&
106 C && C % N == 0)
107 return true;
108
109 if (isPowerOf2_32(Value: N)) {
110 KnownBits KB = llvm::computeKnownBits(V, DL);
111 return KB.countMinTrailingZeros() >= Log2_32(Value: N);
112 }
113
114 return false;
115}
116
117/// Do the common operand retrieval and validition required by the
118/// routines below.
119static bool getMemOperands(unsigned Factor, VectorType *VTy, Type *XLenTy,
120 Instruction *I, Value *&Ptr, Value *&Mask,
121 Value *&VL, Align &Alignment) {
122
123 IRBuilder<> Builder(I);
124 const DataLayout &DL = I->getDataLayout();
125 ElementCount EC = VTy->getElementCount();
126 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
127 assert(LI->isSimple());
128 Ptr = LI->getPointerOperand();
129 Alignment = LI->getAlign();
130 assert(!Mask && "Unexpected mask on a load");
131 Mask = Builder.getAllOnesMask(NumElts: EC);
132 VL = isa<FixedVectorType>(Val: VTy) ? Builder.CreateElementCount(Ty: XLenTy, EC)
133 : Constant::getAllOnesValue(Ty: XLenTy);
134 return true;
135 }
136 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
137 assert(SI->isSimple());
138 Ptr = SI->getPointerOperand();
139 Alignment = SI->getAlign();
140 assert(!Mask && "Unexpected mask on a store");
141 Mask = Builder.getAllOnesMask(NumElts: EC);
142 VL = isa<FixedVectorType>(Val: VTy) ? Builder.CreateElementCount(Ty: XLenTy, EC)
143 : Constant::getAllOnesValue(Ty: XLenTy);
144 return true;
145 }
146
147 auto *II = cast<IntrinsicInst>(Val: I);
148 switch (II->getIntrinsicID()) {
149 default:
150 llvm_unreachable("Unsupported intrinsic type");
151 case Intrinsic::vp_load:
152 case Intrinsic::vp_store: {
153 auto *VPLdSt = cast<VPIntrinsic>(Val: I);
154 Ptr = VPLdSt->getMemoryPointerParam();
155 Alignment = VPLdSt->getPointerAlignment().value_or(
156 u: DL.getABITypeAlign(Ty: VTy->getElementType()));
157
158 assert(Mask && "vp.load and vp.store needs a mask!");
159
160 Value *WideEVL = VPLdSt->getVectorLengthParam();
161 // Conservatively check if EVL is a multiple of factor, otherwise some
162 // (trailing) elements might be lost after the transformation.
163 if (!isMultipleOfN(V: WideEVL, DL: I->getDataLayout(), N: Factor))
164 return false;
165
166 auto *FactorC = ConstantInt::get(Ty: WideEVL->getType(), V: Factor);
167 VL = Builder.CreateZExt(V: Builder.CreateExactUDiv(LHS: WideEVL, RHS: FactorC), DestTy: XLenTy);
168 return true;
169 }
170 case Intrinsic::masked_load: {
171 Ptr = II->getOperand(i_nocapture: 0);
172 Alignment = II->getParamAlign(ArgNo: 0).valueOrOne();
173
174 if (!isa<UndefValue>(Val: II->getOperand(i_nocapture: 2)))
175 return false;
176
177 assert(Mask && "masked.load needs a mask!");
178
179 VL = isa<FixedVectorType>(Val: VTy)
180 ? Builder.CreateElementCount(Ty: XLenTy, EC: VTy->getElementCount())
181 : Constant::getAllOnesValue(Ty: XLenTy);
182 return true;
183 }
184 case Intrinsic::masked_store: {
185 Ptr = II->getOperand(i_nocapture: 1);
186 Alignment = II->getParamAlign(ArgNo: 1).valueOrOne();
187
188 assert(Mask && "masked.store needs a mask!");
189
190 VL = isa<FixedVectorType>(Val: VTy)
191 ? Builder.CreateElementCount(Ty: XLenTy, EC: VTy->getElementCount())
192 : Constant::getAllOnesValue(Ty: XLenTy);
193 return true;
194 }
195 }
196}
197
198/// Lower an interleaved load into a vlsegN intrinsic.
199///
200/// E.g. Lower an interleaved load (Factor = 2):
201/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr
202/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
203/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
204///
205/// Into:
206/// %ld2 = { <4 x i32>, <4 x i32> } call llvm.riscv.seg2.load.v4i32.p0.i64(
207/// %ptr, i64 4)
208/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
209/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
210bool RISCVTargetLowering::lowerInterleavedLoad(
211 Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
212 ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
213 assert(Indices.size() == Shuffles.size());
214 assert(GapMask.getBitWidth() == Factor);
215
216 // We only support cases where the skipped fields are the trailing ones.
217 // TODO: Lower to strided load if there is only a single active field.
218 unsigned MaskFactor = GapMask.popcount();
219 if (MaskFactor < 2 || !GapMask.isMask())
220 return false;
221 IRBuilder<> Builder(Load);
222
223 const DataLayout &DL = Load->getDataLayout();
224 auto *VTy = cast<FixedVectorType>(Val: Shuffles[0]->getType());
225 auto *XLenTy = Builder.getIntNTy(N: Subtarget.getXLen());
226
227 Value *Ptr, *VL;
228 Align Alignment;
229 if (!getMemOperands(Factor: MaskFactor, VTy, XLenTy, I: Load, Ptr, Mask, VL, Alignment))
230 return false;
231
232 Type *PtrTy = Ptr->getType();
233 unsigned AS = PtrTy->getPointerAddressSpace();
234 if (!isLegalInterleavedAccessType(VTy, Factor: MaskFactor, Alignment, AddrSpace: AS, DL))
235 return false;
236
237 CallInst *SegLoad = nullptr;
238 if (MaskFactor < Factor) {
239 // Lower to strided segmented load.
240 unsigned ScalarSizeInBytes = DL.getTypeStoreSize(Ty: VTy->getElementType());
241 Value *Stride = ConstantInt::get(Ty: XLenTy, V: Factor * ScalarSizeInBytes);
242 SegLoad = Builder.CreateIntrinsic(ID: FixedVlssegIntrIds[MaskFactor - 2],
243 Types: {VTy, PtrTy, XLenTy, XLenTy},
244 Args: {Ptr, Stride, Mask, VL});
245 } else {
246 // Lower to normal segmented load.
247 SegLoad = Builder.CreateIntrinsic(ID: FixedVlsegIntrIds[Factor - 2],
248 Types: {VTy, PtrTy, XLenTy}, Args: {Ptr, Mask, VL});
249 }
250
251 for (unsigned i = 0; i < Shuffles.size(); i++) {
252 unsigned FactorIdx = Indices[i];
253 if (FactorIdx >= MaskFactor) {
254 // Replace masked-off factors (that are still extracted) with poison.
255 Shuffles[i]->replaceAllUsesWith(V: PoisonValue::get(T: VTy));
256 } else {
257 Value *SubVec = Builder.CreateExtractValue(Agg: SegLoad, Idxs: FactorIdx);
258 Shuffles[i]->replaceAllUsesWith(V: SubVec);
259 }
260 }
261
262 return true;
263}
264
265/// Lower an interleaved store into a vssegN intrinsic.
266///
267/// E.g. Lower an interleaved store (Factor = 3):
268/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
269/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
270/// store <12 x i32> %i.vec, <12 x i32>* %ptr
271///
272/// Into:
273/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
274/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
275/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
276/// call void llvm.riscv.seg3.store.v4i32.p0.i64(%sub.v0, %sub.v1, %sub.v2,
277/// %ptr, i32 4)
278///
279/// Note that the new shufflevectors will be removed and we'll only generate one
280/// vsseg3 instruction in CodeGen.
281bool RISCVTargetLowering::lowerInterleavedStore(Instruction *Store,
282 Value *LaneMask,
283 ShuffleVectorInst *SVI,
284 unsigned Factor,
285 const APInt &GapMask) const {
286 assert(GapMask.getBitWidth() == Factor);
287
288 // We only support cases where the skipped fields are the trailing ones.
289 // TODO: Lower to strided store if there is only a single active field.
290 unsigned MaskFactor = GapMask.popcount();
291 if (MaskFactor < 2 || !GapMask.isMask())
292 return false;
293
294 IRBuilder<> Builder(Store);
295 const DataLayout &DL = Store->getDataLayout();
296 auto Mask = SVI->getShuffleMask();
297 auto *ShuffleVTy = cast<FixedVectorType>(Val: SVI->getType());
298 // Given SVI : <n*factor x ty>, then VTy : <n x ty>
299 auto *VTy = FixedVectorType::get(ElementType: ShuffleVTy->getElementType(),
300 NumElts: ShuffleVTy->getNumElements() / Factor);
301 auto *XLenTy = Builder.getIntNTy(N: Subtarget.getXLen());
302
303 Value *Ptr, *VL;
304 Align Alignment;
305 if (!getMemOperands(Factor: MaskFactor, VTy, XLenTy, I: Store, Ptr, Mask&: LaneMask, VL,
306 Alignment))
307 return false;
308
309 Type *PtrTy = Ptr->getType();
310 unsigned AS = PtrTy->getPointerAddressSpace();
311 if (!isLegalInterleavedAccessType(VTy, Factor: MaskFactor, Alignment, AddrSpace: AS, DL))
312 return false;
313
314 Function *SegStoreFunc;
315 if (MaskFactor < Factor)
316 // Strided segmented store.
317 SegStoreFunc = Intrinsic::getOrInsertDeclaration(
318 M: Store->getModule(), id: FixedVsssegIntrIds[MaskFactor - 2],
319 Tys: {VTy, PtrTy, XLenTy, XLenTy});
320 else
321 // Normal segmented store.
322 SegStoreFunc = Intrinsic::getOrInsertDeclaration(
323 M: Store->getModule(), id: FixedVssegIntrIds[Factor - 2],
324 Tys: {VTy, PtrTy, XLenTy});
325
326 SmallVector<Value *, 10> Ops;
327 SmallVector<int, 16> NewShuffleMask;
328
329 for (unsigned i = 0; i < MaskFactor; i++) {
330 // Collect shuffle mask for this lane.
331 for (unsigned j = 0; j < VTy->getNumElements(); j++)
332 NewShuffleMask.push_back(Elt: Mask[i + Factor * j]);
333
334 Value *Shuffle = Builder.CreateShuffleVector(
335 V1: SVI->getOperand(i_nocapture: 0), V2: SVI->getOperand(i_nocapture: 1), Mask: NewShuffleMask);
336 Ops.push_back(Elt: Shuffle);
337
338 NewShuffleMask.clear();
339 }
340 Ops.push_back(Elt: Ptr);
341 if (MaskFactor < Factor) {
342 // Insert the stride argument.
343 unsigned ScalarSizeInBytes = DL.getTypeStoreSize(Ty: VTy->getElementType());
344 Ops.push_back(Elt: ConstantInt::get(Ty: XLenTy, V: Factor * ScalarSizeInBytes));
345 }
346 Ops.append(IL: {LaneMask, VL});
347 Builder.CreateCall(Callee: SegStoreFunc, Args: Ops);
348
349 return true;
350}
351
352bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
353 Instruction *Load, Value *Mask, IntrinsicInst *DI) const {
354 const unsigned Factor = getDeinterleaveIntrinsicFactor(ID: DI->getIntrinsicID());
355 if (Factor > 8)
356 return false;
357
358 IRBuilder<> Builder(Load);
359
360 VectorType *ResVTy = getDeinterleavedVectorType(DI);
361
362 const DataLayout &DL = Load->getDataLayout();
363 auto *XLenTy = Builder.getIntNTy(N: Subtarget.getXLen());
364
365 Value *Ptr, *VL;
366 Align Alignment;
367 if (!getMemOperands(Factor, VTy: ResVTy, XLenTy, I: Load, Ptr, Mask, VL, Alignment))
368 return false;
369
370 Type *PtrTy = Ptr->getType();
371 unsigned AS = PtrTy->getPointerAddressSpace();
372 if (!isLegalInterleavedAccessType(VTy: ResVTy, Factor, Alignment, AddrSpace: AS, DL))
373 return false;
374
375 Value *Return;
376 if (isa<FixedVectorType>(Val: ResVTy)) {
377 Return = Builder.CreateIntrinsic(ID: FixedVlsegIntrIds[Factor - 2],
378 Types: {ResVTy, PtrTy, XLenTy}, Args: {Ptr, Mask, VL});
379 } else {
380 unsigned SEW = DL.getTypeSizeInBits(Ty: ResVTy->getElementType());
381 unsigned NumElts = ResVTy->getElementCount().getKnownMinValue();
382 Type *VecTupTy = TargetExtType::get(
383 Context&: Load->getContext(), Name: "riscv.vector.tuple",
384 Types: ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: NumElts * SEW / 8),
385 Ints: Factor);
386 Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
387 M: Load->getModule(), id: ScalableVlsegIntrIds[Factor - 2],
388 Tys: {VecTupTy, PtrTy, Mask->getType(), VL->getType()});
389
390 Value *Operands[] = {
391 PoisonValue::get(T: VecTupTy),
392 Ptr,
393 Mask,
394 VL,
395 ConstantInt::get(Ty: XLenTy,
396 V: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC),
397 ConstantInt::get(Ty: XLenTy, V: Log2_64(Value: SEW))};
398
399 CallInst *Vlseg = Builder.CreateCall(Callee: VlsegNFunc, Args: Operands);
400
401 SmallVector<Type *, 2> AggrTypes{Factor, ResVTy};
402 Return = PoisonValue::get(T: StructType::get(Context&: Load->getContext(), Elements: AggrTypes));
403 for (unsigned i = 0; i < Factor; ++i) {
404 Value *VecExtract = Builder.CreateIntrinsic(
405 ID: Intrinsic::riscv_tuple_extract, Types: {ResVTy, VecTupTy},
406 Args: {Vlseg, Builder.getInt32(C: i)});
407 Return = Builder.CreateInsertValue(Agg: Return, Val: VecExtract, Idxs: i);
408 }
409 }
410
411 DI->replaceAllUsesWith(V: Return);
412 return true;
413}
414
415bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
416 Instruction *Store, Value *Mask, ArrayRef<Value *> InterleaveValues) const {
417 unsigned Factor = InterleaveValues.size();
418 if (Factor > 8)
419 return false;
420
421 IRBuilder<> Builder(Store);
422
423 auto *InVTy = cast<VectorType>(Val: InterleaveValues[0]->getType());
424 const DataLayout &DL = Store->getDataLayout();
425 Type *XLenTy = Builder.getIntNTy(N: Subtarget.getXLen());
426
427 Value *Ptr, *VL;
428 Align Alignment;
429 if (!getMemOperands(Factor, VTy: InVTy, XLenTy, I: Store, Ptr, Mask, VL, Alignment))
430 return false;
431 Type *PtrTy = Ptr->getType();
432 unsigned AS = Ptr->getType()->getPointerAddressSpace();
433 if (!isLegalInterleavedAccessType(VTy: InVTy, Factor, Alignment, AddrSpace: AS, DL))
434 return false;
435
436 if (isa<FixedVectorType>(Val: InVTy)) {
437 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
438 M: Store->getModule(), id: FixedVssegIntrIds[Factor - 2],
439 Tys: {InVTy, PtrTy, XLenTy});
440 SmallVector<Value *, 10> Ops(InterleaveValues);
441 Ops.append(IL: {Ptr, Mask, VL});
442 Builder.CreateCall(Callee: VssegNFunc, Args: Ops);
443 return true;
444 }
445 unsigned SEW = DL.getTypeSizeInBits(Ty: InVTy->getElementType());
446 unsigned NumElts = InVTy->getElementCount().getKnownMinValue();
447 Type *VecTupTy = TargetExtType::get(
448 Context&: Store->getContext(), Name: "riscv.vector.tuple",
449 Types: ScalableVectorType::get(ElementType: Builder.getInt8Ty(), MinNumElts: NumElts * SEW / 8), Ints: Factor);
450
451 Value *StoredVal = PoisonValue::get(T: VecTupTy);
452 for (unsigned i = 0; i < Factor; ++i)
453 StoredVal = Builder.CreateIntrinsic(
454 ID: Intrinsic::riscv_tuple_insert, Types: {VecTupTy, InVTy},
455 Args: {StoredVal, InterleaveValues[i], Builder.getInt32(C: i)});
456
457 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
458 M: Store->getModule(), id: ScalableVssegIntrIds[Factor - 2],
459 Tys: {VecTupTy, PtrTy, Mask->getType(), VL->getType()});
460
461 Value *Operands[] = {StoredVal, Ptr, Mask, VL,
462 ConstantInt::get(Ty: XLenTy, V: Log2_64(Value: SEW))};
463 Builder.CreateCall(Callee: VssegNFunc, Args: Operands);
464 return true;
465}
466