1//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V.
10// This pass modifies such loads to have an IR we can directly lower to valid
11// logical SPIR-V.
12// OpenCL can avoid this because they rely on ptrcast, which is not supported
13// by logical SPIR-V.
14//
15// This pass relies on the assign_ptr_type intrinsic to deduce the type of the
16// pointed values, must replace all occurences of `ptrcast`. This is why
17// unhandled cases are reported as unreachable: we MUST cover all cases.
18//
19// 1. Loading the first element of an array
20//
21// %array = [10 x i32]
22// %value = load i32, ptr %array
23//
24// LLVM can skip the GEP instruction, and only request loading the first 4
25// bytes. In logical SPIR-V, we need an OpAccessChain to access the first
26// element. This pass will add a getelementptr instruction before the load.
27//
28//
29// 2. Implicit downcast from load
30//
31// %1 = getelementptr <4 x i32>, ptr %vec4, i64 0
32// %2 = load <3 x i32>, ptr %1
33//
34// The pointer in the GEP instruction is only used for offset computations,
35// but it doesn't NEED to match the pointed type. OpAccessChain however
36// requires this. Also, LLVM loads define the bitwidth of the load, not the
37// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP
38// instruction basetype, but we only want to load the first 3 elements, hence
39// do a partial load. In logical SPIR-V, this is not legal. What we must do
40// is load the full vector (basetype), extract 3 elements, and recombine them
41// to form a 3-element vector.
42//
43//===----------------------------------------------------------------------===//
44
45#include "SPIRVLegalizePointerCast.h"
46#include "SPIRV.h"
47#include "SPIRVSubtarget.h"
48#include "SPIRVTargetMachine.h"
49#include "SPIRVUtils.h"
50#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/IntrinsicInst.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/IntrinsicsSPIRV.h"
54#include "llvm/Transforms/Utils/Cloning.h"
55#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
56
57using namespace llvm;
58
59namespace {
60class SPIRVLegalizePointerCastImpl {
61
62 // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current
63 // builder position.
64 void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) {
65 Value *OfType = PoisonValue::get(T: Ty);
66 CallInst *AssignCI = buildIntrWithMD(IntrID: Intrinsic::spv_assign_type,
67 Types: {Arg->getType()}, Arg: OfType, Arg2: Arg, Imms: {}, B);
68 GR->addAssignPtrTypeInstr(Val: Arg, AssignPtrTyCI: AssignCI);
69 }
70
71 static FixedVectorType *makeVectorFromTotalBits(Type *ElemTy,
72 TypeSize TotalBits) {
73 unsigned ElemBits = ElemTy->getScalarSizeInBits();
74 assert(ElemBits && TotalBits % ElemBits == 0 &&
75 "TotalBits must be divisible by element bit size");
76 return FixedVectorType::get(ElementType: ElemTy, NumElts: TotalBits / ElemBits);
77 }
78
79 Value *resizeVectorBitsWithShuffle(IRBuilder<> &B, Value *V,
80 FixedVectorType *DstTy) {
81 auto *SrcTy = cast<FixedVectorType>(Val: V->getType());
82 assert(SrcTy->getElementType() == DstTy->getElementType() &&
83 "shuffle resize expects identical element types");
84
85 const unsigned NumNeeded = DstTy->getNumElements();
86 const unsigned NumSource = SrcTy->getNumElements();
87
88 SmallVector<int> Mask(NumNeeded);
89 for (unsigned I = 0; I < NumNeeded; ++I)
90 Mask[I] = (I < NumSource) ? static_cast<int>(I) : -1;
91
92 Value *Resized = B.CreateShuffleVector(V1: V, V2: V, Mask);
93 buildAssignType(B, Ty: DstTy, Arg: Resized);
94 return Resized;
95 }
96
97 // Loads parts of the vector of type |SourceType| from the pointer |Source|
98 // and create a new vector of type |TargetType|. |TargetType| must be a vector
99 // type.
100 // Returns the loaded value.
101 Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType,
102 FixedVectorType *TargetType, Value *Source) {
103 LoadInst *NewLoad = B.CreateLoad(Ty: SourceType, Ptr: Source);
104 buildAssignType(B, Ty: SourceType, Arg: NewLoad);
105 Value *AssignValue = NewLoad;
106 if (TargetType->getElementType() != SourceType->getElementType()) {
107 const DataLayout &DL = B.GetInsertBlock()->getModule()->getDataLayout();
108 TypeSize TargetTypeSize = DL.getTypeSizeInBits(Ty: TargetType);
109 TypeSize SourceTypeSize = DL.getTypeSizeInBits(Ty: SourceType);
110
111 Value *BitcastSrcVal = NewLoad;
112 FixedVectorType *BitcastSrcTy =
113 cast<FixedVectorType>(Val: BitcastSrcVal->getType());
114 FixedVectorType *BitcastDstTy = TargetType;
115
116 if (TargetTypeSize != SourceTypeSize) {
117 unsigned TargetElemBits =
118 TargetType->getElementType()->getScalarSizeInBits();
119 if (SourceTypeSize % TargetElemBits == 0) {
120 // No Resize needed. Same total bits as source, but use target element
121 // type.
122 BitcastDstTy = makeVectorFromTotalBits(ElemTy: TargetType->getElementType(),
123 TotalBits: SourceTypeSize);
124 } else {
125 // Resize source to target total bitwidth using source element type.
126 BitcastSrcTy = makeVectorFromTotalBits(ElemTy: SourceType->getElementType(),
127 TotalBits: TargetTypeSize);
128 BitcastSrcVal = resizeVectorBitsWithShuffle(B, V: NewLoad, DstTy: BitcastSrcTy);
129 }
130 }
131 AssignValue =
132 B.CreateIntrinsic(ID: Intrinsic::spv_bitcast,
133 OverloadTypes: {BitcastDstTy, BitcastSrcTy}, Args: {BitcastSrcVal});
134 buildAssignType(B, Ty: BitcastDstTy, Arg: AssignValue);
135 if (BitcastDstTy == TargetType)
136 return AssignValue;
137 }
138
139 assert(TargetType->getNumElements() < SourceType->getNumElements());
140 SmallVector<int> Mask(/* Size= */ TargetType->getNumElements());
141 for (unsigned I = 0; I < TargetType->getNumElements(); ++I)
142 Mask[I] = I;
143 Value *Output = B.CreateShuffleVector(V1: AssignValue, V2: AssignValue, Mask);
144 buildAssignType(B, Ty: TargetType, Arg: Output);
145 return Output;
146 }
147
148 // Returns true if |FromTy| has a memory layout compatible with loading or
149 // storing |ToTy|.
150 bool isCompatibleMemoryLayout(Type *ToTy, Type *FromTy) {
151 if (ToTy == FromTy)
152 return true;
153 auto *SVT = dyn_cast<FixedVectorType>(Val: FromTy);
154 auto *DVT = dyn_cast<FixedVectorType>(Val: ToTy);
155 if (SVT && DVT)
156 return true;
157 auto *SAT = dyn_cast<ArrayType>(Val: FromTy);
158 if (SAT && DVT) {
159 if (SAT->getElementType() == DVT->getElementType())
160 return true;
161 if (auto *MAT = dyn_cast<FixedVectorType>(Val: SAT->getElementType()))
162 if (MAT->getElementType() == DVT->getElementType())
163 return true;
164 }
165 return false;
166 }
167
168 // Traverses the aggregate type to find the first sub-type that matches
169 // the TargetElemType's memory layout, optionally emitting a GEP intrinsic.
170 std::optional<std::pair<Value *, Type *>>
171 getPointerToFirstCompatibleType(IRBuilder<> &B, Value *BasePtr,
172 Type *PointerType, Type *TargetElemType,
173 bool IsInBounds) {
174 Type *CurrentTy = GR->findDeducedElementType(Val: BasePtr);
175 assert(CurrentTy && "Could not deduce aggregate type");
176 SmallVector<Value *, 8> Args{/* isInBounds= */ B.getInt1(V: IsInBounds),
177 BasePtr};
178 Args.push_back(Elt: B.getInt32(C: 0)); // Pointer offset
179
180 while (!isCompatibleMemoryLayout(ToTy: TargetElemType, FromTy: CurrentTy)) {
181 if (auto *ST = dyn_cast<StructType>(Val: CurrentTy)) {
182 if (ST->getNumElements() == 0)
183 return std::nullopt;
184 CurrentTy = ST->getTypeAtIndex(N: 0u);
185 } else if (auto *AT = dyn_cast<ArrayType>(Val: CurrentTy)) {
186 CurrentTy = AT->getElementType();
187 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurrentTy)) {
188 CurrentTy = VT->getElementType();
189 } else {
190 return std::nullopt;
191 }
192 Args.push_back(Elt: B.getInt32(C: 0));
193 }
194
195 Value *GEP = BasePtr;
196 if (Args.size() > 3) {
197 std::array<Type *, 2> Types = {PointerType, BasePtr->getType()};
198 GEP = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
199 GR->buildAssignPtr(B, ElemTy: CurrentTy, Arg: GEP);
200 }
201
202 return std::make_pair(x&: GEP, y&: CurrentTy);
203 }
204
205 // Builds a legalized load from a pointer, drilling down through
206 // memory layouts to find a compatible type. Load flags will be
207 // copied from |BadLoad|, which should be the load being legalized.
208 Value *buildLegalizedLoad(IRBuilder<> &B, Type *ElementType, Value *Source,
209 LoadInst *BadLoad) {
210 auto ResultOpt = getPointerToFirstCompatibleType(
211 B, BasePtr: Source, PointerType: BadLoad->getPointerOperandType(), TargetElemType: ElementType, IsInBounds: false);
212 assert(ResultOpt && "Failed to load from aggregate: "
213 "Could not find compatible memory layout.");
214 auto [GEP, CurrentTy] = *ResultOpt;
215
216 auto *SAT = dyn_cast<ArrayType>(Val: CurrentTy);
217 auto *SVT = dyn_cast<FixedVectorType>(Val: CurrentTy);
218 auto *DVT = dyn_cast<FixedVectorType>(Val: ElementType);
219 auto *MAT =
220 SAT ? dyn_cast<FixedVectorType>(Val: SAT->getElementType()) : nullptr;
221
222 if (ElementType == CurrentTy) {
223 LoadInst *LI = B.CreateLoad(Ty: ElementType, Ptr: GEP);
224 LI->setAlignment(BadLoad->getAlign());
225 buildAssignType(B, Ty: ElementType, Arg: LI);
226 return LI;
227 }
228 if (SVT && DVT)
229 return loadVectorFromVector(B, SourceType: SVT, TargetType: DVT, Source: GEP);
230 if (SAT && DVT && SAT->getElementType() == DVT->getElementType())
231 return loadVectorFromArray(B, TargetType: DVT, Source: GEP);
232 if (MAT && DVT && MAT->getElementType() == DVT->getElementType())
233 return loadVectorFromMatrixArray(B, TargetType: DVT, Source: GEP, ArrElemVecTy: MAT);
234
235 llvm_unreachable("Failed to load from aggregate.");
236 }
237 Value *
238 buildVectorFromLoadedElements(IRBuilder<> &B, FixedVectorType *TargetType,
239 SmallVector<Value *, 4> &LoadedElements) {
240 // <1 x T> shares the SPIR-V type with T, so emitting OpCompositeInsert on
241 // a scalar would be invalid. Bridge with spv_bitcast instead.
242 if (TargetType->getNumElements() == 1) {
243 Value *Scalar = LoadedElements[0];
244 Value *NewVector = B.CreateIntrinsic(
245 ID: Intrinsic::spv_bitcast, OverloadTypes: {TargetType, Scalar->getType()}, Args: {Scalar});
246 buildAssignType(B, Ty: TargetType, Arg: NewVector);
247 return NewVector;
248 }
249
250 // Build the vector from the loaded elements.
251 Value *NewVector = PoisonValue::get(T: TargetType);
252 buildAssignType(B, Ty: TargetType, Arg: NewVector);
253
254 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
255 Value *Index = B.getInt32(C: I);
256 SmallVector<Type *, 4> Types = {TargetType, TargetType,
257 TargetType->getElementType(),
258 Index->getType()};
259 SmallVector<Value *> Args = {NewVector, LoadedElements[I], Index};
260 NewVector = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, OverloadTypes: {Types}, Args: {Args});
261 buildAssignType(B, Ty: TargetType, Arg: NewVector);
262 }
263 return NewVector;
264 }
265
266 // Loads elements from a matrix with an array of vector memory layout and
267 // constructs a vector.
268 Value *loadVectorFromMatrixArray(IRBuilder<> &B, FixedVectorType *TargetType,
269 Value *Source,
270 FixedVectorType *ArrElemVecTy) {
271 Type *TargetElemTy = TargetType->getElementType();
272 unsigned ScalarsPerArrayElement = ArrElemVecTy->getNumElements();
273 // Load each element of the array.
274 SmallVector<Value *, 4> LoadedElements;
275 std::array<Type *, 2> Types = {Source->getType(), Source->getType()};
276 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
277 unsigned ArrayIndex = I / ScalarsPerArrayElement;
278 unsigned ElementIndexInArrayElem = I % ScalarsPerArrayElement;
279 // Create a GEP to access the i-th element of the array.
280 std::array<Value *, 4> Args = {
281 B.getInt1(/*Inbounds=*/V: false), Source, B.getInt32(C: 0),
282 ConstantInt::get(Ty: B.getInt32Ty(), V: ArrayIndex)};
283 auto *ElementPtr = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
284 GR->buildAssignPtr(B, ElemTy: ArrElemVecTy, Arg: ElementPtr);
285 Value *LoadVec = B.CreateLoad(Ty: ArrElemVecTy, Ptr: ElementPtr);
286 buildAssignType(B, Ty: ArrElemVecTy, Arg: LoadVec);
287 LoadedElements.push_back(Elt: makeExtractElement(B, ElementType: TargetElemTy, Vector: LoadVec,
288 Index: ElementIndexInArrayElem));
289 }
290 return buildVectorFromLoadedElements(B, TargetType, LoadedElements);
291 }
292 // Loads elements from an array and constructs a vector.
293 Value *loadVectorFromArray(IRBuilder<> &B, FixedVectorType *TargetType,
294 Value *Source) {
295 // Load each element of the array.
296 SmallVector<Value *, 4> LoadedElements;
297 std::array<Type *, 2> Types = {Source->getType(), Source->getType()};
298 for (unsigned I = 0, E = TargetType->getNumElements(); I < E; ++I) {
299 // Create a GEP to access the i-th element of the array.
300 std::array<Value *, 4> Args = {B.getInt1(/*Inbounds=*/V: false), Source,
301 B.getInt32(C: 0),
302 ConstantInt::get(Ty: B.getInt32Ty(), V: I)};
303 auto *ElementPtr = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
304 GR->buildAssignPtr(B, ElemTy: TargetType->getElementType(), Arg: ElementPtr);
305
306 // Load the value from the element pointer.
307 Value *Load = B.CreateLoad(Ty: TargetType->getElementType(), Ptr: ElementPtr);
308 buildAssignType(B, Ty: TargetType->getElementType(), Arg: Load);
309 LoadedElements.push_back(Elt: Load);
310 }
311 return buildVectorFromLoadedElements(B, TargetType, LoadedElements);
312 }
313
314 // Stores elements from a vector into a matrix (an array of vectors).
315 void storeMatrixArrayFromVector(IRBuilder<> &B, Value *SrcVector,
316 Value *DstArrayPtr, ArrayType *ArrTy,
317 Align Alignment) {
318 auto *SrcVecTy = cast<FixedVectorType>(Val: SrcVector->getType());
319 auto *ArrElemVecTy = cast<FixedVectorType>(Val: ArrTy->getElementType());
320 Type *ElemTy = ArrElemVecTy->getElementType();
321 unsigned ScalarsPerArrayElement = ArrElemVecTy->getNumElements();
322 unsigned SrcNumElements = SrcVecTy->getNumElements();
323 assert(
324 SrcNumElements % ScalarsPerArrayElement == 0 &&
325 "Source vector size must be a multiple of array element vector size");
326
327 std::array<Type *, 2> Types = {DstArrayPtr->getType(),
328 DstArrayPtr->getType()};
329
330 for (unsigned I = 0; I < SrcNumElements; I += ScalarsPerArrayElement) {
331 unsigned ArrayIndex = I / ScalarsPerArrayElement;
332 // Create a GEP to access the array element.
333 std::array<Value *, 4> Args = {
334 B.getInt1(/*Inbounds=*/V: false), DstArrayPtr, B.getInt32(C: 0),
335 ConstantInt::get(Ty: B.getInt32Ty(), V: ArrayIndex)};
336 auto *ElementPtr = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
337 GR->buildAssignPtr(B, ElemTy: ArrElemVecTy, Arg: ElementPtr);
338
339 // Extract scalar elements from the source vector for this array slot.
340 SmallVector<Value *, 4> Elements;
341 for (unsigned J = 0; J < ScalarsPerArrayElement; ++J)
342 Elements.push_back(Elt: makeExtractElement(B, ElementType: ElemTy, Vector: SrcVector, Index: I + J));
343
344 // Build a vector from the extracted elements and store it.
345 Value *Vec = buildVectorFromLoadedElements(B, TargetType: ArrElemVecTy, LoadedElements&: Elements);
346 StoreInst *SI = B.CreateStore(Val: Vec, Ptr: ElementPtr);
347 SI->setAlignment(Alignment);
348 }
349 }
350
351 // Stores elements from a vector into an array.
352 void storeArrayFromVector(IRBuilder<> &B, Value *SrcVector,
353 Value *DstArrayPtr, ArrayType *ArrTy,
354 Align Alignment) {
355 auto *VecTy = cast<FixedVectorType>(Val: SrcVector->getType());
356 Type *ElemTy = ArrTy->getElementType();
357
358 // Ensure the element types of the array and vector are the same.
359 assert(VecTy->getElementType() == ElemTy &&
360 "Element types of array and vector must be the same.");
361 std::array<Type *, 2> Types = {DstArrayPtr->getType(),
362 DstArrayPtr->getType()};
363
364 for (unsigned I = 0, E = VecTy->getNumElements(); I < E; ++I) {
365 // Create a GEP to access the i-th element of the array.
366 std::array<Value *, 4> Args = {B.getInt1(/*Inbounds=*/V: false), DstArrayPtr,
367 B.getInt32(C: 0),
368 ConstantInt::get(Ty: B.getInt32Ty(), V: I)};
369 auto *ElementPtr = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
370 GR->buildAssignPtr(B, ElemTy, Arg: ElementPtr);
371
372 // Extract the element from the vector and store it.
373 Value *Element =
374 E == 1 ? SrcVector : makeExtractElement(B, ElementType: ElemTy, Vector: SrcVector, Index: I);
375 StoreInst *SI = B.CreateStore(Val: Element, Ptr: ElementPtr);
376 SI->setAlignment(Alignment);
377 }
378 }
379
380 // Replaces the load instruction to get rid of the ptrcast used as source
381 // operand.
382 void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand,
383 Value *OriginalOperand) {
384 Type *ToTy = GR->findDeducedElementType(Val: CastedOperand);
385 B.SetInsertPoint(LI);
386
387 Value *Output = buildLegalizedLoad(B, ElementType: ToTy, Source: OriginalOperand, BadLoad: LI);
388
389 GR->replaceAllUsesWith(Old: LI, New: Output, /* DeleteOld= */ true);
390 DeadInstructions.push_back(x: LI);
391 }
392
393 // Creates an spv_insertelt instruction (equivalent to llvm's insertelement).
394 Value *makeInsertElement(IRBuilder<> &B, Value *Vector, Value *Element,
395 unsigned Index) {
396 Type *Int32Ty = Type::getInt32Ty(C&: B.getContext());
397 SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),
398 Element->getType(), Int32Ty};
399 SmallVector<Value *> Args = {Vector, Element, B.getInt32(C: Index)};
400 Value *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, OverloadTypes: {Types}, Args: {Args});
401 buildAssignType(B, Ty: Vector->getType(), Arg: NewI);
402 return NewI;
403 }
404
405 // Creates an spv_extractelt instruction (equivalent to llvm's
406 // extractelement).
407 Value *makeExtractElement(IRBuilder<> &B, Type *ElementType, Value *Vector,
408 unsigned Index) {
409 Type *Int32Ty = Type::getInt32Ty(C&: B.getContext());
410 SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};
411 SmallVector<Value *> Args = {Vector, B.getInt32(C: Index)};
412 Value *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {Types}, Args: {Args});
413 buildAssignType(B, Ty: ElementType, Arg: NewI);
414 return NewI;
415 }
416
417 // Stores the given Src vector operand into the Dst vector, adjusting the size
418 // if required.
419 Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst,
420 Align Alignment) {
421 FixedVectorType *SrcType = cast<FixedVectorType>(Val: Src->getType());
422 FixedVectorType *DstType =
423 cast<FixedVectorType>(Val: GR->findDeducedElementType(Val: Dst));
424 auto dstNumElements = DstType->getNumElements();
425 auto srcNumElements = SrcType->getNumElements();
426
427 // if the element type differs, it is a bitcast.
428 if (DstType->getElementType() != SrcType->getElementType()) {
429 // Support bitcast between vectors of different sizes only if
430 // the total bitwidth is the same.
431 [[maybe_unused]] auto dstBitWidth =
432 DstType->getElementType()->getScalarSizeInBits() * dstNumElements;
433 [[maybe_unused]] auto srcBitWidth =
434 SrcType->getElementType()->getScalarSizeInBits() * srcNumElements;
435 assert(dstBitWidth == srcBitWidth &&
436 "Unsupported bitcast between vectors of different sizes.");
437
438 Src =
439 B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, OverloadTypes: {DstType, SrcType}, Args: {Src});
440 buildAssignType(B, Ty: DstType, Arg: Src);
441 SrcType = DstType;
442
443 StoreInst *SI = B.CreateStore(Val: Src, Ptr: Dst);
444 SI->setAlignment(Alignment);
445 return SI;
446 }
447
448 assert(DstType->getNumElements() >= SrcType->getNumElements());
449 LoadInst *LI = B.CreateLoad(Ty: DstType, Ptr: Dst);
450 LI->setAlignment(Alignment);
451 Value *OldValues = LI;
452 buildAssignType(B, Ty: OldValues->getType(), Arg: OldValues);
453 Value *NewValues = Src;
454
455 for (unsigned I = 0; I < SrcType->getNumElements(); ++I) {
456 Value *Element =
457 makeExtractElement(B, ElementType: SrcType->getElementType(), Vector: NewValues, Index: I);
458 OldValues = makeInsertElement(B, Vector: OldValues, Element, Index: I);
459 }
460
461 StoreInst *SI = B.CreateStore(Val: OldValues, Ptr: Dst);
462 SI->setAlignment(Alignment);
463 return SI;
464 }
465
466 // Builds a legalized store to a pointer, drilling down through
467 // memory layouts to find a compatible type.
468 void buildLegalizedStore(IRBuilder<> &B, Value *Src, Value *Dst,
469 Align Alignment) {
470 auto ResultOpt = getPointerToFirstCompatibleType(B, BasePtr: Dst, PointerType: Dst->getType(),
471 TargetElemType: Src->getType(), IsInBounds: true);
472 assert(ResultOpt && "Failed to store to aggregate: "
473 "Could not find compatible memory layout.");
474 auto [GEP, CurrentTy] = *ResultOpt;
475
476 auto *DAT = dyn_cast<ArrayType>(Val: CurrentTy);
477 auto *DVT = dyn_cast<FixedVectorType>(Val: CurrentTy);
478 auto *SVT = dyn_cast<FixedVectorType>(Val: Src->getType());
479 auto *DMAT =
480 DAT ? dyn_cast<FixedVectorType>(Val: DAT->getElementType()) : nullptr;
481
482 if (Src->getType() == CurrentTy) {
483 StoreInst *SI = B.CreateStore(Val: Src, Ptr: GEP);
484 SI->setAlignment(Alignment);
485 return;
486 }
487 if (DVT && SVT) {
488 storeVectorFromVector(B, Src, Dst: GEP, Alignment);
489 return;
490 }
491 if (DAT && SVT && SVT->getElementType() == DAT->getElementType()) {
492 storeArrayFromVector(B, SrcVector: Src, DstArrayPtr: GEP, ArrTy: DAT, Alignment);
493 return;
494 }
495 if (DMAT && SVT && DMAT->getElementType() == SVT->getElementType()) {
496 storeMatrixArrayFromVector(B, SrcVector: Src, DstArrayPtr: GEP, ArrTy: DAT, Alignment);
497 return;
498 }
499
500 llvm_unreachable("Failed to store to aggregate.");
501 }
502
503 // Transforms a store instruction (or SPV intrinsic) using a ptrcast as
504 // operand into a valid logical SPIR-V store with no ptrcast.
505 void transformStore(IRBuilder<> &B, Instruction *BadStore, Value *Src,
506 Value *Dst, Align Alignment) {
507 B.SetInsertPoint(BadStore);
508 buildLegalizedStore(B, Src, Dst, Alignment);
509 DeadInstructions.push_back(x: BadStore);
510 }
511
512 void legalizePointerCast(IntrinsicInst *II) {
513 Value *CastedOperand = II;
514 Value *OriginalOperand = II->getOperand(i_nocapture: 0);
515
516 IRBuilder<> B(II->getContext());
517 std::vector<Value *> Users;
518 for (Use &U : II->uses())
519 Users.push_back(x: U.getUser());
520
521 for (Value *User : Users) {
522 if (LoadInst *LI = dyn_cast<LoadInst>(Val: User)) {
523 transformLoad(B, LI, CastedOperand, OriginalOperand);
524 continue;
525 }
526
527 if (StoreInst *SI = dyn_cast<StoreInst>(Val: User)) {
528 transformStore(B, BadStore: SI, Src: SI->getValueOperand(), Dst: OriginalOperand,
529 Alignment: SI->getAlign());
530 continue;
531 }
532
533 if (IntrinsicInst *Intrin = dyn_cast<IntrinsicInst>(Val: User)) {
534 if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) {
535 DeadInstructions.push_back(x: Intrin);
536 continue;
537 }
538
539 if (Intrin->getIntrinsicID() == Intrinsic::spv_gep) {
540 GR->replaceAllUsesWith(Old: CastedOperand, New: OriginalOperand,
541 /* DeleteOld= */ false);
542 continue;
543 }
544
545 if (Intrin->getIntrinsicID() == Intrinsic::spv_store) {
546 Align Alignment;
547 if (ConstantInt *C = dyn_cast<ConstantInt>(Val: Intrin->getOperand(i_nocapture: 3)))
548 Alignment = Align(C->getZExtValue());
549 transformStore(B, BadStore: Intrin, Src: Intrin->getArgOperand(i: 0), Dst: OriginalOperand,
550 Alignment);
551 continue;
552 }
553 }
554
555 llvm_unreachable("Unsupported ptrcast user. Please fix.");
556 }
557
558 DeadInstructions.push_back(x: II);
559 }
560
561public:
562 SPIRVLegalizePointerCastImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
563
564 bool run(Function &F) {
565 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
566 GR = ST.getSPIRVGlobalRegistry();
567 DeadInstructions.clear();
568
569 std::vector<IntrinsicInst *> WorkList;
570 for (auto &BB : F) {
571 for (auto &I : BB) {
572 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
573 if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast)
574 WorkList.push_back(x: II);
575 }
576 }
577
578 for (IntrinsicInst *II : WorkList)
579 legalizePointerCast(II);
580
581 for (Instruction *I : DeadInstructions)
582 I->eraseFromParent();
583
584 return DeadInstructions.size() != 0;
585 }
586
587private:
588 const SPIRVTargetMachine &TM;
589 SPIRVGlobalRegistry *GR = nullptr;
590 std::vector<Instruction *> DeadInstructions;
591};
592
593class SPIRVLegalizePointerCastLegacy : public FunctionPass {
594public:
595 static char ID;
596 SPIRVLegalizePointerCastLegacy(const SPIRVTargetMachine &TM)
597 : FunctionPass(ID), TM(TM) {}
598
599 bool runOnFunction(Function &F) override {
600 return SPIRVLegalizePointerCastImpl(TM).run(F);
601 }
602
603private:
604 const SPIRVTargetMachine &TM;
605};
606} // namespace
607
608PreservedAnalyses SPIRVLegalizePointerCast::run(Function &F,
609 FunctionAnalysisManager &AM) {
610 return SPIRVLegalizePointerCastImpl(TM).run(F) ? PreservedAnalyses::none()
611 : PreservedAnalyses::all();
612}
613
614char SPIRVLegalizePointerCastLegacy::ID = 0;
615INITIALIZE_PASS(SPIRVLegalizePointerCastLegacy, "spirv-legalize-pointer-cast",
616 "SPIRV legalize pointer cast pass", false, false)
617
618FunctionPass *llvm::createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM) {
619 return new SPIRVLegalizePointerCastLegacy(*TM);
620}
621