1//===- SPIRVISelLowering.cpp - SPIR-V DAG Lowering Impl ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SPIRVTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVISelLowering.h"
14#include "SPIRV.h"
15#include "SPIRVInstrInfo.h"
16#include "SPIRVRegisterBankInfo.h"
17#include "SPIRVRegisterInfo.h"
18#include "SPIRVSubtarget.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetLowering.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
24
25#define DEBUG_TYPE "spirv-lower"
26
27using namespace llvm;
28
29SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
30 const SPIRVSubtarget &ST)
31 : TargetLowering(TM, ST), STI(ST) {
32 // Even with SPV_ALTERA_arbitrary_precision_integers enabled, atomic sizes are
33 // limited by atomicrmw xchg operation, which only supports operand up to 64
34 // bits wide, as defined in SPIR-V legalizer. Currently, spirv-val doesn't
35 // consider 128-bit OpTypeInt as valid either.
36 setMaxAtomicSizeInBitsSupported(64);
37 setMinCmpXchgSizeInBits(8);
38}
39
40// Returns true of the types logically match, as defined in
41// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
42static bool typesLogicallyMatch(const SPIRVTypeInst Ty1,
43 const SPIRVTypeInst Ty2,
44 SPIRVGlobalRegistry &GR) {
45 if (Ty1->getOpcode() != Ty2->getOpcode())
46 return false;
47
48 if (Ty1->getNumOperands() != Ty2->getNumOperands())
49 return false;
50
51 if (Ty1->getOpcode() == SPIRV::OpTypeArray) {
52 // Array must have the same size.
53 if (Ty1->getOperand(i: 2).getReg() != Ty2->getOperand(i: 2).getReg())
54 return false;
55
56 SPIRVTypeInst ElemType1 =
57 GR.getSPIRVTypeForVReg(VReg: Ty1->getOperand(i: 1).getReg());
58 SPIRVTypeInst ElemType2 =
59 GR.getSPIRVTypeForVReg(VReg: Ty2->getOperand(i: 1).getReg());
60 return ElemType1 == ElemType2 ||
61 typesLogicallyMatch(Ty1: ElemType1, Ty2: ElemType2, GR);
62 }
63
64 if (Ty1->getOpcode() == SPIRV::OpTypeStruct) {
65 for (unsigned I = 1; I < Ty1->getNumOperands(); I++) {
66 SPIRVTypeInst ElemType1 =
67 GR.getSPIRVTypeForVReg(VReg: Ty1->getOperand(i: I).getReg());
68 SPIRVTypeInst ElemType2 =
69 GR.getSPIRVTypeForVReg(VReg: Ty2->getOperand(i: I).getReg());
70 if (ElemType1 != ElemType2 &&
71 !typesLogicallyMatch(Ty1: ElemType1, Ty2: ElemType2, GR))
72 return false;
73 }
74 return true;
75 }
76 return false;
77}
78
79unsigned SPIRVTargetLowering::getNumRegistersForCallingConv(
80 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
81 // This code avoids CallLowering fail inside getVectorTypeBreakdown
82 // on v3i1 arguments. Maybe we need to return 1 for all types.
83 // TODO: remove it once this case is supported by the default implementation.
84 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
85 (VT.getVectorElementType() == MVT::i1 ||
86 VT.getVectorElementType() == MVT::i8))
87 return 1;
88 if (!VT.isVector() && VT.isInteger() && VT.getSizeInBits() <= 64)
89 return 1;
90 return getNumRegisters(Context, VT);
91}
92
93MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
94 CallingConv::ID CC,
95 EVT VT) const {
96 // This code avoids CallLowering fail inside getVectorTypeBreakdown
97 // on v3i1 arguments. Maybe we need to return i32 for all types.
98 // TODO: remove it once this case is supported by the default implementation.
99 if (VT.isVector()) {
100 if (VT.getVectorNumElements() == 3) {
101 if (VT.getVectorElementType() == MVT::i1)
102 return MVT::v4i1;
103 else if (VT.getVectorElementType() == MVT::i8)
104 return MVT::v4i8;
105 } else if (!isPowerOf2_32(Value: VT.getVectorNumElements()) &&
106 STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector)) {
107 // Non POT element counts are not yet supported by GISEL.
108 return MVT::getVectorVT(
109 VT: VT.getVectorElementType().getSimpleVT(),
110 EC: ElementCount::getFixed(MinVal: VT.getVectorNumElements()));
111 }
112 }
113 return getRegisterType(Context, VT);
114}
115
116void SPIRVTargetLowering::getTgtMemIntrinsic(
117 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
118 MachineFunction &MF, unsigned Intrinsic) const {
119 IntrinsicInfo Info;
120
121 unsigned AlignIdx = 0;
122 unsigned OrderingIdx = 0;
123 unsigned FlagsIdx;
124
125 switch (Intrinsic) {
126 case Intrinsic::spv_load:
127 FlagsIdx = 1;
128 AlignIdx = 2;
129 break;
130 case Intrinsic::spv_store:
131 FlagsIdx = 2;
132 AlignIdx = 3;
133 break;
134 case Intrinsic::spv_atomic_load:
135 FlagsIdx = 1;
136 OrderingIdx = 2;
137 break;
138 case Intrinsic::spv_atomic_store:
139 FlagsIdx = 2;
140 OrderingIdx = 3;
141 break;
142 default:
143 return;
144 }
145
146 Info.flags = static_cast<MachineMemOperand::Flags>(
147 cast<ConstantInt>(Val: I.getOperand(i_nocapture: FlagsIdx))->getZExtValue());
148 Info.memVT = MVT::i64;
149 // TODO: take into account opaque pointers (don't use getElementType).
150 // MVT::getVT(PtrTy->getElementType());
151
152 if (AlignIdx) {
153 auto *AlignOp = cast<ConstantInt>(Val: I.getOperand(i_nocapture: AlignIdx));
154 Info.align = Align(AlignOp->getZExtValue());
155 }
156
157 if (OrderingIdx) {
158 Info.order = static_cast<AtomicOrdering>(
159 cast<ConstantInt>(Val: I.getOperand(i_nocapture: OrderingIdx))->getZExtValue());
160 }
161 Infos.push_back(Elt: Info);
162}
163
164TargetLowering::ConstraintType
165SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
166 // SPIR-V represents inline assembly via OpAsmINTEL where constraints are
167 // passed through as literals defined by client API. Return C_RegisterClass
168 // for non-memory constraints since SPIR-V does not distinguish between
169 // register, immediate, or memory operands at this level. We do have to return
170 // C_Memory for memory constraints as otherwise IRTranslator gets confused
171 // trying to allocate registers for them.
172 if (Constraint == "m")
173 return C_Memory;
174 return C_RegisterClass;
175}
176
177std::pair<unsigned, const TargetRegisterClass *>
178SPIRVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
179 StringRef Constraint,
180 MVT VT) const {
181 const TargetRegisterClass *RC = nullptr;
182 if (Constraint.starts_with(Prefix: "{"))
183 return std::make_pair(x: 0u, y&: RC);
184
185 if (VT.isFloatingPoint())
186 RC = VT.isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;
187 else if (VT.isInteger())
188 RC = VT.isVector() ? &SPIRV::viIDRegClass : &SPIRV::iIDRegClass;
189 else
190 RC = &SPIRV::iIDRegClass;
191
192 return std::make_pair(x: 0u, y&: RC);
193}
194
195inline Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg) {
196 const MachineInstr *Inst = MRI->getVRegDef(Reg: OpReg);
197 return Inst && Inst->getOpcode() == SPIRV::OpFunctionParameter
198 ? Inst->getOperand(i: 1).getReg()
199 : OpReg;
200}
201
202static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
203 SPIRVGlobalRegistry &GR, MachineInstr &I,
204 Register OpReg, unsigned OpIdx,
205 SPIRVTypeInst NewPtrType) {
206 MachineIRBuilder MIB(I);
207 Register NewReg = createVirtualRegister(SpvType: NewPtrType, GR: &GR, MRI, MF: MIB.getMF());
208 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
209 .addDef(RegNo: NewReg)
210 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: NewPtrType))
211 .addUse(RegNo: OpReg)
212 .constrainAllUses(TII: *STI.getInstrInfo(), TRI: *STI.getRegisterInfo(),
213 RBI: *STI.getRegBankInfo());
214 I.getOperand(i: OpIdx).setReg(NewReg);
215}
216
217static SPIRVTypeInst createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I,
218 SPIRVTypeInst OpType, bool ReuseType,
219 SPIRVTypeInst ResType,
220 const Type *ResTy) {
221 SPIRV::StorageClass::StorageClass SC =
222 static_cast<SPIRV::StorageClass::StorageClass>(
223 OpType->getOperand(i: 1).getImm());
224 MachineIRBuilder MIB(I);
225 SPIRVTypeInst NewBaseType =
226 ReuseType ? ResType
227 : GR.getOrCreateSPIRVType(
228 Type: ResTy, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
229 return GR.getOrCreateSPIRVPointerType(BaseType: NewBaseType, MIRBuilder&: MIB, SC);
230}
231
232// Insert a bitcast before the instruction to keep SPIR-V code valid
233// when there is a type mismatch between results and operand types.
234static void validatePtrTypes(const SPIRVSubtarget &STI,
235 MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR,
236 MachineInstr &I, unsigned OpIdx,
237 SPIRVTypeInst ResType,
238 const Type *ResTy = nullptr) {
239 // Get operand type
240 MachineFunction *MF = I.getParent()->getParent();
241 Register OpReg = I.getOperand(i: OpIdx).getReg();
242 Register OpTypeReg = getTypeReg(MRI, OpReg);
243 const MachineInstr *OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
244 if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
245 return;
246 // Get operand's pointee type
247 Register ElemTypeReg = OpType->getOperand(i: 2).getReg();
248 SPIRVTypeInst ElemType = GR.getSPIRVTypeForVReg(VReg: ElemTypeReg, MF);
249 if (!ElemType)
250 return;
251 // Check if we need a bitcast to make a statement valid
252 bool IsSameMF = MF == ResType->getParent()->getParent();
253 bool IsEqualTypes = IsSameMF ? ElemType == ResType
254 : GR.getTypeForSPIRVType(Ty: ElemType) == ResTy;
255 if (IsEqualTypes)
256 return;
257 // There is a type mismatch between results and operand types
258 // and we insert a bitcast before the instruction to keep SPIR-V code valid
259 SPIRVTypeInst NewPtrType =
260 createNewPtrType(GR, I, OpType, ReuseType: IsSameMF, ResType, ResTy);
261 if (!GR.isBitcastCompatible(Type1: NewPtrType, Type2: OpType))
262 report_fatal_error(
263 reason: "insert validation bitcast: incompatible result and operand types");
264 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
265}
266
267// Insert a bitcast before OpGroupWaitEvents if the last argument is a pointer
268// that doesn't point to OpTypeEvent.
269static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI,
270 MachineRegisterInfo *MRI,
271 SPIRVGlobalRegistry &GR,
272 MachineInstr &I) {
273 constexpr unsigned OpIdx = 2;
274 MachineFunction *MF = I.getParent()->getParent();
275 Register OpReg = I.getOperand(i: OpIdx).getReg();
276 Register OpTypeReg = getTypeReg(MRI, OpReg);
277 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
278 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
279 return;
280 SPIRVTypeInst ElemType =
281 GR.getSPIRVTypeForVReg(VReg: OpType->getOperand(i: 2).getReg());
282 if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)
283 return;
284 // Insert a bitcast before the instruction to keep SPIR-V code valid.
285 LLVMContext &Context = MF->getFunction().getContext();
286 SPIRVTypeInst NewPtrType =
287 createNewPtrType(GR, I, OpType, ReuseType: false, ResType: nullptr,
288 ResTy: TargetExtType::get(Context, Name: "spirv.Event"));
289 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
290}
291
292static void validateLifetimeStart(const SPIRVSubtarget &STI,
293 MachineRegisterInfo *MRI,
294 SPIRVGlobalRegistry &GR, MachineInstr &I) {
295 Register PtrReg = I.getOperand(i: 0).getReg();
296 MachineFunction *MF = I.getParent()->getParent();
297 Register PtrTypeReg = getTypeReg(MRI, OpReg: PtrReg);
298 SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(VReg: PtrTypeReg, MF);
299 SPIRVTypeInst PonteeElemType = PtrType ? GR.getPointeeType(PtrType) : nullptr;
300 if (!PonteeElemType || PonteeElemType->getOpcode() == SPIRV::OpTypeVoid ||
301 (PonteeElemType->getOpcode() == SPIRV::OpTypeInt &&
302 PonteeElemType->getOperand(i: 1).getImm() == 8))
303 return;
304 // To keep the code valid a bitcast must be inserted
305 SPIRV::StorageClass::StorageClass SC =
306 static_cast<SPIRV::StorageClass::StorageClass>(
307 PtrType->getOperand(i: 1).getImm());
308 MachineIRBuilder MIB(I);
309 LLVMContext &Context = MF->getFunction().getContext();
310 SPIRVTypeInst NewPtrType =
311 GR.getOrCreateSPIRVPointerType(BaseType: IntegerType::getInt8Ty(C&: Context), MIRBuilder&: MIB, SC);
312 doInsertBitcast(STI, MRI, GR, I, OpReg: PtrReg, OpIdx: 0, NewPtrType);
313}
314
315static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI,
316 MachineRegisterInfo *MRI,
317 SPIRVGlobalRegistry &GR,
318 MachineInstr &I, unsigned OpIdx) {
319 MachineFunction *MF = I.getParent()->getParent();
320 Register OpReg = I.getOperand(i: OpIdx).getReg();
321 Register OpTypeReg = getTypeReg(MRI, OpReg);
322 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
323 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
324 return;
325 SPIRVTypeInst ElemType =
326 GR.getSPIRVTypeForVReg(VReg: OpType->getOperand(i: 2).getReg());
327 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeStruct ||
328 ElemType->getNumOperands() != 2)
329 return;
330 // It's a structure-wrapper around another type with a single member field.
331 SPIRVTypeInst MemberType =
332 GR.getSPIRVTypeForVReg(VReg: ElemType->getOperand(i: 1).getReg());
333 if (!MemberType)
334 return;
335 unsigned MemberTypeOp = MemberType->getOpcode();
336 if (!isVectorType(SPVTy: MemberType) && MemberTypeOp != SPIRV::OpTypeInt &&
337 MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)
338 return;
339 // It's a structure-wrapper around a valid type. Insert a bitcast before the
340 // instruction to keep SPIR-V code valid.
341 SPIRV::StorageClass::StorageClass SC =
342 static_cast<SPIRV::StorageClass::StorageClass>(
343 OpType->getOperand(i: 1).getImm());
344 MachineIRBuilder MIB(I);
345 SPIRVTypeInst NewPtrType =
346 GR.getOrCreateSPIRVPointerType(BaseType: MemberType, MIRBuilder&: MIB, SC);
347 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
348}
349
350// Insert a bitcast before the function call instruction to keep SPIR-V code
351// valid when there is a type mismatch between actual and expected types of an
352// argument:
353// %formal = OpFunctionParameter %formal_type
354// ...
355// %res = OpFunctionCall %ty %fun %actual ...
356// implies that %actual is of %formal_type, and in case of opaque pointers.
357// We may need to insert a bitcast to ensure this.
358void validateFunCallMachineDef(const SPIRVSubtarget &STI,
359 MachineRegisterInfo *DefMRI,
360 MachineRegisterInfo *CallMRI,
361 SPIRVGlobalRegistry &GR, MachineInstr &FunCall,
362 MachineInstr *FunDef) {
363 if (FunDef->getOpcode() != SPIRV::OpFunction)
364 return;
365 unsigned OpIdx = 3;
366 for (FunDef = FunDef->getNextNode();
367 FunDef && FunDef->getOpcode() == SPIRV::OpFunctionParameter &&
368 OpIdx < FunCall.getNumOperands();
369 FunDef = FunDef->getNextNode(), OpIdx++) {
370 SPIRVTypeInst DefPtrType =
371 DefMRI->getVRegDef(Reg: FunDef->getOperand(i: 1).getReg());
372 SPIRVTypeInst DefElemType =
373 DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer
374 ? GR.getSPIRVTypeForVReg(VReg: DefPtrType->getOperand(i: 2).getReg(),
375 MF: DefPtrType->getParent()->getParent())
376 : nullptr;
377 if (DefElemType) {
378 const Type *DefElemTy = GR.getTypeForSPIRVType(Ty: DefElemType);
379 // validatePtrTypes() works in the context if the call site
380 // When we process historical records about forward calls
381 // we need to switch context to the (forward) call site and
382 // then restore it back to the current machine function.
383 MachineFunction *CurMF =
384 GR.setCurrentFunc(*FunCall.getParent()->getParent());
385 validatePtrTypes(STI, MRI: CallMRI, GR, I&: FunCall, OpIdx, ResType: DefElemType,
386 ResTy: DefElemTy);
387 GR.setCurrentFunc(*CurMF);
388 }
389 }
390}
391
392// Ensure there is no mismatch between actual and expected arg types: calls
393// with a processed definition. Return Function pointer if it's a forward
394// call (ahead of definition), and nullptr otherwise.
395const Function *validateFunCall(const SPIRVSubtarget &STI,
396 MachineRegisterInfo *CallMRI,
397 SPIRVGlobalRegistry &GR,
398 MachineInstr &FunCall) {
399 const GlobalValue *GV = FunCall.getOperand(i: 2).getGlobal();
400 const Function *F = dyn_cast<Function>(Val: GV);
401 MachineInstr *FunDef =
402 const_cast<MachineInstr *>(GR.getFunctionDefinition(F));
403 if (!FunDef)
404 return F;
405 MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();
406 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);
407 return nullptr;
408}
409
410// Ensure there is no mismatch between actual and expected arg types: calls
411// ahead of a processed definition.
412void validateForwardCalls(const SPIRVSubtarget &STI,
413 MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR,
414 MachineInstr &FunDef) {
415 const Function *F = GR.getFunctionByDefinition(MI: &FunDef);
416 if (SmallPtrSet<MachineInstr *, 8> *FwdCalls = GR.getForwardCalls(F))
417 for (MachineInstr *FunCall : *FwdCalls) {
418 MachineRegisterInfo *CallMRI =
419 &FunCall->getParent()->getParent()->getRegInfo();
420 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall&: *FunCall, FunDef: &FunDef);
421 }
422}
423
424// Validation of an access chain.
425void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
426 SPIRVGlobalRegistry &GR, MachineInstr &I) {
427 SPIRVTypeInst BaseTypeInst = GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 0).getReg());
428 if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {
429 SPIRVTypeInst BaseElemType =
430 GR.getSPIRVTypeForVReg(VReg: BaseTypeInst->getOperand(i: 2).getReg());
431 validatePtrTypes(STI, MRI, GR, I, OpIdx: 2, ResType: BaseElemType);
432 }
433}
434
435static void validateVec1Ops(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
436 SPIRVGlobalRegistry &GR, MachineInstr &MI) {
437 // IRTranslator does not believe that rank-1 vectors exist, unlike upstream
438 // LLVM which happily creates <1 x T> vectors. This leads to operations over
439 // <1 x T> vectors getting translated as their scalar counterparts, which is
440 // wrong if we used SPV_EXT_long_vector to preserve the actual vector-ness.
441 switch (MI.getOpcode()) {
442 case SPIRV::OpBitwiseAndS:
443 case SPIRV::OpBitwiseOrS:
444 case SPIRV::OpBitwiseXorS:
445 case SPIRV::OpFAddS:
446 case SPIRV::OpFDivS:
447 case SPIRV::OpFMulS:
448 case SPIRV::OpFNegate:
449 case SPIRV::OpFRemS:
450 case SPIRV::OpFSubS:
451 case SPIRV::OpIAddCarryS:
452 case SPIRV::OpIAddS:
453 case SPIRV::OpIMulS:
454 case SPIRV::OpISubBorrowS:
455 case SPIRV::OpISubS:
456 case SPIRV::OpSDivS:
457 case SPIRV::OpSRemS:
458 case SPIRV::OpShiftLeftLogicalS:
459 case SPIRV::OpShiftRightArithmeticS:
460 case SPIRV::OpShiftRightLogicalS:
461 case SPIRV::OpStrictFAddS:
462 case SPIRV::OpStrictFDivS:
463 case SPIRV::OpStrictFMulS:
464 case SPIRV::OpStrictFRemS:
465 case SPIRV::OpStrictFSubS:
466 case SPIRV::OpUDivS:
467 case SPIRV::OpUModS: {
468 SPIRVTypeInst ResTy = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
469
470 if (!isVectorType(SPVTy: ResTy))
471 return;
472
473 // Restore original Vec1 type.
474 Register NewResultReg = createVirtualRegister(SpvType: ResTy, GR: &GR, MRI, MF: *MI.getMF());
475 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: NewResultReg);
476 // Vector opcodes are always next after scalar (if this ceases to hold we
477 // will have to adapt).
478 MI.setDesc(STI.getInstrInfo()->get(Opcode: MI.getOpcode() + 1));
479 // IRTranslator would've inserted COPYs from the vector into a scalar, which
480 // are spurious and have to be walked through.
481 for (unsigned I = 2; I != MI.getNumOperands(); ++I) {
482 MachineOperand &Op = MI.getOperand(i: I);
483 if (!Op.isReg())
484 continue;
485
486 SPIRVTypeInst OpTy = GR.getSPIRVTypeForVReg(VReg: Op.getReg());
487 if (OpTy == ResTy)
488 continue;
489
490 MachineInstr *OpDef = getDef(MO: Op, MRI);
491 assert(OpDef &&
492 GR.getSPIRVTypeForVReg(OpDef->getOperand(0).getReg()) == ResTy &&
493 "Expected to find Result Type (Vec1)!");
494 MI.substituteRegister(FromReg: Op.getReg(), ToReg: OpDef->getOperand(i: 0).getReg(), SubIdx: 0,
495 RegInfo: *STI.getRegisterInfo());
496 }
497 break;
498 }
499 case TargetOpcode::COPY: {
500 Register ResVReg = MI.getOperand(i: 0).getReg();
501 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
502 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(VReg: ResVReg);
503
504 if (!SrcTy || !DstTy || isVectorType(SPVTy: DstTy) || !isVectorType(SPVTy: SrcTy))
505 return;
506
507 Register ExtractReg = createVirtualRegister(SpvType: DstTy, GR: &GR, MRI, MF: *MI.getMF());
508 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
509 MCID: STI.getInstrInfo()->get(Opcode: SPIRV::OpCompositeExtract))
510 .addDef(RegNo: ExtractReg)
511 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: DstTy))
512 .addUse(RegNo: MI.getOperand(i: 1).getReg())
513 .addImm(Val: 0);
514 for (auto &&U : MRI->use_instructions(Reg: ResVReg))
515 U.substituteRegister(FromReg: ResVReg, ToReg: ExtractReg, SubIdx: 0, RegInfo: *STI.getRegisterInfo());
516 break;
517 }
518 default:
519 break;
520 }
521}
522
523// TODO: the logic of inserting additional bitcast's is to be moved
524// to pre-IRTranslation passes eventually
525void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
526 // finalizeLowering() is called twice (see GlobalISel/InstructionSelect.cpp)
527 // We'd like to avoid the needless second processing pass.
528 if (MF.getRegInfo().reservedRegsFrozen())
529 return;
530
531 MachineRegisterInfo *MRI = &MF.getRegInfo();
532 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
533 GR.setCurrentFunc(MF);
534 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
535 MachineBasicBlock *MBB = &*I;
536 for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();
537 MBBI != MBBE;) {
538 MachineInstr &MI = *MBBI++;
539 validateVec1Ops(STI, MRI, GR, MI);
540 switch (MI.getOpcode()) {
541 case SPIRV::OpAtomicLoad:
542 case SPIRV::OpAtomicExchange:
543 case SPIRV::OpAtomicCompareExchange:
544 case SPIRV::OpAtomicCompareExchangeWeak:
545 case SPIRV::OpAtomicIIncrement:
546 case SPIRV::OpAtomicIDecrement:
547 case SPIRV::OpAtomicIAdd:
548 case SPIRV::OpAtomicISub:
549 case SPIRV::OpAtomicSMin:
550 case SPIRV::OpAtomicUMin:
551 case SPIRV::OpAtomicSMax:
552 case SPIRV::OpAtomicUMax:
553 case SPIRV::OpAtomicAnd:
554 case SPIRV::OpAtomicOr:
555 case SPIRV::OpAtomicXor:
556 // for the above listed instructions
557 // OpAtomicXXX <ResType>, ptr %Op, ...
558 // implies that %Op is a pointer to <ResType>
559 case SPIRV::OpLoad:
560 // OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>
561 if (enforcePtrTypeCompatibility(I&: MI, PtrOpIdx: 2, OpIdx: 0))
562 break;
563
564 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 2,
565 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 0).getReg()));
566 break;
567 case SPIRV::OpAtomicStore:
568 // OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>
569 // implies that %Op points to the <Obj>'s type
570 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 0,
571 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 3).getReg()));
572 break;
573 case SPIRV::OpStore:
574 // OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type
575 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 0,
576 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg()));
577 break;
578 case SPIRV::OpPtrCastToGeneric:
579 case SPIRV::OpGenericCastToPtr:
580 case SPIRV::OpGenericCastToPtrExplicit:
581 validateAccessChain(STI, MRI, GR, I&: MI);
582 break;
583 case SPIRV::OpPtrAccessChain:
584 case SPIRV::OpInBoundsPtrAccessChain:
585 if (MI.getNumOperands() == 4)
586 validateAccessChain(STI, MRI, GR, I&: MI);
587 break;
588
589 case SPIRV::OpFunctionCall:
590 // ensure there is no mismatch between actual and expected arg types:
591 // calls with a processed definition
592 if (MI.getNumOperands() > 3)
593 if (const Function *F = validateFunCall(STI, CallMRI: MRI, GR, FunCall&: MI))
594 GR.addForwardCall(F, MI: &MI);
595 break;
596 case SPIRV::OpFunction:
597 // ensure there is no mismatch between actual and expected arg types:
598 // calls ahead of a processed definition
599 validateForwardCalls(STI, DefMRI: MRI, GR, FunDef&: MI);
600 break;
601
602 // ensure that LLVM IR add/sub instructions result in logical SPIR-V
603 // instructions when applied to bool type
604 case SPIRV::OpIAddS:
605 case SPIRV::OpIAddV:
606 case SPIRV::OpISubS:
607 case SPIRV::OpISubV:
608 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
609 TypeOpcode: SPIRV::OpTypeBool))
610 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalNotEqual));
611 break;
612 // multiplication of bool operands is equivalent to a logical AND
613 case SPIRV::OpIMulS:
614 case SPIRV::OpIMulV:
615 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
616 TypeOpcode: SPIRV::OpTypeBool))
617 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalAnd));
618 break;
619
620 // ensure that LLVM IR bitwise instructions result in logical SPIR-V
621 // instructions when applied to bool type
622 case SPIRV::OpBitwiseOrS:
623 case SPIRV::OpBitwiseOrV:
624 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
625 TypeOpcode: SPIRV::OpTypeBool))
626 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalOr));
627 break;
628 case SPIRV::OpBitwiseAndS:
629 case SPIRV::OpBitwiseAndV:
630 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
631 TypeOpcode: SPIRV::OpTypeBool))
632 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalAnd));
633 break;
634 case SPIRV::OpBitwiseXorS:
635 case SPIRV::OpBitwiseXorV:
636 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
637 TypeOpcode: SPIRV::OpTypeBool))
638 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalNotEqual));
639 break;
640 case SPIRV::OpLifetimeStart:
641 case SPIRV::OpLifetimeStop:
642 if (MI.getOperand(i: 1).getImm() > 0)
643 validateLifetimeStart(STI, MRI, GR, I&: MI);
644 break;
645 case SPIRV::OpGroupAsyncCopy:
646 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI, OpIdx: 3);
647 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI, OpIdx: 4);
648 break;
649 case SPIRV::OpGroupWaitEvents:
650 // OpGroupWaitEvents ..., ..., <pointer to OpTypeEvent>
651 validateGroupWaitEventsPtr(STI, MRI, GR, I&: MI);
652 break;
653 case SPIRV::OpConstantI: {
654 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
655 if (Type->getOpcode() != SPIRV::OpTypeInt && MI.getOperand(i: 2).isImm() &&
656 MI.getOperand(i: 2).getImm() == 0) {
657 // Validate the null constant of a target extension type
658 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
659 for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
660 MI.removeOperand(OpNo: i);
661 }
662 } break;
663 case SPIRV::OpExtInst: {
664 // prefetch
665 if (!MI.getOperand(i: 2).isImm() || !MI.getOperand(i: 3).isImm() ||
666 MI.getOperand(i: 2).getImm() != SPIRV::InstructionSet::OpenCL_std)
667 continue;
668 switch (MI.getOperand(i: 3).getImm()) {
669 case SPIRV::OpenCLExtInst::frexp:
670 case SPIRV::OpenCLExtInst::lgamma_r:
671 case SPIRV::OpenCLExtInst::remquo: {
672 // The last operand must be of a pointer to i32 or vector of i32
673 // values.
674 MachineIRBuilder MIB(MI);
675 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder&: MIB);
676 SPIRVTypeInst RetType = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
677 assert(RetType && "Expected return type");
678 validatePtrTypes(
679 STI, MRI, GR, I&: MI, OpIdx: MI.getNumOperands() - 1,
680 ResType: (!isVectorType(SPVTy: RetType))
681 ? Int32Type
682 : GR.getOrCreateSPIRVVectorType(
683 BaseType: Int32Type, NumElements: GR.getScalarOrVectorComponentCount(Type: RetType),
684 MIRBuilder&: MIB, EmitIR: false));
685 } break;
686 case SPIRV::OpenCLExtInst::fract:
687 case SPIRV::OpenCLExtInst::modf:
688 case SPIRV::OpenCLExtInst::sincos:
689 // The last operand must be of a pointer to the base type represented
690 // by the previous operand.
691 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
692 "Expected v-reg");
693 validatePtrTypes(
694 STI, MRI, GR, I&: MI, OpIdx: MI.getNumOperands() - 1,
695 ResType: GR.getSPIRVTypeForVReg(
696 VReg: MI.getOperand(i: MI.getNumOperands() - 2).getReg()));
697 break;
698 case SPIRV::OpenCLExtInst::prefetch:
699 // Expected `ptr` type is a pointer to float, integer or vector, but
700 // the pontee value can be wrapped into a struct.
701 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
702 "Expected v-reg");
703 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI,
704 OpIdx: MI.getNumOperands() - 2);
705 break;
706 }
707 } break;
708 }
709 }
710 }
711 TargetLowering::finalizeLowering(MF);
712}
713
714// Modifies either operand PtrOpIdx or OpIdx so that the pointee type of
715// PtrOpIdx matches the type for operand OpIdx. Returns true if they already
716// match or if the instruction was modified to make them match.
717bool SPIRVTargetLowering::enforcePtrTypeCompatibility(
718 MachineInstr &I, unsigned int PtrOpIdx, unsigned int OpIdx) const {
719 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
720 SPIRVTypeInst PtrType = GR.getResultType(VReg: I.getOperand(i: PtrOpIdx).getReg());
721
722 if (PtrType && PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
723 return true;
724
725 SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
726 SPIRVTypeInst OpType = GR.getResultType(VReg: I.getOperand(i: OpIdx).getReg());
727
728 if (PointeeType == OpType)
729 return true;
730
731 // getPointeeType yields nullptr for anything that is not an OpTypePointer.
732 // The early return above does not cover an untyped pointer nested in another
733 // type, such as a vector of pointers built for a scalarized vector GEP.
734 // typesLogicallyMatch dereferences both of its arguments, so bail out before
735 // calling it.
736 if (PointeeType && OpType && typesLogicallyMatch(Ty1: PointeeType, Ty2: OpType, GR)) {
737 // Apply OpCopyLogical to OpIdx.
738 if (I.getOperand(i: OpIdx).isDef() &&
739 insertLogicalCopyOnResult(I, NewResultType: PointeeType)) {
740 return true;
741 }
742
743 llvm_unreachable("Unable to add OpCopyLogical yet.");
744 return false;
745 }
746
747 return false;
748}
749
750bool SPIRVTargetLowering::insertLogicalCopyOnResult(
751 MachineInstr &I, SPIRVTypeInst NewResultType) const {
752 MachineRegisterInfo *MRI = &I.getMF()->getRegInfo();
753 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
754
755 Register NewResultReg =
756 createVirtualRegister(SpvType: NewResultType, GR: &GR, MRI, MF: *I.getMF());
757 Register NewTypeReg = GR.getSPIRVTypeID(SpirvType: NewResultType);
758
759 assert(llvm::size(I.defs()) == 1 && "Expected only one def");
760 MachineOperand &OldResult = *I.defs().begin();
761 Register OldResultReg = OldResult.getReg();
762 MachineOperand &OldType = *I.uses().begin();
763 Register OldTypeReg = OldType.getReg();
764
765 OldResult.setReg(NewResultReg);
766 OldType.setReg(NewTypeReg);
767
768 MachineIRBuilder MIB(*I.getNextNode());
769 MIB.buildInstr(Opcode: SPIRV::OpCopyLogical)
770 .addDef(RegNo: OldResultReg)
771 .addUse(RegNo: OldTypeReg)
772 .addUse(RegNo: NewResultReg)
773 .constrainAllUses(TII: *STI.getInstrInfo(), TRI: *STI.getRegisterInfo(),
774 RBI: *STI.getRegBankInfo());
775 return true;
776}
777
778TargetLowering::AtomicExpansionKind
779SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
780 switch (RMW->getOperation()) {
781 case AtomicRMWInst::FAdd:
782 case AtomicRMWInst::FSub:
783 case AtomicRMWInst::FMin:
784 case AtomicRMWInst::FMax:
785 return AtomicExpansionKind::None;
786 case AtomicRMWInst::UIncWrap:
787 case AtomicRMWInst::UDecWrap:
788 case AtomicRMWInst::Nand:
789 return AtomicExpansionKind::CmpXChg;
790 default:
791 return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
792 }
793}
794
795TargetLowering::AtomicExpansionKind
796SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
797 // TODO: Pointer operand should be cast to integer in atomicrmw xchg, since
798 // SPIR-V only supports atomic exchange for integer and floating-point types.
799 return AtomicExpansionKind::None;
800}
801
802TargetLowering::AtomicExpansionKind
803SPIRVTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
804 // TODO: pointer load should return CastToInteger, but
805 // convertAtomicLoadToIntegerType uses BitCast which asserts on pointer types.
806 return AtomicExpansionKind::None;
807}
808
809TargetLowering::AtomicExpansionKind
810SPIRVTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
811 // TODO: pointer store should return CastToInteger, but
812 // convertAtomicStoreToIntegerType uses BitCast which asserts on pointer
813 // types.
814 return AtomicExpansionKind::None;
815}
816