1//===- SPIRVISelLowering.cpp - SPIR-V DAG Lowering Impl ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SPIRVTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVISelLowering.h"
14#include "SPIRV.h"
15#include "SPIRVInstrInfo.h"
16#include "SPIRVRegisterBankInfo.h"
17#include "SPIRVRegisterInfo.h"
18#include "SPIRVSubtarget.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetLowering.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
24
25#define DEBUG_TYPE "spirv-lower"
26
27using namespace llvm;
28
29SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
30 const SPIRVSubtarget &ST)
31 : TargetLowering(TM, ST), STI(ST) {
32 // Even with SPV_ALTERA_arbitrary_precision_integers enabled, atomic sizes are
33 // limited by atomicrmw xchg operation, which only supports operand up to 64
34 // bits wide, as defined in SPIR-V legalizer. Currently, spirv-val doesn't
35 // consider 128-bit OpTypeInt as valid either.
36 setMaxAtomicSizeInBitsSupported(64);
37 setMinCmpXchgSizeInBits(8);
38}
39
40// Returns true of the types logically match, as defined in
41// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
42static bool typesLogicallyMatch(const SPIRVTypeInst Ty1,
43 const SPIRVTypeInst Ty2,
44 SPIRVGlobalRegistry &GR) {
45 if (Ty1->getOpcode() != Ty2->getOpcode())
46 return false;
47
48 if (Ty1->getNumOperands() != Ty2->getNumOperands())
49 return false;
50
51 if (Ty1->getOpcode() == SPIRV::OpTypeArray) {
52 // Array must have the same size.
53 if (Ty1->getOperand(i: 2).getReg() != Ty2->getOperand(i: 2).getReg())
54 return false;
55
56 SPIRVTypeInst ElemType1 =
57 GR.getSPIRVTypeForVReg(VReg: Ty1->getOperand(i: 1).getReg());
58 SPIRVTypeInst ElemType2 =
59 GR.getSPIRVTypeForVReg(VReg: Ty2->getOperand(i: 1).getReg());
60 return ElemType1 == ElemType2 ||
61 typesLogicallyMatch(Ty1: ElemType1, Ty2: ElemType2, GR);
62 }
63
64 if (Ty1->getOpcode() == SPIRV::OpTypeStruct) {
65 for (unsigned I = 1; I < Ty1->getNumOperands(); I++) {
66 SPIRVTypeInst ElemType1 =
67 GR.getSPIRVTypeForVReg(VReg: Ty1->getOperand(i: I).getReg());
68 SPIRVTypeInst ElemType2 =
69 GR.getSPIRVTypeForVReg(VReg: Ty2->getOperand(i: I).getReg());
70 if (ElemType1 != ElemType2 &&
71 !typesLogicallyMatch(Ty1: ElemType1, Ty2: ElemType2, GR))
72 return false;
73 }
74 return true;
75 }
76 return false;
77}
78
79unsigned SPIRVTargetLowering::getNumRegistersForCallingConv(
80 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
81 // This code avoids CallLowering fail inside getVectorTypeBreakdown
82 // on v3i1 arguments. Maybe we need to return 1 for all types.
83 // TODO: remove it once this case is supported by the default implementation.
84 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
85 (VT.getVectorElementType() == MVT::i1 ||
86 VT.getVectorElementType() == MVT::i8))
87 return 1;
88 if (!VT.isVector() && VT.isInteger() && VT.getSizeInBits() <= 64)
89 return 1;
90 return getNumRegisters(Context, VT);
91}
92
93MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
94 CallingConv::ID CC,
95 EVT VT) const {
96 // This code avoids CallLowering fail inside getVectorTypeBreakdown
97 // on v3i1 arguments. Maybe we need to return i32 for all types.
98 // TODO: remove it once this case is supported by the default implementation.
99 if (VT.isVector()) {
100 if (VT.getVectorNumElements() == 3) {
101 if (VT.getVectorElementType() == MVT::i1)
102 return MVT::v4i1;
103 else if (VT.getVectorElementType() == MVT::i8)
104 return MVT::v4i8;
105 } else if (!isPowerOf2_32(Value: VT.getVectorNumElements()) &&
106 STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector)) {
107 // Non POT element counts are not yet supported by GISEL.
108 return MVT::getVectorVT(
109 VT: VT.getVectorElementType().getSimpleVT(),
110 EC: ElementCount::getFixed(MinVal: VT.getVectorNumElements()));
111 }
112 }
113 return getRegisterType(Context, VT);
114}
115
116void SPIRVTargetLowering::getTgtMemIntrinsic(
117 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
118 MachineFunction &MF, unsigned Intrinsic) const {
119 IntrinsicInfo Info;
120
121 unsigned AlignIdx = 0;
122 unsigned OrderingIdx = 0;
123 unsigned FlagsIdx;
124
125 switch (Intrinsic) {
126 case Intrinsic::spv_load:
127 FlagsIdx = 1;
128 AlignIdx = 2;
129 break;
130 case Intrinsic::spv_store:
131 FlagsIdx = 2;
132 AlignIdx = 3;
133 break;
134 case Intrinsic::spv_atomic_load:
135 FlagsIdx = 1;
136 OrderingIdx = 2;
137 break;
138 case Intrinsic::spv_atomic_store:
139 FlagsIdx = 2;
140 OrderingIdx = 3;
141 break;
142 default:
143 return;
144 }
145
146 Info.flags = static_cast<MachineMemOperand::Flags>(
147 cast<ConstantInt>(Val: I.getOperand(i_nocapture: FlagsIdx))->getZExtValue());
148 Info.memVT = MVT::i64;
149 // TODO: take into account opaque pointers (don't use getElementType).
150 // MVT::getVT(PtrTy->getElementType());
151
152 if (AlignIdx) {
153 auto *AlignOp = cast<ConstantInt>(Val: I.getOperand(i_nocapture: AlignIdx));
154 Info.align = Align(AlignOp->getZExtValue());
155 }
156
157 if (OrderingIdx) {
158 Info.order = static_cast<AtomicOrdering>(
159 cast<ConstantInt>(Val: I.getOperand(i_nocapture: OrderingIdx))->getZExtValue());
160 }
161 Infos.push_back(Elt: Info);
162}
163
164TargetLowering::ConstraintType
165SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
166 // SPIR-V represents inline assembly via OpAsmINTEL where constraints are
167 // passed through as literals defined by client API. Return C_RegisterClass
168 // for non-memory constraints since SPIR-V does not distinguish between
169 // register, immediate, or memory operands at this level. We do have to return
170 // C_Memory for memory constraints as otherwise IRTranslator gets confused
171 // trying to allocate registers for them.
172 if (Constraint == "m")
173 return C_Memory;
174 return C_RegisterClass;
175}
176
177std::pair<unsigned, const TargetRegisterClass *>
178SPIRVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
179 StringRef Constraint,
180 MVT VT) const {
181 const TargetRegisterClass *RC = nullptr;
182 if (Constraint.starts_with(Prefix: "{"))
183 return std::make_pair(x: 0u, y&: RC);
184
185 if (VT.isFloatingPoint())
186 RC = VT.isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;
187 else if (VT.isInteger())
188 RC = VT.isVector() ? &SPIRV::viIDRegClass : &SPIRV::iIDRegClass;
189 else
190 RC = &SPIRV::iIDRegClass;
191
192 return std::make_pair(x: 0u, y&: RC);
193}
194
195inline Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg) {
196 const MachineInstr *Inst = MRI->getVRegDef(Reg: OpReg);
197 return Inst && Inst->getOpcode() == SPIRV::OpFunctionParameter
198 ? Inst->getOperand(i: 1).getReg()
199 : OpReg;
200}
201
202static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
203 SPIRVGlobalRegistry &GR, MachineInstr &I,
204 Register OpReg, unsigned OpIdx,
205 SPIRVTypeInst NewPtrType) {
206 MachineIRBuilder MIB(I);
207 Register NewReg = createVirtualRegister(SpvType: NewPtrType, GR: &GR, MRI, MF: MIB.getMF());
208 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
209 .addDef(RegNo: NewReg)
210 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: NewPtrType))
211 .addUse(RegNo: OpReg)
212 .constrainAllUses(TII: *STI.getInstrInfo(), TRI: *STI.getRegisterInfo(),
213 RBI: *STI.getRegBankInfo());
214 I.getOperand(i: OpIdx).setReg(NewReg);
215}
216
217static SPIRVTypeInst createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I,
218 SPIRVTypeInst OpType, bool ReuseType,
219 SPIRVTypeInst ResType,
220 const Type *ResTy) {
221 SPIRV::StorageClass::StorageClass SC =
222 static_cast<SPIRV::StorageClass::StorageClass>(
223 OpType->getOperand(i: 1).getImm());
224 MachineIRBuilder MIB(I);
225 SPIRVTypeInst NewBaseType =
226 ReuseType ? ResType
227 : GR.getOrCreateSPIRVType(
228 Type: ResTy, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
229 return GR.getOrCreateSPIRVPointerType(BaseType: NewBaseType, MIRBuilder&: MIB, SC);
230}
231
232// Insert a bitcast before the instruction to keep SPIR-V code valid
233// when there is a type mismatch between results and operand types.
234static void validatePtrTypes(const SPIRVSubtarget &STI,
235 MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR,
236 MachineInstr &I, unsigned OpIdx,
237 SPIRVTypeInst ResType,
238 const Type *ResTy = nullptr) {
239 // Get operand type
240 MachineFunction *MF = I.getParent()->getParent();
241 Register OpReg = I.getOperand(i: OpIdx).getReg();
242 Register OpTypeReg = getTypeReg(MRI, OpReg);
243 const MachineInstr *OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
244 if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
245 return;
246 // Get operand's pointee type
247 Register ElemTypeReg = OpType->getOperand(i: 2).getReg();
248 SPIRVTypeInst ElemType = GR.getSPIRVTypeForVReg(VReg: ElemTypeReg, MF);
249 if (!ElemType)
250 return;
251 // Check if we need a bitcast to make a statement valid
252 bool IsSameMF = MF == ResType->getParent()->getParent();
253 bool IsEqualTypes = IsSameMF ? ElemType == ResType
254 : GR.getTypeForSPIRVType(Ty: ElemType) == ResTy;
255 if (IsEqualTypes)
256 return;
257 // There is a type mismatch between results and operand types
258 // and we insert a bitcast before the instruction to keep SPIR-V code valid
259 SPIRVTypeInst NewPtrType =
260 createNewPtrType(GR, I, OpType, ReuseType: IsSameMF, ResType, ResTy);
261 if (!GR.isBitcastCompatible(Type1: NewPtrType, Type2: OpType))
262 report_fatal_error(
263 reason: "insert validation bitcast: incompatible result and operand types");
264 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
265}
266
267// Insert a bitcast before OpGroupWaitEvents if the last argument is a pointer
268// that doesn't point to OpTypeEvent.
269static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI,
270 MachineRegisterInfo *MRI,
271 SPIRVGlobalRegistry &GR,
272 MachineInstr &I) {
273 constexpr unsigned OpIdx = 2;
274 MachineFunction *MF = I.getParent()->getParent();
275 Register OpReg = I.getOperand(i: OpIdx).getReg();
276 Register OpTypeReg = getTypeReg(MRI, OpReg);
277 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
278 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
279 return;
280 SPIRVTypeInst ElemType =
281 GR.getSPIRVTypeForVReg(VReg: OpType->getOperand(i: 2).getReg());
282 if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)
283 return;
284 // Insert a bitcast before the instruction to keep SPIR-V code valid.
285 LLVMContext &Context = MF->getFunction().getContext();
286 SPIRVTypeInst NewPtrType =
287 createNewPtrType(GR, I, OpType, ReuseType: false, ResType: nullptr,
288 ResTy: TargetExtType::get(Context, Name: "spirv.Event"));
289 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
290}
291
292// Per the SPIR-V spec, the Size operand of OpLifetimeStart/OpLifetimeStop must
293// be 0 if Pointer points to a non-void type.
294static void validateLifetimeStart(MachineRegisterInfo *MRI,
295 SPIRVGlobalRegistry &GR, MachineInstr &I) {
296 Register PtrReg = I.getOperand(i: 0).getReg();
297 MachineFunction *MF = I.getParent()->getParent();
298 Register PtrTypeReg = getTypeReg(MRI, OpReg: PtrReg);
299 SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(VReg: PtrTypeReg, MF);
300 SPIRVTypeInst PointeeElemType =
301 PtrType ? GR.getPointeeType(PtrType) : nullptr;
302 if (!PointeeElemType || PointeeElemType->getOpcode() == SPIRV::OpTypeVoid ||
303 (PointeeElemType->getOpcode() == SPIRV::OpTypeInt &&
304 PointeeElemType->getOperand(i: 1).getImm() == 8))
305 return;
306 I.getOperand(i: 1).setImm(0);
307}
308
309static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI,
310 MachineRegisterInfo *MRI,
311 SPIRVGlobalRegistry &GR,
312 MachineInstr &I, unsigned OpIdx) {
313 MachineFunction *MF = I.getParent()->getParent();
314 Register OpReg = I.getOperand(i: OpIdx).getReg();
315 Register OpTypeReg = getTypeReg(MRI, OpReg);
316 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpTypeReg, MF);
317 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
318 return;
319 SPIRVTypeInst ElemType =
320 GR.getSPIRVTypeForVReg(VReg: OpType->getOperand(i: 2).getReg());
321 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeStruct ||
322 ElemType->getNumOperands() != 2)
323 return;
324 // It's a structure-wrapper around another type with a single member field.
325 SPIRVTypeInst MemberType =
326 GR.getSPIRVTypeForVReg(VReg: ElemType->getOperand(i: 1).getReg());
327 if (!MemberType)
328 return;
329 unsigned MemberTypeOp = MemberType->getOpcode();
330 if (!isVectorType(SPVTy: MemberType) && MemberTypeOp != SPIRV::OpTypeInt &&
331 MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)
332 return;
333 // It's a structure-wrapper around a valid type. Insert a bitcast before the
334 // instruction to keep SPIR-V code valid.
335 SPIRV::StorageClass::StorageClass SC =
336 static_cast<SPIRV::StorageClass::StorageClass>(
337 OpType->getOperand(i: 1).getImm());
338 MachineIRBuilder MIB(I);
339 SPIRVTypeInst NewPtrType =
340 GR.getOrCreateSPIRVPointerType(BaseType: MemberType, MIRBuilder&: MIB, SC);
341 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
342}
343
344// Insert a bitcast before the function call instruction to keep SPIR-V code
345// valid when there is a type mismatch between actual and expected types of an
346// argument:
347// %formal = OpFunctionParameter %formal_type
348// ...
349// %res = OpFunctionCall %ty %fun %actual ...
350// implies that %actual is of %formal_type, and in case of opaque pointers.
351// We may need to insert a bitcast to ensure this.
352void validateFunCallMachineDef(const SPIRVSubtarget &STI,
353 MachineRegisterInfo *DefMRI,
354 MachineRegisterInfo *CallMRI,
355 SPIRVGlobalRegistry &GR, MachineInstr &FunCall,
356 MachineInstr *FunDef) {
357 if (FunDef->getOpcode() != SPIRV::OpFunction)
358 return;
359 unsigned OpIdx = 3;
360 for (FunDef = FunDef->getNextNode();
361 FunDef && FunDef->getOpcode() == SPIRV::OpFunctionParameter &&
362 OpIdx < FunCall.getNumOperands();
363 FunDef = FunDef->getNextNode(), OpIdx++) {
364 SPIRVTypeInst DefPtrType =
365 DefMRI->getVRegDef(Reg: FunDef->getOperand(i: 1).getReg());
366 SPIRVTypeInst DefElemType =
367 DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer
368 ? GR.getSPIRVTypeForVReg(VReg: DefPtrType->getOperand(i: 2).getReg(),
369 MF: DefPtrType->getParent()->getParent())
370 : nullptr;
371 if (DefElemType) {
372 const Type *DefElemTy = GR.getTypeForSPIRVType(Ty: DefElemType);
373 // validatePtrTypes() works in the context if the call site
374 // When we process historical records about forward calls
375 // we need to switch context to the (forward) call site and
376 // then restore it back to the current machine function.
377 MachineFunction *CurMF =
378 GR.setCurrentFunc(*FunCall.getParent()->getParent());
379 validatePtrTypes(STI, MRI: CallMRI, GR, I&: FunCall, OpIdx, ResType: DefElemType,
380 ResTy: DefElemTy);
381 GR.setCurrentFunc(*CurMF);
382 }
383 }
384}
385
386// Ensure there is no mismatch between actual and expected arg types: calls
387// with a processed definition. Return Function pointer if it's a forward
388// call (ahead of definition), and nullptr otherwise.
389const Function *validateFunCall(const SPIRVSubtarget &STI,
390 MachineRegisterInfo *CallMRI,
391 SPIRVGlobalRegistry &GR,
392 MachineInstr &FunCall) {
393 const GlobalValue *GV = FunCall.getOperand(i: 2).getGlobal();
394 const Function *F = dyn_cast<Function>(Val: GV);
395 MachineInstr *FunDef =
396 const_cast<MachineInstr *>(GR.getFunctionDefinition(F));
397 if (!FunDef)
398 return F;
399 MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();
400 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);
401 return nullptr;
402}
403
404// Ensure there is no mismatch between actual and expected arg types: calls
405// ahead of a processed definition.
406void validateForwardCalls(const SPIRVSubtarget &STI,
407 MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR,
408 MachineInstr &FunDef) {
409 const Function *F = GR.getFunctionByDefinition(MI: &FunDef);
410 if (SmallPtrSet<MachineInstr *, 8> *FwdCalls = GR.getForwardCalls(F))
411 for (MachineInstr *FunCall : *FwdCalls) {
412 MachineRegisterInfo *CallMRI =
413 &FunCall->getParent()->getParent()->getRegInfo();
414 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall&: *FunCall, FunDef: &FunDef);
415 }
416}
417
418// Validation of an access chain.
419void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
420 SPIRVGlobalRegistry &GR, MachineInstr &I) {
421 SPIRVTypeInst BaseTypeInst = GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 0).getReg());
422 if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {
423 SPIRVTypeInst BaseElemType =
424 GR.getSPIRVTypeForVReg(VReg: BaseTypeInst->getOperand(i: 2).getReg());
425 validatePtrTypes(STI, MRI, GR, I, OpIdx: 2, ResType: BaseElemType);
426 }
427}
428
429static void validateVec1Ops(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
430 SPIRVGlobalRegistry &GR, MachineInstr &MI) {
431 // IRTranslator does not believe that rank-1 vectors exist, unlike upstream
432 // LLVM which happily creates <1 x T> vectors. This leads to operations over
433 // <1 x T> vectors getting translated as their scalar counterparts, which is
434 // wrong if we used SPV_EXT_long_vector to preserve the actual vector-ness.
435 switch (MI.getOpcode()) {
436 case SPIRV::OpBitwiseAndS:
437 case SPIRV::OpBitwiseOrS:
438 case SPIRV::OpBitwiseXorS:
439 case SPIRV::OpFAddS:
440 case SPIRV::OpFDivS:
441 case SPIRV::OpFMulS:
442 case SPIRV::OpFNegate:
443 case SPIRV::OpFRemS:
444 case SPIRV::OpFSubS:
445 case SPIRV::OpIAddCarryS:
446 case SPIRV::OpIAddS:
447 case SPIRV::OpIMulS:
448 case SPIRV::OpISubBorrowS:
449 case SPIRV::OpISubS:
450 case SPIRV::OpSDivS:
451 case SPIRV::OpSRemS:
452 case SPIRV::OpShiftLeftLogicalS:
453 case SPIRV::OpShiftRightArithmeticS:
454 case SPIRV::OpShiftRightLogicalS:
455 case SPIRV::OpStrictFAddS:
456 case SPIRV::OpStrictFDivS:
457 case SPIRV::OpStrictFMulS:
458 case SPIRV::OpStrictFRemS:
459 case SPIRV::OpStrictFSubS:
460 case SPIRV::OpUDivS:
461 case SPIRV::OpUModS: {
462 SPIRVTypeInst ResTy = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
463
464 if (!isVectorType(SPVTy: ResTy))
465 return;
466
467 // Restore original Vec1 type.
468 Register NewResultReg = createVirtualRegister(SpvType: ResTy, GR: &GR, MRI, MF: *MI.getMF());
469 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: NewResultReg);
470 // Vector opcodes are always next after scalar (if this ceases to hold we
471 // will have to adapt).
472 MI.setDesc(STI.getInstrInfo()->get(Opcode: MI.getOpcode() + 1));
473 // IRTranslator would've inserted COPYs from the vector into a scalar, which
474 // are spurious and have to be walked through.
475 for (unsigned I = 2; I != MI.getNumOperands(); ++I) {
476 MachineOperand &Op = MI.getOperand(i: I);
477 if (!Op.isReg())
478 continue;
479
480 SPIRVTypeInst OpTy = GR.getSPIRVTypeForVReg(VReg: Op.getReg());
481 if (OpTy == ResTy)
482 continue;
483
484 MachineInstr *OpDef = getDef(MO: Op, MRI);
485 assert(OpDef &&
486 GR.getSPIRVTypeForVReg(OpDef->getOperand(0).getReg()) == ResTy &&
487 "Expected to find Result Type (Vec1)!");
488 MI.substituteRegister(FromReg: Op.getReg(), ToReg: OpDef->getOperand(i: 0).getReg(), SubIdx: 0,
489 RegInfo: *STI.getRegisterInfo());
490 }
491 break;
492 }
493 case TargetOpcode::COPY: {
494 Register ResVReg = MI.getOperand(i: 0).getReg();
495 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
496 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(VReg: ResVReg);
497
498 if (!SrcTy || !DstTy || isVectorType(SPVTy: DstTy) || !isVectorType(SPVTy: SrcTy))
499 return;
500
501 Register ExtractReg = createVirtualRegister(SpvType: DstTy, GR: &GR, MRI, MF: *MI.getMF());
502 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
503 MCID: STI.getInstrInfo()->get(Opcode: SPIRV::OpCompositeExtract))
504 .addDef(RegNo: ExtractReg)
505 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: DstTy))
506 .addUse(RegNo: MI.getOperand(i: 1).getReg())
507 .addImm(Val: 0);
508 for (auto &&U : MRI->use_instructions(Reg: ResVReg))
509 U.substituteRegister(FromReg: ResVReg, ToReg: ExtractReg, SubIdx: 0, RegInfo: *STI.getRegisterInfo());
510 break;
511 }
512 default:
513 break;
514 }
515}
516
517// TODO: the logic of inserting additional bitcast's is to be moved
518// to pre-IRTranslation passes eventually
519void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
520 if (MF.getRegInfo().reservedRegsFrozen())
521 return;
522
523 MachineRegisterInfo *MRI = &MF.getRegInfo();
524 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
525 GR.setCurrentFunc(MF);
526 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
527 MachineBasicBlock *MBB = &*I;
528 for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();
529 MBBI != MBBE;) {
530 MachineInstr &MI = *MBBI++;
531 validateVec1Ops(STI, MRI, GR, MI);
532 switch (MI.getOpcode()) {
533 case SPIRV::OpAtomicLoad:
534 case SPIRV::OpAtomicExchange:
535 case SPIRV::OpAtomicCompareExchange:
536 case SPIRV::OpAtomicCompareExchangeWeak:
537 case SPIRV::OpAtomicIIncrement:
538 case SPIRV::OpAtomicIDecrement:
539 case SPIRV::OpAtomicIAdd:
540 case SPIRV::OpAtomicISub:
541 case SPIRV::OpAtomicSMin:
542 case SPIRV::OpAtomicUMin:
543 case SPIRV::OpAtomicSMax:
544 case SPIRV::OpAtomicUMax:
545 case SPIRV::OpAtomicAnd:
546 case SPIRV::OpAtomicOr:
547 case SPIRV::OpAtomicXor:
548 // for the above listed instructions
549 // OpAtomicXXX <ResType>, ptr %Op, ...
550 // implies that %Op is a pointer to <ResType>
551 case SPIRV::OpLoad:
552 // OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>
553 if (enforcePtrTypeCompatibility(I&: MI, PtrOpIdx: 2, OpIdx: 0))
554 break;
555
556 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 2,
557 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 0).getReg()));
558 break;
559 case SPIRV::OpAtomicStore:
560 // OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>
561 // implies that %Op points to the <Obj>'s type
562 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 0,
563 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 3).getReg()));
564 break;
565 case SPIRV::OpStore:
566 // OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type
567 validatePtrTypes(STI, MRI, GR, I&: MI, OpIdx: 0,
568 ResType: GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg()));
569 break;
570 case SPIRV::OpPtrCastToGeneric:
571 case SPIRV::OpGenericCastToPtr:
572 case SPIRV::OpGenericCastToPtrExplicit:
573 validateAccessChain(STI, MRI, GR, I&: MI);
574 break;
575 case SPIRV::OpPtrAccessChain:
576 case SPIRV::OpInBoundsPtrAccessChain:
577 if (MI.getNumOperands() == 4)
578 validateAccessChain(STI, MRI, GR, I&: MI);
579 break;
580
581 case SPIRV::OpFunctionCall:
582 // ensure there is no mismatch between actual and expected arg types:
583 // calls with a processed definition
584 if (MI.getNumOperands() > 3)
585 if (const Function *F = validateFunCall(STI, CallMRI: MRI, GR, FunCall&: MI))
586 GR.addForwardCall(F, MI: &MI);
587 break;
588 case SPIRV::OpFunction:
589 // ensure there is no mismatch between actual and expected arg types:
590 // calls ahead of a processed definition
591 validateForwardCalls(STI, DefMRI: MRI, GR, FunDef&: MI);
592 break;
593
594 // ensure that LLVM IR add/sub instructions result in logical SPIR-V
595 // instructions when applied to bool type
596 case SPIRV::OpIAddS:
597 case SPIRV::OpIAddV:
598 case SPIRV::OpISubS:
599 case SPIRV::OpISubV:
600 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
601 TypeOpcode: SPIRV::OpTypeBool))
602 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalNotEqual));
603 break;
604 // multiplication of bool operands is equivalent to a logical AND
605 case SPIRV::OpIMulS:
606 case SPIRV::OpIMulV:
607 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
608 TypeOpcode: SPIRV::OpTypeBool))
609 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalAnd));
610 break;
611
612 // ensure that LLVM IR bitwise instructions result in logical SPIR-V
613 // instructions when applied to bool type
614 case SPIRV::OpBitwiseOrS:
615 case SPIRV::OpBitwiseOrV:
616 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
617 TypeOpcode: SPIRV::OpTypeBool))
618 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalOr));
619 break;
620 case SPIRV::OpBitwiseAndS:
621 case SPIRV::OpBitwiseAndV:
622 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
623 TypeOpcode: SPIRV::OpTypeBool))
624 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalAnd));
625 break;
626 case SPIRV::OpBitwiseXorS:
627 case SPIRV::OpBitwiseXorV:
628 if (GR.isScalarOrVectorOfType(VReg: MI.getOperand(i: 1).getReg(),
629 TypeOpcode: SPIRV::OpTypeBool))
630 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpLogicalNotEqual));
631 break;
632 case SPIRV::OpLifetimeStart:
633 case SPIRV::OpLifetimeStop:
634 if (MI.getOperand(i: 1).getImm() > 0)
635 validateLifetimeStart(MRI, GR, I&: MI);
636 break;
637 case SPIRV::OpGroupAsyncCopy:
638 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI, OpIdx: 3);
639 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI, OpIdx: 4);
640 break;
641 case SPIRV::OpGroupWaitEvents:
642 // OpGroupWaitEvents ..., ..., <pointer to OpTypeEvent>
643 validateGroupWaitEventsPtr(STI, MRI, GR, I&: MI);
644 break;
645 case SPIRV::OpConstantI: {
646 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
647 if (Type->getOpcode() != SPIRV::OpTypeInt && MI.getOperand(i: 2).isImm() &&
648 MI.getOperand(i: 2).getImm() == 0) {
649 // Validate the null constant of a target extension type
650 MI.setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
651 for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
652 MI.removeOperand(OpNo: i);
653 }
654 } break;
655 case SPIRV::OpExtInst: {
656 // prefetch
657 if (!MI.getOperand(i: 2).isImm() || !MI.getOperand(i: 3).isImm() ||
658 MI.getOperand(i: 2).getImm() != SPIRV::InstructionSet::OpenCL_std)
659 continue;
660 switch (MI.getOperand(i: 3).getImm()) {
661 case SPIRV::OpenCLExtInst::frexp:
662 case SPIRV::OpenCLExtInst::lgamma_r:
663 case SPIRV::OpenCLExtInst::remquo: {
664 // The last operand must be of a pointer to i32 or vector of i32
665 // values.
666 MachineIRBuilder MIB(MI);
667 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder&: MIB);
668 SPIRVTypeInst RetType = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
669 assert(RetType && "Expected return type");
670 validatePtrTypes(
671 STI, MRI, GR, I&: MI, OpIdx: MI.getNumOperands() - 1,
672 ResType: (!isVectorType(SPVTy: RetType))
673 ? Int32Type
674 : GR.getOrCreateSPIRVVectorType(
675 BaseType: Int32Type, NumElements: GR.getScalarOrVectorComponentCount(Type: RetType),
676 MIRBuilder&: MIB, EmitIR: false));
677 } break;
678 case SPIRV::OpenCLExtInst::fract:
679 case SPIRV::OpenCLExtInst::modf:
680 case SPIRV::OpenCLExtInst::sincos:
681 // The last operand must be of a pointer to the base type represented
682 // by the previous operand.
683 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
684 "Expected v-reg");
685 validatePtrTypes(
686 STI, MRI, GR, I&: MI, OpIdx: MI.getNumOperands() - 1,
687 ResType: GR.getSPIRVTypeForVReg(
688 VReg: MI.getOperand(i: MI.getNumOperands() - 2).getReg()));
689 break;
690 case SPIRV::OpenCLExtInst::prefetch:
691 // Expected `ptr` type is a pointer to float, integer or vector, but
692 // the pontee value can be wrapped into a struct.
693 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
694 "Expected v-reg");
695 validatePtrUnwrapStructField(STI, MRI, GR, I&: MI,
696 OpIdx: MI.getNumOperands() - 2);
697 break;
698 }
699 } break;
700 }
701 }
702 }
703 TargetLowering::finalizeLowering(MF);
704}
705
706// Modifies either operand PtrOpIdx or OpIdx so that the pointee type of
707// PtrOpIdx matches the type for operand OpIdx. Returns true if they already
708// match or if the instruction was modified to make them match.
709bool SPIRVTargetLowering::enforcePtrTypeCompatibility(
710 MachineInstr &I, unsigned int PtrOpIdx, unsigned int OpIdx) const {
711 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
712 SPIRVTypeInst PtrType = GR.getResultType(VReg: I.getOperand(i: PtrOpIdx).getReg());
713
714 if (PtrType && PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
715 return true;
716
717 SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
718 SPIRVTypeInst OpType = GR.getResultType(VReg: I.getOperand(i: OpIdx).getReg());
719
720 if (PointeeType == OpType)
721 return true;
722
723 // getPointeeType yields nullptr for anything that is not an OpTypePointer.
724 // The early return above does not cover an untyped pointer nested in another
725 // type, such as a vector of pointers built for a scalarized vector GEP.
726 // typesLogicallyMatch dereferences both of its arguments, so bail out before
727 // calling it.
728 if (PointeeType && OpType && typesLogicallyMatch(Ty1: PointeeType, Ty2: OpType, GR)) {
729 // Apply OpCopyLogical to OpIdx.
730 if (I.getOperand(i: OpIdx).isDef() &&
731 insertLogicalCopyOnResult(I, NewResultType: PointeeType)) {
732 return true;
733 }
734
735 llvm_unreachable("Unable to add OpCopyLogical yet.");
736 return false;
737 }
738
739 return false;
740}
741
742bool SPIRVTargetLowering::insertLogicalCopyOnResult(
743 MachineInstr &I, SPIRVTypeInst NewResultType) const {
744 MachineRegisterInfo *MRI = &I.getMF()->getRegInfo();
745 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
746
747 Register NewResultReg =
748 createVirtualRegister(SpvType: NewResultType, GR: &GR, MRI, MF: *I.getMF());
749 Register NewTypeReg = GR.getSPIRVTypeID(SpirvType: NewResultType);
750
751 assert(llvm::size(I.defs()) == 1 && "Expected only one def");
752 MachineOperand &OldResult = *I.defs().begin();
753 Register OldResultReg = OldResult.getReg();
754 MachineOperand &OldType = *I.uses().begin();
755 Register OldTypeReg = OldType.getReg();
756
757 OldResult.setReg(NewResultReg);
758 OldType.setReg(NewTypeReg);
759
760 MachineIRBuilder MIB(*I.getNextNode());
761 MIB.buildInstr(Opcode: SPIRV::OpCopyLogical)
762 .addDef(RegNo: OldResultReg)
763 .addUse(RegNo: OldTypeReg)
764 .addUse(RegNo: NewResultReg)
765 .constrainAllUses(TII: *STI.getInstrInfo(), TRI: *STI.getRegisterInfo(),
766 RBI: *STI.getRegBankInfo());
767 return true;
768}
769
770TargetLowering::AtomicExpansionKind
771SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
772 switch (RMW->getOperation()) {
773 case AtomicRMWInst::FAdd:
774 case AtomicRMWInst::FSub:
775 case AtomicRMWInst::FMin:
776 case AtomicRMWInst::FMax:
777 return AtomicExpansionKind::None;
778 case AtomicRMWInst::UIncWrap:
779 case AtomicRMWInst::UDecWrap:
780 // AMD targets lower these to a helper call in SPIRVEmitIntrinsics, so keep
781 // them unexpanded there; other targets need the generic expansion.
782 return STI.getTargetTriple().getVendor() == Triple::AMD
783 ? AtomicExpansionKind::None
784 : AtomicExpansionKind::CmpXChg;
785 case AtomicRMWInst::Nand:
786 return AtomicExpansionKind::CmpXChg;
787 default:
788 return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
789 }
790}
791
792TargetLowering::AtomicExpansionKind
793SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
794 // TODO: Pointer operand should be cast to integer in atomicrmw xchg, since
795 // SPIR-V only supports atomic exchange for integer and floating-point types.
796 return AtomicExpansionKind::None;
797}
798
799TargetLowering::AtomicExpansionKind
800SPIRVTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
801 // TODO: pointer load should return CastToInteger, but
802 // convertAtomicLoadToIntegerType uses BitCast which asserts on pointer types.
803 return AtomicExpansionKind::None;
804}
805
806TargetLowering::AtomicExpansionKind
807SPIRVTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
808 // TODO: pointer store should return CastToInteger, but
809 // convertAtomicStoreToIntegerType uses BitCast which asserts on pointer
810 // types.
811 return AtomicExpansionKind::None;
812}
813