1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVUtils.h"
23#include "llvm/ADT/APFloat.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/Register.h"
31#include "llvm/CodeGen/TargetOpcodes.h"
32#include "llvm/IR/IntrinsicsSPIRV.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35
36#define DEBUG_TYPE "spirv-isel"
37
38using namespace llvm;
39namespace CL = SPIRV::OpenCLExtInst;
40namespace GL = SPIRV::GLSLExtInst;
41
42using ExtInstList =
43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
44
45namespace {
46
47llvm::SPIRV::SelectionControl::SelectionControl
48getSelectionOperandForImm(int Imm) {
49 if (Imm == 2)
50 return SPIRV::SelectionControl::Flatten;
51 if (Imm == 1)
52 return SPIRV::SelectionControl::DontFlatten;
53 if (Imm == 0)
54 return SPIRV::SelectionControl::None;
55 llvm_unreachable("Invalid immediate");
56}
57
58#define GET_GLOBALISEL_PREDICATE_BITSET
59#include "SPIRVGenGlobalISel.inc"
60#undef GET_GLOBALISEL_PREDICATE_BITSET
61
62class SPIRVInstructionSelector : public InstructionSelector {
63 const SPIRVSubtarget &STI;
64 const SPIRVInstrInfo &TII;
65 const SPIRVRegisterInfo &TRI;
66 const RegisterBankInfo &RBI;
67 SPIRVGlobalRegistry &GR;
68 MachineRegisterInfo *MRI;
69 MachineFunction *HasVRegsReset = nullptr;
70
71 /// We need to keep track of the number we give to anonymous global values to
72 /// generate the same name every time when this is needed.
73 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
74 SmallPtrSet<MachineInstr *, 8> DeadMIs;
75
76public:
77 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
78 const SPIRVSubtarget &ST,
79 const RegisterBankInfo &RBI);
80 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
81 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
82 BlockFrequencyInfo *BFI) override;
83 // Common selection code. Instruction-specific selection occurs in spvSelect.
84 bool select(MachineInstr &I) override;
85 static const char *getName() { return DEBUG_TYPE; }
86
87#define GET_GLOBALISEL_PREDICATES_DECL
88#include "SPIRVGenGlobalISel.inc"
89#undef GET_GLOBALISEL_PREDICATES_DECL
90
91#define GET_GLOBALISEL_TEMPORARIES_DECL
92#include "SPIRVGenGlobalISel.inc"
93#undef GET_GLOBALISEL_TEMPORARIES_DECL
94
95private:
96 void resetVRegsType(MachineFunction &MF);
97
98 // tblgen-erated 'select' implementation, used as the initial selector for
99 // the patterns that don't require complex C++.
100 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
101
102 // All instruction-specific selection that didn't happen in "select()".
103 // Is basically a large Switch/Case delegating to all other select method.
104 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
105 MachineInstr &I) const;
106
107 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType,
108 MachineInstr &I, bool IsSigned) const;
109
110 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType,
111 MachineInstr &I) const;
112
113 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType,
114 MachineInstr &I, unsigned ExtendOpcode,
115 unsigned BitSetOpcode) const;
116
117 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType,
118 MachineInstr &I, Register SrcReg,
119 unsigned BitSetOpcode) const;
120
121 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType,
122 MachineInstr &I, Register SrcReg,
123 unsigned BitSetOpcode, bool SwapPrimarySide) const;
124
125 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType,
126 MachineInstr &I, Register SrcReg,
127 unsigned BitSetOpcode,
128 bool SwapPrimarySide) const;
129
130 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
131 const MachineInstr *Init = nullptr) const;
132
133 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType,
134 MachineInstr &I, std::vector<Register> SrcRegs,
135 unsigned Opcode) const;
136
137 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
138 unsigned Opcode) const;
139
140 bool selectBitcast(Register ResVReg, const SPIRVType *ResType,
141 MachineInstr &I) const;
142
143 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
144 MachineInstr &I) const;
145 bool selectStore(MachineInstr &I) const;
146
147 bool selectStackSave(Register ResVReg, const SPIRVType *ResType,
148 MachineInstr &I) const;
149 bool selectStackRestore(MachineInstr &I) const;
150
151 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
152
153 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
154 MachineInstr &I, unsigned NewOpcode,
155 unsigned NegateOpcode = 0) const;
156
157 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
158 MachineInstr &I) const;
159
160 bool selectFence(MachineInstr &I) const;
161
162 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
163 MachineInstr &I) const;
164
165 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
166 MachineInstr &I, unsigned OpType) const;
167
168 bool selectAll(Register ResVReg, const SPIRVType *ResType,
169 MachineInstr &I) const;
170
171 bool selectAny(Register ResVReg, const SPIRVType *ResType,
172 MachineInstr &I) const;
173
174 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
175 MachineInstr &I) const;
176
177 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType,
178 MachineInstr &I) const;
179 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType,
180 MachineInstr &I) const;
181
182 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
183 unsigned comparisonOpcode, MachineInstr &I) const;
184 bool selectDiscard(Register ResVReg, const SPIRVType *ResType,
185 MachineInstr &I) const;
186
187 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
188 MachineInstr &I) const;
189 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
190 MachineInstr &I) const;
191
192 bool selectSign(Register ResVReg, const SPIRVType *ResType,
193 MachineInstr &I) const;
194
195 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType,
196 MachineInstr &I) const;
197
198 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType,
199 MachineInstr &I, unsigned Opcode) const;
200
201 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType,
202 MachineInstr &I, bool Signed) const;
203
204 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType,
205 MachineInstr &I) const;
206
207 template <bool Signed>
208 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType,
209 MachineInstr &I) const;
210 template <bool Signed>
211 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType,
212 MachineInstr &I) const;
213
214 bool selectWaveReduceMax(Register ResVReg, const SPIRVType *ResType,
215 MachineInstr &I, bool IsUnsigned) const;
216
217 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType,
218 MachineInstr &I) const;
219
220 bool selectConst(Register ResVReg, const SPIRVType *ResType,
221 MachineInstr &I) const;
222
223 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
224 bool IsSigned) const;
225 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
226 bool IsSigned, unsigned Opcode) const;
227 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
228 bool IsSigned) const;
229
230 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
231 MachineInstr &I) const;
232
233 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
234 bool IsSigned) const;
235
236 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
237 const SPIRVType *intTy, const SPIRVType *boolTy) const;
238
239 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
240 MachineInstr &I) const;
241 bool selectFreeze(Register ResVReg, const SPIRVType *ResType,
242 MachineInstr &I) const;
243 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
244 MachineInstr &I) const;
245 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
246 MachineInstr &I) const;
247 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
248 MachineInstr &I) const;
249 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
250 MachineInstr &I) const;
251 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
252 MachineInstr &I) const;
253 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
254 MachineInstr &I) const;
255
256 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
257 MachineInstr &I) const;
258 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType,
259 MachineInstr &I) const;
260
261 bool selectBranch(MachineInstr &I) const;
262 bool selectBranchCond(MachineInstr &I) const;
263
264 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
265 MachineInstr &I) const;
266
267 bool selectExtInst(Register ResVReg, const SPIRVType *RestType,
268 MachineInstr &I, GL::GLSLExtInst GLInst) const;
269 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
270 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
271 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
272 MachineInstr &I, CL::OpenCLExtInst CLInst,
273 GL::GLSLExtInst GLInst) const;
274 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
275 MachineInstr &I, const ExtInstList &ExtInsts) const;
276
277 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
278 MachineInstr &I) const;
279
280 bool selectSaturate(Register ResVReg, const SPIRVType *ResType,
281 MachineInstr &I) const;
282
283 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType,
284 MachineInstr &I, unsigned Opcode) const;
285
286 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType,
287 MachineInstr &I) const;
288
289 bool selectUnmergeValues(MachineInstr &I) const;
290
291 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType,
292 MachineInstr &I) const;
293
294 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType,
295 MachineInstr &I) const;
296 bool selectImageWriteIntrinsic(MachineInstr &I) const;
297 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType,
298 MachineInstr &I) const;
299
300 // Utilities
301 std::pair<Register, bool>
302 buildI32Constant(uint32_t Val, MachineInstr &I,
303 const SPIRVType *ResType = nullptr) const;
304
305 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
306 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
307 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
308 MachineInstr &I) const;
309 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const;
310
311 bool wrapIntoSpecConstantOp(MachineInstr &I,
312 SmallVector<Register> &CompositeArgs) const;
313
314 Register getUcharPtrTypeReg(MachineInstr &I,
315 SPIRV::StorageClass::StorageClass SC) const;
316 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
317 Register Src, Register DestType,
318 uint32_t Opcode) const;
319 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
320 SPIRVType *SrcPtrTy) const;
321 Register buildPointerToResource(const SPIRVType *ResType,
322 SPIRV::StorageClass::StorageClass SC,
323 uint32_t Set, uint32_t Binding,
324 uint32_t ArraySize, Register IndexReg,
325 bool IsNonUniform, StringRef Name,
326 MachineIRBuilder MIRBuilder) const;
327 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const;
328 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType,
329 Register &ReadReg, MachineInstr &InsertionPoint) const;
330 bool generateImageRead(Register &ResVReg, const SPIRVType *ResType,
331 Register ImageReg, Register IdxReg, DebugLoc Loc,
332 MachineInstr &Pos) const;
333 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
334 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
335 Register ResVReg, const SPIRVType *ResType,
336 MachineInstr &I) const;
337 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
338 Register ResVReg, const SPIRVType *ResType,
339 MachineInstr &I) const;
340 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType,
341 GIntrinsic &HandleDef, MachineInstr &Pos) const;
342};
343
344bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
345 const TargetExtType *TET = cast<TargetExtType>(Val: HandleType);
346 if (TET->getTargetExtName() == "spirv.Image") {
347 return false;
348 }
349 assert(TET->getTargetExtName() == "spirv.SignedImage");
350 return TET->getTypeParameter(i: 0)->isIntegerTy();
351}
352} // end anonymous namespace
353
354#define GET_GLOBALISEL_IMPL
355#include "SPIRVGenGlobalISel.inc"
356#undef GET_GLOBALISEL_IMPL
357
358SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
359 const SPIRVSubtarget &ST,
360 const RegisterBankInfo &RBI)
361 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
362 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
363#define GET_GLOBALISEL_PREDICATES_INIT
364#include "SPIRVGenGlobalISel.inc"
365#undef GET_GLOBALISEL_PREDICATES_INIT
366#define GET_GLOBALISEL_TEMPORARIES_INIT
367#include "SPIRVGenGlobalISel.inc"
368#undef GET_GLOBALISEL_TEMPORARIES_INIT
369{
370}
371
372void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
373 GISelValueTracking *VT,
374 CodeGenCoverage *CoverageInfo,
375 ProfileSummaryInfo *PSI,
376 BlockFrequencyInfo *BFI) {
377 MRI = &MF.getRegInfo();
378 GR.setCurrentFunc(MF);
379 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
380}
381
382// Ensure that register classes correspond to pattern matching rules.
383void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
384 if (HasVRegsReset == &MF)
385 return;
386 HasVRegsReset = &MF;
387
388 MachineRegisterInfo &MRI = MF.getRegInfo();
389 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
390 Register Reg = Register::index2VirtReg(Index: I);
391 LLT RegType = MRI.getType(Reg);
392 if (RegType.isScalar())
393 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: 64));
394 else if (RegType.isPointer())
395 MRI.setType(VReg: Reg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
396 else if (RegType.isVector())
397 MRI.setType(VReg: Reg, Ty: LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::scalar(SizeInBits: 64)));
398 }
399 for (const auto &MBB : MF) {
400 for (const auto &MI : MBB) {
401 if (isPreISelGenericOpcode(Opcode: MI.getOpcode()))
402 GR.erase(MI: &MI);
403 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
404 continue;
405
406 Register DstReg = MI.getOperand(i: 0).getReg();
407 LLT DstType = MRI.getType(Reg: DstReg);
408 Register SrcReg = MI.getOperand(i: 1).getReg();
409 LLT SrcType = MRI.getType(Reg: SrcReg);
410 if (DstType != SrcType)
411 MRI.setType(VReg: DstReg, Ty: MRI.getType(Reg: SrcReg));
412
413 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(Reg: DstReg);
414 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(Reg: SrcReg);
415 if (DstRC != SrcRC && SrcRC)
416 MRI.setRegClass(Reg: DstReg, RC: SrcRC);
417 }
418 }
419}
420
421// Return true if the type represents a constant register
422static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef,
423 SmallPtrSet<SPIRVType *, 4> &Visited) {
424 OpDef = passCopy(Def: OpDef, MRI);
425
426 if (Visited.contains(Ptr: OpDef))
427 return true;
428 Visited.insert(Ptr: OpDef);
429
430 unsigned Opcode = OpDef->getOpcode();
431 switch (Opcode) {
432 case TargetOpcode::G_CONSTANT:
433 case TargetOpcode::G_FCONSTANT:
434 return true;
435 case TargetOpcode::G_INTRINSIC:
436 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
437 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
438 return cast<GIntrinsic>(Val&: *OpDef).getIntrinsicID() ==
439 Intrinsic::spv_const_composite;
440 case TargetOpcode::G_BUILD_VECTOR:
441 case TargetOpcode::G_SPLAT_VECTOR: {
442 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands();
443 i++) {
444 MachineInstr *OpNestedDef =
445 OpDef->getOperand(i).isReg()
446 ? MRI->getVRegDef(Reg: OpDef->getOperand(i).getReg())
447 : nullptr;
448 if (OpNestedDef && !isConstReg(MRI, OpDef: OpNestedDef, Visited))
449 return false;
450 }
451 return true;
452 case SPIRV::OpConstantTrue:
453 case SPIRV::OpConstantFalse:
454 case SPIRV::OpConstantI:
455 case SPIRV::OpConstantF:
456 case SPIRV::OpConstantComposite:
457 case SPIRV::OpConstantCompositeContinuedINTEL:
458 case SPIRV::OpConstantSampler:
459 case SPIRV::OpConstantNull:
460 case SPIRV::OpUndef:
461 case SPIRV::OpConstantFunctionPointerINTEL:
462 return true;
463 }
464 }
465 return false;
466}
467
468// Return true if the virtual register represents a constant
469static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
470 SmallPtrSet<SPIRVType *, 4> Visited;
471 if (MachineInstr *OpDef = MRI->getVRegDef(Reg: OpReg))
472 return isConstReg(MRI, OpDef, Visited);
473 return false;
474}
475
476bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
477 for (const auto &MO : MI.all_defs()) {
478 Register Reg = MO.getReg();
479 if (Reg.isPhysical() || !MRI.use_nodbg_empty(RegNo: Reg))
480 return false;
481 }
482 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
483 MI.isLifetimeMarker())
484 return false;
485 if (MI.isPHI())
486 return true;
487 if (MI.mayStore() || MI.isCall() ||
488 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
489 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo())
490 return false;
491 return true;
492}
493
494bool SPIRVInstructionSelector::select(MachineInstr &I) {
495 resetVRegsType(MF&: *I.getParent()->getParent());
496
497 assert(I.getParent() && "Instruction should be in a basic block!");
498 assert(I.getParent()->getParent() && "Instruction should be in a function!");
499
500 Register Opcode = I.getOpcode();
501 // If it's not a GMIR instruction, we've selected it already.
502 if (!isPreISelGenericOpcode(Opcode)) {
503 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
504 Register DstReg = I.getOperand(i: 0).getReg();
505 Register SrcReg = I.getOperand(i: 1).getReg();
506 auto *Def = MRI->getVRegDef(Reg: SrcReg);
507 if (isTypeFoldingSupported(Opcode: Def->getOpcode()) &&
508 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
509 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
510 bool Res = selectImpl(I, CoverageInfo&: *CoverageInfo);
511 LLVM_DEBUG({
512 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
513 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
514 I.print(dbgs());
515 }
516 });
517 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
518 if (Res) {
519 if (!isTriviallyDead(MI: *Def, MRI: *MRI) && isDead(MI: *Def, MRI: *MRI))
520 DeadMIs.insert(Ptr: Def);
521 return Res;
522 }
523 }
524 MRI->setRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg));
525 MRI->replaceRegWith(FromReg: SrcReg, ToReg: DstReg);
526 GR.invalidateMachineInstr(MI: &I);
527 I.removeFromParent();
528 return true;
529 } else if (I.getNumDefs() == 1) {
530 // Make all vregs 64 bits (for SPIR-V IDs).
531 MRI->setType(VReg: I.getOperand(i: 0).getReg(), Ty: LLT::scalar(SizeInBits: 64));
532 }
533 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
534 }
535
536 if (DeadMIs.contains(Ptr: &I)) {
537 // if the instruction has been already made dead by folding it away
538 // erase it
539 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
540 salvageDebugInfo(MRI: *MRI, MI&: I);
541 GR.invalidateMachineInstr(MI: &I);
542 I.eraseFromParent();
543 return true;
544 }
545
546 if (I.getNumOperands() != I.getNumExplicitOperands()) {
547 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
548 return false;
549 }
550
551 // Common code for getting return reg+type, and removing selected instr
552 // from parent occurs here. Instr-specific selection happens in spvSelect().
553 bool HasDefs = I.getNumDefs() > 0;
554 Register ResVReg = HasDefs ? I.getOperand(i: 0).getReg() : Register(0);
555 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(VReg: ResVReg) : nullptr;
556 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
557 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
558 if (spvSelect(ResVReg, ResType, I)) {
559 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
560 for (unsigned i = 0; i < I.getNumDefs(); ++i)
561 MRI->setType(VReg: I.getOperand(i).getReg(), Ty: LLT::scalar(SizeInBits: 64));
562 GR.invalidateMachineInstr(MI: &I);
563 I.removeFromParent();
564 return true;
565 }
566 return false;
567}
568
569static bool mayApplyGenericSelection(unsigned Opcode) {
570 switch (Opcode) {
571 case TargetOpcode::G_CONSTANT:
572 case TargetOpcode::G_FCONSTANT:
573 return false;
574 case TargetOpcode::G_SADDO:
575 case TargetOpcode::G_SSUBO:
576 return true;
577 }
578 return isTypeFoldingSupported(Opcode);
579}
580
581bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
582 MachineInstr &I) const {
583 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: DestReg);
584 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(Reg: SrcReg);
585 if (DstRC != SrcRC && SrcRC)
586 MRI->setRegClass(Reg: DestReg, RC: SrcRC);
587 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
588 MCID: TII.get(Opcode: TargetOpcode::COPY))
589 .addDef(RegNo: DestReg)
590 .addUse(RegNo: SrcReg)
591 .constrainAllUses(TII, TRI, RBI);
592}
593
594bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
595 const SPIRVType *ResType,
596 MachineInstr &I) const {
597 const unsigned Opcode = I.getOpcode();
598 if (mayApplyGenericSelection(Opcode))
599 return selectImpl(I, CoverageInfo&: *CoverageInfo);
600 switch (Opcode) {
601 case TargetOpcode::G_CONSTANT:
602 case TargetOpcode::G_FCONSTANT:
603 return selectConst(ResVReg, ResType, I);
604 case TargetOpcode::G_GLOBAL_VALUE:
605 return selectGlobalValue(ResVReg, I);
606 case TargetOpcode::G_IMPLICIT_DEF:
607 return selectOpUndef(ResVReg, ResType, I);
608 case TargetOpcode::G_FREEZE:
609 return selectFreeze(ResVReg, ResType, I);
610
611 case TargetOpcode::G_INTRINSIC:
612 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
613 case TargetOpcode::G_INTRINSIC_CONVERGENT:
614 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
615 return selectIntrinsic(ResVReg, ResType, I);
616 case TargetOpcode::G_BITREVERSE:
617 return selectBitreverse(ResVReg, ResType, I);
618
619 case TargetOpcode::G_BUILD_VECTOR:
620 return selectBuildVector(ResVReg, ResType, I);
621 case TargetOpcode::G_SPLAT_VECTOR:
622 return selectSplatVector(ResVReg, ResType, I);
623
624 case TargetOpcode::G_SHUFFLE_VECTOR: {
625 MachineBasicBlock &BB = *I.getParent();
626 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
627 .addDef(RegNo: ResVReg)
628 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
629 .addUse(RegNo: I.getOperand(i: 1).getReg())
630 .addUse(RegNo: I.getOperand(i: 2).getReg());
631 for (auto V : I.getOperand(i: 3).getShuffleMask())
632 MIB.addImm(Val: V);
633 return MIB.constrainAllUses(TII, TRI, RBI);
634 }
635 case TargetOpcode::G_MEMMOVE:
636 case TargetOpcode::G_MEMCPY:
637 case TargetOpcode::G_MEMSET:
638 return selectMemOperation(ResVReg, I);
639
640 case TargetOpcode::G_ICMP:
641 return selectICmp(ResVReg, ResType, I);
642 case TargetOpcode::G_FCMP:
643 return selectFCmp(ResVReg, ResType, I);
644
645 case TargetOpcode::G_FRAME_INDEX:
646 return selectFrameIndex(ResVReg, ResType, I);
647
648 case TargetOpcode::G_LOAD:
649 return selectLoad(ResVReg, ResType, I);
650 case TargetOpcode::G_STORE:
651 return selectStore(I);
652
653 case TargetOpcode::G_BR:
654 return selectBranch(I);
655 case TargetOpcode::G_BRCOND:
656 return selectBranchCond(I);
657
658 case TargetOpcode::G_PHI:
659 return selectPhi(ResVReg, ResType, I);
660
661 case TargetOpcode::G_FPTOSI:
662 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
663 case TargetOpcode::G_FPTOUI:
664 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
665
666 case TargetOpcode::G_SITOFP:
667 return selectIToF(ResVReg, ResType, I, IsSigned: true, Opcode: SPIRV::OpConvertSToF);
668 case TargetOpcode::G_UITOFP:
669 return selectIToF(ResVReg, ResType, I, IsSigned: false, Opcode: SPIRV::OpConvertUToF);
670
671 case TargetOpcode::G_CTPOP:
672 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitCount);
673 case TargetOpcode::G_SMIN:
674 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_min, GLInst: GL::SMin);
675 case TargetOpcode::G_UMIN:
676 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_min, GLInst: GL::UMin);
677
678 case TargetOpcode::G_SMAX:
679 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_max, GLInst: GL::SMax);
680 case TargetOpcode::G_UMAX:
681 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_max, GLInst: GL::UMax);
682
683 case TargetOpcode::G_SCMP:
684 return selectSUCmp(ResVReg, ResType, I, IsSigned: true);
685 case TargetOpcode::G_UCMP:
686 return selectSUCmp(ResVReg, ResType, I, IsSigned: false);
687
688 case TargetOpcode::G_STRICT_FMA:
689 case TargetOpcode::G_FMA:
690 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fma, GLInst: GL::Fma);
691
692 case TargetOpcode::G_STRICT_FLDEXP:
693 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ldexp);
694
695 case TargetOpcode::G_FPOW:
696 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pow, GLInst: GL::Pow);
697 case TargetOpcode::G_FPOWI:
698 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pown);
699
700 case TargetOpcode::G_FEXP:
701 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp, GLInst: GL::Exp);
702 case TargetOpcode::G_FEXP2:
703 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp2, GLInst: GL::Exp2);
704
705 case TargetOpcode::G_FLOG:
706 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log, GLInst: GL::Log);
707 case TargetOpcode::G_FLOG2:
708 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log2, GLInst: GL::Log2);
709 case TargetOpcode::G_FLOG10:
710 return selectLog10(ResVReg, ResType, I);
711
712 case TargetOpcode::G_FABS:
713 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fabs, GLInst: GL::FAbs);
714 case TargetOpcode::G_ABS:
715 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_abs, GLInst: GL::SAbs);
716
717 case TargetOpcode::G_FMINNUM:
718 case TargetOpcode::G_FMINIMUM:
719 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmin, GLInst: GL::NMin);
720 case TargetOpcode::G_FMAXNUM:
721 case TargetOpcode::G_FMAXIMUM:
722 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmax, GLInst: GL::NMax);
723
724 case TargetOpcode::G_FCOPYSIGN:
725 return selectExtInst(ResVReg, ResType, I, CLInst: CL::copysign);
726
727 case TargetOpcode::G_FCEIL:
728 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ceil, GLInst: GL::Ceil);
729 case TargetOpcode::G_FFLOOR:
730 return selectExtInst(ResVReg, ResType, I, CLInst: CL::floor, GLInst: GL::Floor);
731
732 case TargetOpcode::G_FCOS:
733 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cos, GLInst: GL::Cos);
734 case TargetOpcode::G_FSIN:
735 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sin, GLInst: GL::Sin);
736 case TargetOpcode::G_FTAN:
737 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tan, GLInst: GL::Tan);
738 case TargetOpcode::G_FACOS:
739 return selectExtInst(ResVReg, ResType, I, CLInst: CL::acos, GLInst: GL::Acos);
740 case TargetOpcode::G_FASIN:
741 return selectExtInst(ResVReg, ResType, I, CLInst: CL::asin, GLInst: GL::Asin);
742 case TargetOpcode::G_FATAN:
743 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan, GLInst: GL::Atan);
744 case TargetOpcode::G_FATAN2:
745 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan2, GLInst: GL::Atan2);
746 case TargetOpcode::G_FCOSH:
747 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cosh, GLInst: GL::Cosh);
748 case TargetOpcode::G_FSINH:
749 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sinh, GLInst: GL::Sinh);
750 case TargetOpcode::G_FTANH:
751 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tanh, GLInst: GL::Tanh);
752
753 case TargetOpcode::G_STRICT_FSQRT:
754 case TargetOpcode::G_FSQRT:
755 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sqrt, GLInst: GL::Sqrt);
756
757 case TargetOpcode::G_CTTZ:
758 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
759 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ctz);
760 case TargetOpcode::G_CTLZ:
761 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
762 return selectExtInst(ResVReg, ResType, I, CLInst: CL::clz);
763
764 case TargetOpcode::G_INTRINSIC_ROUND:
765 return selectExtInst(ResVReg, ResType, I, CLInst: CL::round, GLInst: GL::Round);
766 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
767 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
768 case TargetOpcode::G_INTRINSIC_TRUNC:
769 return selectExtInst(ResVReg, ResType, I, CLInst: CL::trunc, GLInst: GL::Trunc);
770 case TargetOpcode::G_FRINT:
771 case TargetOpcode::G_FNEARBYINT:
772 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
773
774 case TargetOpcode::G_SMULH:
775 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_mul_hi);
776 case TargetOpcode::G_UMULH:
777 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_mul_hi);
778
779 case TargetOpcode::G_SADDSAT:
780 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_add_sat);
781 case TargetOpcode::G_UADDSAT:
782 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_add_sat);
783 case TargetOpcode::G_SSUBSAT:
784 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_sub_sat);
785 case TargetOpcode::G_USUBSAT:
786 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_sub_sat);
787
788 case TargetOpcode::G_UADDO:
789 return selectOverflowArith(ResVReg, ResType, I,
790 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
791 ? SPIRV::OpIAddCarryV
792 : SPIRV::OpIAddCarryS);
793 case TargetOpcode::G_USUBO:
794 return selectOverflowArith(ResVReg, ResType, I,
795 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
796 ? SPIRV::OpISubBorrowV
797 : SPIRV::OpISubBorrowS);
798 case TargetOpcode::G_UMULO:
799 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpUMulExtended);
800 case TargetOpcode::G_SMULO:
801 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpSMulExtended);
802
803 case TargetOpcode::G_SEXT:
804 return selectExt(ResVReg, ResType, I, IsSigned: true);
805 case TargetOpcode::G_ANYEXT:
806 case TargetOpcode::G_ZEXT:
807 return selectExt(ResVReg, ResType, I, IsSigned: false);
808 case TargetOpcode::G_TRUNC:
809 return selectTrunc(ResVReg, ResType, I);
810 case TargetOpcode::G_FPTRUNC:
811 case TargetOpcode::G_FPEXT:
812 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpFConvert);
813
814 case TargetOpcode::G_PTRTOINT:
815 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertPtrToU);
816 case TargetOpcode::G_INTTOPTR:
817 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertUToPtr);
818 case TargetOpcode::G_BITCAST:
819 return selectBitcast(ResVReg, ResType, I);
820 case TargetOpcode::G_ADDRSPACE_CAST:
821 return selectAddrSpaceCast(ResVReg, ResType, I);
822 case TargetOpcode::G_PTR_ADD: {
823 // Currently, we get G_PTR_ADD only applied to global variables.
824 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
825 Register GV = I.getOperand(i: 1).getReg();
826 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(RegNo: GV);
827 (void)II;
828 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
829 (*II).getOpcode() == TargetOpcode::COPY ||
830 (*II).getOpcode() == SPIRV::OpVariable) &&
831 getImm(I.getOperand(2), MRI));
832 // It may be the initialization of a global variable.
833 bool IsGVInit = false;
834 for (MachineRegisterInfo::use_instr_iterator
835 UseIt = MRI->use_instr_begin(RegNo: I.getOperand(i: 0).getReg()),
836 UseEnd = MRI->use_instr_end();
837 UseIt != UseEnd; UseIt = std::next(x: UseIt)) {
838 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
839 (*UseIt).getOpcode() == SPIRV::OpVariable) {
840 IsGVInit = true;
841 break;
842 }
843 }
844 MachineBasicBlock &BB = *I.getParent();
845 if (!IsGVInit) {
846 SPIRVType *GVType = GR.getSPIRVTypeForVReg(VReg: GV);
847 SPIRVType *GVPointeeType = GR.getPointeeType(PtrType: GVType);
848 SPIRVType *ResPointeeType = GR.getPointeeType(PtrType: ResType);
849 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
850 // Build a new virtual register that is associated with the required
851 // data type.
852 Register NewVReg = MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: GV));
853 MRI->setRegClass(Reg: NewVReg, RC: MRI->getRegClass(Reg: GV));
854 // Having a correctly typed base we are ready to build the actually
855 // required GEP. It may not be a constant though, because all Operands
856 // of OpSpecConstantOp is to originate from other const instructions,
857 // and only the AccessChain named opcodes accept a global OpVariable
858 // instruction. We can't use an AccessChain opcode because of the type
859 // mismatch between result and base types.
860 if (!GR.isBitcastCompatible(Type1: ResType, Type2: GVType))
861 report_fatal_error(
862 reason: "incompatible result and operand types in a bitcast");
863 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
864 MachineInstrBuilder MIB =
865 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitcast))
866 .addDef(RegNo: NewVReg)
867 .addUse(RegNo: ResTypeReg)
868 .addUse(RegNo: GV);
869 return MIB.constrainAllUses(TII, TRI, RBI) &&
870 BuildMI(BB, I, MIMD: I.getDebugLoc(),
871 MCID: TII.get(Opcode: STI.isLogicalSPIRV()
872 ? SPIRV::OpInBoundsAccessChain
873 : SPIRV::OpInBoundsPtrAccessChain))
874 .addDef(RegNo: ResVReg)
875 .addUse(RegNo: ResTypeReg)
876 .addUse(RegNo: NewVReg)
877 .addUse(RegNo: I.getOperand(i: 2).getReg())
878 .constrainAllUses(TII, TRI, RBI);
879 } else {
880 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
881 .addDef(RegNo: ResVReg)
882 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
883 .addImm(
884 Val: static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
885 .addUse(RegNo: GV)
886 .addUse(RegNo: I.getOperand(i: 2).getReg())
887 .constrainAllUses(TII, TRI, RBI);
888 }
889 }
890 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
891 // initialize a global variable with a constant expression (e.g., the test
892 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
893 Register Idx = buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
894 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
895 .addDef(RegNo: ResVReg)
896 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
897 .addImm(Val: static_cast<uint32_t>(
898 SPIRV::Opcode::InBoundsPtrAccessChain))
899 .addUse(RegNo: GV)
900 .addUse(RegNo: Idx)
901 .addUse(RegNo: I.getOperand(i: 2).getReg());
902 return MIB.constrainAllUses(TII, TRI, RBI);
903 }
904
905 case TargetOpcode::G_ATOMICRMW_OR:
906 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicOr);
907 case TargetOpcode::G_ATOMICRMW_ADD:
908 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicIAdd);
909 case TargetOpcode::G_ATOMICRMW_AND:
910 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicAnd);
911 case TargetOpcode::G_ATOMICRMW_MAX:
912 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMax);
913 case TargetOpcode::G_ATOMICRMW_MIN:
914 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMin);
915 case TargetOpcode::G_ATOMICRMW_SUB:
916 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicISub);
917 case TargetOpcode::G_ATOMICRMW_XOR:
918 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicXor);
919 case TargetOpcode::G_ATOMICRMW_UMAX:
920 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMax);
921 case TargetOpcode::G_ATOMICRMW_UMIN:
922 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMin);
923 case TargetOpcode::G_ATOMICRMW_XCHG:
924 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicExchange);
925 case TargetOpcode::G_ATOMIC_CMPXCHG:
926 return selectAtomicCmpXchg(ResVReg, ResType, I);
927
928 case TargetOpcode::G_ATOMICRMW_FADD:
929 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT);
930 case TargetOpcode::G_ATOMICRMW_FSUB:
931 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
932 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT,
933 NegateOpcode: SPIRV::OpFNegate);
934 case TargetOpcode::G_ATOMICRMW_FMIN:
935 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMinEXT);
936 case TargetOpcode::G_ATOMICRMW_FMAX:
937 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMaxEXT);
938
939 case TargetOpcode::G_FENCE:
940 return selectFence(I);
941
942 case TargetOpcode::G_STACKSAVE:
943 return selectStackSave(ResVReg, ResType, I);
944 case TargetOpcode::G_STACKRESTORE:
945 return selectStackRestore(I);
946
947 case TargetOpcode::G_UNMERGE_VALUES:
948 return selectUnmergeValues(I);
949
950 // Discard gen opcodes for intrinsics which we do not expect to actually
951 // represent code after lowering or intrinsics which are not implemented but
952 // should not crash when found in a customer's LLVM IR input.
953 case TargetOpcode::G_TRAP:
954 case TargetOpcode::G_DEBUGTRAP:
955 case TargetOpcode::G_UBSANTRAP:
956 case TargetOpcode::DBG_LABEL:
957 return true;
958
959 default:
960 return false;
961 }
962}
963
964bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
965 const SPIRVType *ResType,
966 MachineInstr &I,
967 GL::GLSLExtInst GLInst) const {
968 if (!STI.canUseExtInstSet(
969 E: SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) {
970 std::string DiagMsg;
971 raw_string_ostream OS(DiagMsg);
972 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
973 DiagMsg += " is only supported with the GLSL extended instruction set.\n";
974 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
975 }
976 return selectExtInst(ResVReg, ResType, I,
977 ExtInsts: {{SPIRV::InstructionSet::GLSL_std_450, GLInst}});
978}
979
980bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
981 const SPIRVType *ResType,
982 MachineInstr &I,
983 CL::OpenCLExtInst CLInst) const {
984 return selectExtInst(ResVReg, ResType, I,
985 ExtInsts: {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
986}
987
988bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
989 const SPIRVType *ResType,
990 MachineInstr &I,
991 CL::OpenCLExtInst CLInst,
992 GL::GLSLExtInst GLInst) const {
993 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
994 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
995 return selectExtInst(ResVReg, ResType, I, ExtInsts);
996}
997
998bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
999 const SPIRVType *ResType,
1000 MachineInstr &I,
1001 const ExtInstList &Insts) const {
1002
1003 for (const auto &Ex : Insts) {
1004 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1005 uint32_t Opcode = Ex.second;
1006 if (STI.canUseExtInstSet(E: Set)) {
1007 MachineBasicBlock &BB = *I.getParent();
1008 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1009 .addDef(RegNo: ResVReg)
1010 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1011 .addImm(Val: static_cast<uint32_t>(Set))
1012 .addImm(Val: Opcode);
1013 const unsigned NumOps = I.getNumOperands();
1014 unsigned Index = 1;
1015 if (Index < NumOps &&
1016 I.getOperand(i: Index).getType() ==
1017 MachineOperand::MachineOperandType::MO_IntrinsicID)
1018 Index = 2;
1019 for (; Index < NumOps; ++Index)
1020 MIB.add(MO: I.getOperand(i: Index));
1021 return MIB.constrainAllUses(TII, TRI, RBI);
1022 }
1023 }
1024 return false;
1025}
1026
1027bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1028 const SPIRVType *ResType,
1029 MachineInstr &I,
1030 std::vector<Register> Srcs,
1031 unsigned Opcode) const {
1032 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1033 .addDef(RegNo: ResVReg)
1034 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
1035 for (Register SReg : Srcs) {
1036 MIB.addUse(RegNo: SReg);
1037 }
1038 return MIB.constrainAllUses(TII, TRI, RBI);
1039}
1040
1041bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1042 const SPIRVType *ResType,
1043 MachineInstr &I,
1044 unsigned Opcode) const {
1045 if (STI.isPhysicalSPIRV() && I.getOperand(i: 1).isReg()) {
1046 Register SrcReg = I.getOperand(i: 1).getReg();
1047 bool IsGV = false;
1048 for (MachineRegisterInfo::def_instr_iterator DefIt =
1049 MRI->def_instr_begin(RegNo: SrcReg);
1050 DefIt != MRI->def_instr_end(); DefIt = std::next(x: DefIt)) {
1051 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1052 (*DefIt).getOpcode() == SPIRV::OpVariable) {
1053 IsGV = true;
1054 break;
1055 }
1056 }
1057 if (IsGV) {
1058 uint32_t SpecOpcode = 0;
1059 switch (Opcode) {
1060 case SPIRV::OpConvertPtrToU:
1061 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1062 break;
1063 case SPIRV::OpConvertUToPtr:
1064 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1065 break;
1066 }
1067 if (SpecOpcode)
1068 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1069 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1070 .addDef(RegNo: ResVReg)
1071 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1072 .addImm(Val: SpecOpcode)
1073 .addUse(RegNo: SrcReg)
1074 .constrainAllUses(TII, TRI, RBI);
1075 }
1076 }
1077 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {I.getOperand(i: 1).getReg()},
1078 Opcode);
1079}
1080
1081bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1082 const SPIRVType *ResType,
1083 MachineInstr &I) const {
1084 Register OpReg = I.getOperand(i: 1).getReg();
1085 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
1086 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
1087 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
1088 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitcast);
1089}
1090
1091static void addMemoryOperands(MachineMemOperand *MemOp,
1092 MachineInstrBuilder &MIB,
1093 MachineIRBuilder &MIRBuilder,
1094 SPIRVGlobalRegistry &GR) {
1095 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1096 if (MemOp->isVolatile())
1097 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1098 if (MemOp->isNonTemporal())
1099 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1100 if (MemOp->getAlign().value())
1101 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1102
1103 [[maybe_unused]] MachineInstr *AliasList = nullptr;
1104 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
1105 const SPIRVSubtarget *ST =
1106 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1107 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
1108 if (auto *MD = MemOp->getAAInfo().Scope) {
1109 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1110 if (AliasList)
1111 SpvMemOp |=
1112 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
1113 }
1114 if (auto *MD = MemOp->getAAInfo().NoAlias) {
1115 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1116 if (NoAliasList)
1117 SpvMemOp |=
1118 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
1119 }
1120 }
1121
1122 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1123 MIB.addImm(Val: SpvMemOp);
1124 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1125 MIB.addImm(Val: MemOp->getAlign().value());
1126 if (AliasList)
1127 MIB.addUse(RegNo: AliasList->getOperand(i: 0).getReg());
1128 if (NoAliasList)
1129 MIB.addUse(RegNo: NoAliasList->getOperand(i: 0).getReg());
1130 }
1131}
1132
1133static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
1134 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1135 if (Flags & MachineMemOperand::Flags::MOVolatile)
1136 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1137 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1138 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1139
1140 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1141 MIB.addImm(Val: SpvMemOp);
1142}
1143
1144bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1145 const SPIRVType *ResType,
1146 MachineInstr &I) const {
1147 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1148 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1149
1150 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1151 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1152 if (IntPtrDef &&
1153 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1154 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1155 SPIRVType *HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1156 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1157 Register NewHandleReg =
1158 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1159 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1160 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1161 return false;
1162 }
1163
1164 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1165 return generateImageRead(ResVReg, ResType, ImageReg: NewHandleReg, IdxReg,
1166 Loc: I.getDebugLoc(), Pos&: I);
1167 }
1168 }
1169
1170 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1171 .addDef(RegNo: ResVReg)
1172 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1173 .addUse(RegNo: Ptr);
1174 if (!I.getNumMemOperands()) {
1175 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1176 I.getOpcode() ==
1177 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1178 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1179 } else {
1180 MachineIRBuilder MIRBuilder(I);
1181 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1182 }
1183 return MIB.constrainAllUses(TII, TRI, RBI);
1184}
1185
1186bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1187 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1188 Register StoreVal = I.getOperand(i: 0 + OpOffset).getReg();
1189 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1190
1191 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1192 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1193 if (IntPtrDef &&
1194 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1195 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1196 Register NewHandleReg =
1197 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1198 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1199 SPIRVType *HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1200 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1201 return false;
1202 }
1203
1204 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1205 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1206 auto BMI = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1207 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
1208 .addUse(RegNo: NewHandleReg)
1209 .addUse(RegNo: IdxReg)
1210 .addUse(RegNo: StoreVal);
1211
1212 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(Ty: HandleType);
1213 if (sampledTypeIsSignedInteger(HandleType: LLVMHandleType))
1214 BMI.addImm(Val: 0x1000); // SignExtend
1215
1216 return BMI.constrainAllUses(TII, TRI, RBI);
1217 }
1218 }
1219
1220 MachineBasicBlock &BB = *I.getParent();
1221 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpStore))
1222 .addUse(RegNo: Ptr)
1223 .addUse(RegNo: StoreVal);
1224 if (!I.getNumMemOperands()) {
1225 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1226 I.getOpcode() ==
1227 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1228 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1229 } else {
1230 MachineIRBuilder MIRBuilder(I);
1231 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1232 }
1233 return MIB.constrainAllUses(TII, TRI, RBI);
1234}
1235
1236bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
1237 const SPIRVType *ResType,
1238 MachineInstr &I) const {
1239 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1240 report_fatal_error(
1241 reason: "llvm.stacksave intrinsic: this instruction requires the following "
1242 "SPIR-V extension: SPV_INTEL_variable_length_array",
1243 gen_crash_diag: false);
1244 MachineBasicBlock &BB = *I.getParent();
1245 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSaveMemoryINTEL))
1246 .addDef(RegNo: ResVReg)
1247 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1248 .constrainAllUses(TII, TRI, RBI);
1249}
1250
1251bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
1252 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1253 report_fatal_error(
1254 reason: "llvm.stackrestore intrinsic: this instruction requires the following "
1255 "SPIR-V extension: SPV_INTEL_variable_length_array",
1256 gen_crash_diag: false);
1257 if (!I.getOperand(i: 0).isReg())
1258 return false;
1259 MachineBasicBlock &BB = *I.getParent();
1260 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpRestoreMemoryINTEL))
1261 .addUse(RegNo: I.getOperand(i: 0).getReg())
1262 .constrainAllUses(TII, TRI, RBI);
1263}
1264
1265bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
1266 MachineInstr &I) const {
1267 MachineBasicBlock &BB = *I.getParent();
1268 Register SrcReg = I.getOperand(i: 1).getReg();
1269 bool Result = true;
1270 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
1271 MachineIRBuilder MIRBuilder(I);
1272 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1273 unsigned Val = getIConstVal(ConstReg: I.getOperand(i: 1).getReg(), MRI);
1274 unsigned Num = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1275 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1276 Type *ArrTy = ArrayType::get(ElementType: ValTy, NumElements: Num);
1277 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
1278 BaseType: ArrTy, MIRBuilder, SC: SPIRV::StorageClass::UniformConstant);
1279
1280 SPIRVType *SpvArrTy = GR.getOrCreateSPIRVType(
1281 Type: ArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::None, EmitIR: false);
1282 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvType: SpvArrTy, TII);
1283 // TODO: check if we have such GV, add init, use buildGlobalVariable.
1284 Function &CurFunction = GR.CurMF->getFunction();
1285 Type *LLVMArrTy =
1286 ArrayType::get(ElementType: IntegerType::get(C&: CurFunction.getContext(), NumBits: 8), NumElements: Num);
1287 // Module takes ownership of the global var.
1288 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
1289 true, GlobalValue::InternalLinkage,
1290 Constant::getNullValue(Ty: LLVMArrTy));
1291 Register VarReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1292 auto MIBVar =
1293 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1294 .addDef(RegNo: VarReg)
1295 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VarTy))
1296 .addImm(Val: SPIRV::StorageClass::UniformConstant)
1297 .addUse(RegNo: Const);
1298 Result &= MIBVar.constrainAllUses(TII, TRI, RBI);
1299
1300 GR.add(V: GV, MI: MIBVar);
1301 GR.addGlobalObject(V: GV, MF: GR.CurMF, R: VarReg);
1302
1303 buildOpDecorate(Reg: VarReg, I, TII, Dec: SPIRV::Decoration::Constant, DecArgs: {});
1304 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
1305 BaseType: ValTy, I, SC: SPIRV::StorageClass::UniformConstant);
1306 SrcReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1307 selectOpWithSrcs(ResVReg: SrcReg, ResType: SourceTy, I, Srcs: {VarReg}, Opcode: SPIRV::OpBitcast);
1308 }
1309 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemorySized))
1310 .addUse(RegNo: I.getOperand(i: 0).getReg())
1311 .addUse(RegNo: SrcReg)
1312 .addUse(RegNo: I.getOperand(i: 2).getReg());
1313 if (I.getNumMemOperands()) {
1314 MachineIRBuilder MIRBuilder(I);
1315 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1316 }
1317 Result &= MIB.constrainAllUses(TII, TRI, RBI);
1318 if (ResVReg.isValid() && ResVReg != MIB->getOperand(i: 0).getReg())
1319 Result &= BuildCOPY(DestReg: ResVReg, SrcReg: MIB->getOperand(i: 0).getReg(), I);
1320 return Result;
1321}
1322
1323bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
1324 const SPIRVType *ResType,
1325 MachineInstr &I,
1326 unsigned NewOpcode,
1327 unsigned NegateOpcode) const {
1328 bool Result = true;
1329 assert(I.hasOneMemOperand());
1330 const MachineMemOperand *MemOp = *I.memoperands_begin();
1331 uint32_t Scope = static_cast<uint32_t>(getMemScope(
1332 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1333 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1334 Register ScopeReg = ScopeConstant.first;
1335 Result &= ScopeConstant.second;
1336
1337 Register Ptr = I.getOperand(i: 1).getReg();
1338 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
1339 // auto ScSem =
1340 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
1341 AtomicOrdering AO = MemOp->getSuccessOrdering();
1342 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1343 auto MemSemConstant = buildI32Constant(Val: MemSem /*| ScSem*/, I);
1344 Register MemSemReg = MemSemConstant.first;
1345 Result &= MemSemConstant.second;
1346
1347 Register ValueReg = I.getOperand(i: 2).getReg();
1348 if (NegateOpcode != 0) {
1349 // Translation with negative value operand is requested
1350 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: MRI->getMF());
1351 Result &= selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {ValueReg}, Opcode: NegateOpcode);
1352 ValueReg = TmpReg;
1353 }
1354
1355 return Result &&
1356 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: NewOpcode))
1357 .addDef(RegNo: ResVReg)
1358 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1359 .addUse(RegNo: Ptr)
1360 .addUse(RegNo: ScopeReg)
1361 .addUse(RegNo: MemSemReg)
1362 .addUse(RegNo: ValueReg)
1363 .constrainAllUses(TII, TRI, RBI);
1364}
1365
1366bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
1367 unsigned ArgI = I.getNumOperands() - 1;
1368 Register SrcReg =
1369 I.getOperand(i: ArgI).isReg() ? I.getOperand(i: ArgI).getReg() : Register(0);
1370 SPIRVType *DefType =
1371 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: SrcReg) : nullptr;
1372 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector)
1373 report_fatal_error(
1374 reason: "cannot select G_UNMERGE_VALUES with a non-vector argument");
1375
1376 SPIRVType *ScalarType =
1377 GR.getSPIRVTypeForVReg(VReg: DefType->getOperand(i: 1).getReg());
1378 MachineBasicBlock &BB = *I.getParent();
1379 bool Res = false;
1380 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1381 Register ResVReg = I.getOperand(i).getReg();
1382 SPIRVType *ResType = GR.getSPIRVTypeForVReg(VReg: ResVReg);
1383 if (!ResType) {
1384 // There was no "assign type" actions, let's fix this now
1385 ResType = ScalarType;
1386 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
1387 MRI->setType(VReg: ResVReg, Ty: LLT::scalar(SizeInBits: GR.getScalarOrVectorBitWidth(Type: ResType)));
1388 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *GR.CurMF);
1389 }
1390 auto MIB =
1391 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1392 .addDef(RegNo: ResVReg)
1393 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1394 .addUse(RegNo: SrcReg)
1395 .addImm(Val: static_cast<int64_t>(i));
1396 Res |= MIB.constrainAllUses(TII, TRI, RBI);
1397 }
1398 return Res;
1399}
1400
1401bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
1402 AtomicOrdering AO = AtomicOrdering(I.getOperand(i: 0).getImm());
1403 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1404 auto MemSemConstant = buildI32Constant(Val: MemSem, I);
1405 Register MemSemReg = MemSemConstant.first;
1406 bool Result = MemSemConstant.second;
1407 SyncScope::ID Ord = SyncScope::ID(I.getOperand(i: 1).getImm());
1408 uint32_t Scope = static_cast<uint32_t>(
1409 getMemScope(Ctx&: GR.CurMF->getFunction().getContext(), Id: Ord));
1410 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1411 Register ScopeReg = ScopeConstant.first;
1412 Result &= ScopeConstant.second;
1413 MachineBasicBlock &BB = *I.getParent();
1414 return Result &&
1415 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMemoryBarrier))
1416 .addUse(RegNo: ScopeReg)
1417 .addUse(RegNo: MemSemReg)
1418 .constrainAllUses(TII, TRI, RBI);
1419}
1420
1421bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
1422 const SPIRVType *ResType,
1423 MachineInstr &I,
1424 unsigned Opcode) const {
1425 Type *ResTy = nullptr;
1426 StringRef ResName;
1427 if (!GR.findValueAttrs(Key: &I, Ty&: ResTy, Name&: ResName))
1428 report_fatal_error(
1429 reason: "Not enough info to select the arithmetic with overflow instruction");
1430 if (!ResTy || !ResTy->isStructTy())
1431 report_fatal_error(reason: "Expect struct type result for the arithmetic "
1432 "with overflow instruction");
1433 // "Result Type must be from OpTypeStruct. The struct must have two members,
1434 // and the two members must be the same type."
1435 Type *ResElemTy = cast<StructType>(Val: ResTy)->getElementType(N: 0);
1436 ResTy = StructType::get(elt1: ResElemTy, elts: ResElemTy);
1437 // Build SPIR-V types and constant(s) if needed.
1438 MachineIRBuilder MIRBuilder(I);
1439 SPIRVType *StructType = GR.getOrCreateSPIRVType(
1440 Type: ResTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
1441 assert(I.getNumDefs() > 1 && "Not enought operands");
1442 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
1443 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
1444 if (N > 1)
1445 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
1446 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
1447 Register ZeroReg = buildZerosVal(ResType, I);
1448 // A new virtual register to store the result struct.
1449 Register StructVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1450 MRI->setRegClass(Reg: StructVReg, RC: &SPIRV::IDRegClass);
1451 // Build the result name if needed.
1452 if (ResName.size() > 0)
1453 buildOpName(Target: StructVReg, Name: ResName, MIRBuilder);
1454 // Build the arithmetic with overflow instruction.
1455 MachineBasicBlock &BB = *I.getParent();
1456 auto MIB =
1457 BuildMI(BB, I: MIRBuilder.getInsertPt(), MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1458 .addDef(RegNo: StructVReg)
1459 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: StructType));
1460 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
1461 MIB.addUse(RegNo: I.getOperand(i).getReg());
1462 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
1463 // Build instructions to extract fields of the instruction's result.
1464 // A new virtual register to store the higher part of the result struct.
1465 Register HigherVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1466 MRI->setRegClass(Reg: HigherVReg, RC: &SPIRV::iIDRegClass);
1467 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1468 auto MIB =
1469 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1470 .addDef(RegNo: i == 1 ? HigherVReg : I.getOperand(i).getReg())
1471 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1472 .addUse(RegNo: StructVReg)
1473 .addImm(Val: i);
1474 Result &= MIB.constrainAllUses(TII, TRI, RBI);
1475 }
1476 // Build boolean value from the higher part.
1477 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
1478 .addDef(RegNo: I.getOperand(i: 1).getReg())
1479 .addUse(RegNo: BoolTypeReg)
1480 .addUse(RegNo: HigherVReg)
1481 .addUse(RegNo: ZeroReg)
1482 .constrainAllUses(TII, TRI, RBI);
1483}
1484
1485bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
1486 const SPIRVType *ResType,
1487 MachineInstr &I) const {
1488 bool Result = true;
1489 Register ScopeReg;
1490 Register MemSemEqReg;
1491 Register MemSemNeqReg;
1492 Register Ptr = I.getOperand(i: 2).getReg();
1493 if (!isa<GIntrinsic>(Val: I)) {
1494 assert(I.hasOneMemOperand());
1495 const MachineMemOperand *MemOp = *I.memoperands_begin();
1496 unsigned Scope = static_cast<uint32_t>(getMemScope(
1497 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1498 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1499 ScopeReg = ScopeConstant.first;
1500 Result &= ScopeConstant.second;
1501
1502 unsigned ScSem = static_cast<uint32_t>(
1503 getMemSemanticsForStorageClass(SC: GR.getPointerStorageClass(VReg: Ptr)));
1504 AtomicOrdering AO = MemOp->getSuccessOrdering();
1505 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(Ord: AO)) | ScSem;
1506 auto MemSemEqConstant = buildI32Constant(Val: MemSemEq, I);
1507 MemSemEqReg = MemSemEqConstant.first;
1508 Result &= MemSemEqConstant.second;
1509 AtomicOrdering FO = MemOp->getFailureOrdering();
1510 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(Ord: FO)) | ScSem;
1511 if (MemSemEq == MemSemNeq)
1512 MemSemNeqReg = MemSemEqReg;
1513 else {
1514 auto MemSemNeqConstant = buildI32Constant(Val: MemSemEq, I);
1515 MemSemNeqReg = MemSemNeqConstant.first;
1516 Result &= MemSemNeqConstant.second;
1517 }
1518 } else {
1519 ScopeReg = I.getOperand(i: 5).getReg();
1520 MemSemEqReg = I.getOperand(i: 6).getReg();
1521 MemSemNeqReg = I.getOperand(i: 7).getReg();
1522 }
1523
1524 Register Cmp = I.getOperand(i: 3).getReg();
1525 Register Val = I.getOperand(i: 4).getReg();
1526 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(VReg: Val);
1527 Register ACmpRes = createVirtualRegister(SpvType: SpvValTy, GR: &GR, MRI, MF: *I.getMF());
1528 const DebugLoc &DL = I.getDebugLoc();
1529 Result &=
1530 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpAtomicCompareExchange))
1531 .addDef(RegNo: ACmpRes)
1532 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvValTy))
1533 .addUse(RegNo: Ptr)
1534 .addUse(RegNo: ScopeReg)
1535 .addUse(RegNo: MemSemEqReg)
1536 .addUse(RegNo: MemSemNeqReg)
1537 .addUse(RegNo: Val)
1538 .addUse(RegNo: Cmp)
1539 .constrainAllUses(TII, TRI, RBI);
1540 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
1541 Register CmpSuccReg = createVirtualRegister(SpvType: BoolTy, GR: &GR, MRI, MF: *I.getMF());
1542 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpIEqual))
1543 .addDef(RegNo: CmpSuccReg)
1544 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
1545 .addUse(RegNo: ACmpRes)
1546 .addUse(RegNo: Cmp)
1547 .constrainAllUses(TII, TRI, RBI);
1548 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: *I.getMF());
1549 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
1550 .addDef(RegNo: TmpReg)
1551 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1552 .addUse(RegNo: ACmpRes)
1553 .addUse(RegNo: GR.getOrCreateUndef(I, SpvType: ResType, TII))
1554 .addImm(Val: 0)
1555 .constrainAllUses(TII, TRI, RBI);
1556 return Result &&
1557 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
1558 .addDef(RegNo: ResVReg)
1559 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1560 .addUse(RegNo: CmpSuccReg)
1561 .addUse(RegNo: TmpReg)
1562 .addImm(Val: 1)
1563 .constrainAllUses(TII, TRI, RBI);
1564}
1565
1566static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
1567 switch (SC) {
1568 case SPIRV::StorageClass::DeviceOnlyINTEL:
1569 case SPIRV::StorageClass::HostOnlyINTEL:
1570 return true;
1571 default:
1572 return false;
1573 }
1574}
1575
1576// Returns true ResVReg is referred only from global vars and OpName's.
1577static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
1578 bool IsGRef = false;
1579 bool IsAllowedRefs =
1580 llvm::all_of(Range: MRI->use_instructions(Reg: ResVReg), P: [&IsGRef](auto const &It) {
1581 unsigned Opcode = It.getOpcode();
1582 if (Opcode == SPIRV::OpConstantComposite ||
1583 Opcode == SPIRV::OpVariable ||
1584 isSpvIntrinsic(It, Intrinsic::spv_init_global))
1585 return IsGRef = true;
1586 return Opcode == SPIRV::OpName;
1587 });
1588 return IsAllowedRefs && IsGRef;
1589}
1590
1591Register SPIRVInstructionSelector::getUcharPtrTypeReg(
1592 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
1593 return GR.getSPIRVTypeID(SpirvType: GR.getOrCreateSPIRVPointerType(
1594 BaseType: Type::getInt8Ty(C&: I.getMF()->getFunction().getContext()), I, SC));
1595}
1596
1597MachineInstrBuilder
1598SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
1599 Register Src, Register DestType,
1600 uint32_t Opcode) const {
1601 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1602 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1603 .addDef(RegNo: Dest)
1604 .addUse(RegNo: DestType)
1605 .addImm(Val: Opcode)
1606 .addUse(RegNo: Src);
1607}
1608
1609MachineInstrBuilder
1610SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
1611 SPIRVType *SrcPtrTy) const {
1612 SPIRVType *GenericPtrTy =
1613 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
1614 Register Tmp = MRI->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
1615 MRI->setType(VReg: Tmp, Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(
1616 SC: SPIRV::StorageClass::Generic),
1617 SizeInBits: GR.getPointerSize()));
1618 MachineFunction *MF = I.getParent()->getParent();
1619 GR.assignSPIRVTypeToVReg(Type: GenericPtrTy, VReg: Tmp, MF: *MF);
1620 MachineInstrBuilder MIB = buildSpecConstantOp(
1621 I, Dest: Tmp, Src: SrcPtr, DestType: GR.getSPIRVTypeID(SpirvType: GenericPtrTy),
1622 Opcode: static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
1623 GR.add(Obj: MIB.getInstr(), MI: MIB);
1624 return MIB;
1625}
1626
1627// In SPIR-V address space casting can only happen to and from the Generic
1628// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
1629// pointers to and from Generic pointers. As such, we can convert e.g. from
1630// Workgroup to Function by going via a Generic pointer as an intermediary. All
1631// other combinations can only be done by a bitcast, and are probably not safe.
1632bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
1633 const SPIRVType *ResType,
1634 MachineInstr &I) const {
1635 MachineBasicBlock &BB = *I.getParent();
1636 const DebugLoc &DL = I.getDebugLoc();
1637
1638 Register SrcPtr = I.getOperand(i: 1).getReg();
1639 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(VReg: SrcPtr);
1640
1641 // don't generate a cast for a null that may be represented by OpTypeInt
1642 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
1643 ResType->getOpcode() != SPIRV::OpTypePointer)
1644 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
1645
1646 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(Type: SrcPtrTy);
1647 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(Type: ResType);
1648
1649 if (isASCastInGVar(MRI, ResVReg)) {
1650 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
1651 // are expressed by OpSpecConstantOp with an Opcode.
1652 // TODO: maybe insert a check whether the Kernel capability was declared and
1653 // so PtrCastToGeneric/GenericCastToPtr are available.
1654 unsigned SpecOpcode =
1655 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC)
1656 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
1657 : (SrcSC == SPIRV::StorageClass::Generic &&
1658 isGenericCastablePtr(SC: DstSC)
1659 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
1660 : 0);
1661 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
1662 // correct value of ResType and use general i8* instead. Maybe this should
1663 // be addressed in the emit-intrinsic step to infer a correct
1664 // OpConstantComposite type.
1665 if (SpecOpcode) {
1666 return buildSpecConstantOp(I, Dest: ResVReg, Src: SrcPtr,
1667 DestType: getUcharPtrTypeReg(I, SC: DstSC), Opcode: SpecOpcode)
1668 .constrainAllUses(TII, TRI, RBI);
1669 } else if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
1670 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
1671 return MIB.constrainAllUses(TII, TRI, RBI) &&
1672 buildSpecConstantOp(
1673 I, Dest: ResVReg, Src: MIB->getOperand(i: 0).getReg(),
1674 DestType: getUcharPtrTypeReg(I, SC: DstSC),
1675 Opcode: static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
1676 .constrainAllUses(TII, TRI, RBI);
1677 }
1678 }
1679
1680 // don't generate a cast between identical storage classes
1681 if (SrcSC == DstSC)
1682 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
1683
1684 if ((SrcSC == SPIRV::StorageClass::Function &&
1685 DstSC == SPIRV::StorageClass::Private) ||
1686 (DstSC == SPIRV::StorageClass::Function &&
1687 SrcSC == SPIRV::StorageClass::Private))
1688 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
1689
1690 // Casting from an eligible pointer to Generic.
1691 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC))
1692 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
1693 // Casting from Generic to an eligible pointer.
1694 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: DstSC))
1695 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
1696 // Casting between 2 eligible pointers using Generic as an intermediary.
1697 if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
1698 SPIRVType *GenericPtrTy =
1699 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
1700 Register Tmp = createVirtualRegister(SpvType: GenericPtrTy, GR: &GR, MRI, MF: MRI->getMF());
1701 bool Result = BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpPtrCastToGeneric))
1702 .addDef(RegNo: Tmp)
1703 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: GenericPtrTy))
1704 .addUse(RegNo: SrcPtr)
1705 .constrainAllUses(TII, TRI, RBI);
1706 return Result && BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtr))
1707 .addDef(RegNo: ResVReg)
1708 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1709 .addUse(RegNo: Tmp)
1710 .constrainAllUses(TII, TRI, RBI);
1711 }
1712
1713 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
1714 // be applied
1715 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
1716 return selectUnOp(ResVReg, ResType, I,
1717 Opcode: SPIRV::OpPtrCastToCrossWorkgroupINTEL);
1718 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(SC: DstSC))
1719 return selectUnOp(ResVReg, ResType, I,
1720 Opcode: SPIRV::OpCrossWorkgroupCastToPtrINTEL);
1721 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::Generic)
1722 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
1723 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(SC: DstSC))
1724 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
1725
1726 // Bitcast for pointers requires that the address spaces must match
1727 return false;
1728}
1729
1730static unsigned getFCmpOpcode(unsigned PredNum) {
1731 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1732 switch (Pred) {
1733 case CmpInst::FCMP_OEQ:
1734 return SPIRV::OpFOrdEqual;
1735 case CmpInst::FCMP_OGE:
1736 return SPIRV::OpFOrdGreaterThanEqual;
1737 case CmpInst::FCMP_OGT:
1738 return SPIRV::OpFOrdGreaterThan;
1739 case CmpInst::FCMP_OLE:
1740 return SPIRV::OpFOrdLessThanEqual;
1741 case CmpInst::FCMP_OLT:
1742 return SPIRV::OpFOrdLessThan;
1743 case CmpInst::FCMP_ONE:
1744 return SPIRV::OpFOrdNotEqual;
1745 case CmpInst::FCMP_ORD:
1746 return SPIRV::OpOrdered;
1747 case CmpInst::FCMP_UEQ:
1748 return SPIRV::OpFUnordEqual;
1749 case CmpInst::FCMP_UGE:
1750 return SPIRV::OpFUnordGreaterThanEqual;
1751 case CmpInst::FCMP_UGT:
1752 return SPIRV::OpFUnordGreaterThan;
1753 case CmpInst::FCMP_ULE:
1754 return SPIRV::OpFUnordLessThanEqual;
1755 case CmpInst::FCMP_ULT:
1756 return SPIRV::OpFUnordLessThan;
1757 case CmpInst::FCMP_UNE:
1758 return SPIRV::OpFUnordNotEqual;
1759 case CmpInst::FCMP_UNO:
1760 return SPIRV::OpUnordered;
1761 default:
1762 llvm_unreachable("Unknown predicate type for FCmp");
1763 }
1764}
1765
1766static unsigned getICmpOpcode(unsigned PredNum) {
1767 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1768 switch (Pred) {
1769 case CmpInst::ICMP_EQ:
1770 return SPIRV::OpIEqual;
1771 case CmpInst::ICMP_NE:
1772 return SPIRV::OpINotEqual;
1773 case CmpInst::ICMP_SGE:
1774 return SPIRV::OpSGreaterThanEqual;
1775 case CmpInst::ICMP_SGT:
1776 return SPIRV::OpSGreaterThan;
1777 case CmpInst::ICMP_SLE:
1778 return SPIRV::OpSLessThanEqual;
1779 case CmpInst::ICMP_SLT:
1780 return SPIRV::OpSLessThan;
1781 case CmpInst::ICMP_UGE:
1782 return SPIRV::OpUGreaterThanEqual;
1783 case CmpInst::ICMP_UGT:
1784 return SPIRV::OpUGreaterThan;
1785 case CmpInst::ICMP_ULE:
1786 return SPIRV::OpULessThanEqual;
1787 case CmpInst::ICMP_ULT:
1788 return SPIRV::OpULessThan;
1789 default:
1790 llvm_unreachable("Unknown predicate type for ICmp");
1791 }
1792}
1793
1794static unsigned getPtrCmpOpcode(unsigned Pred) {
1795 switch (static_cast<CmpInst::Predicate>(Pred)) {
1796 case CmpInst::ICMP_EQ:
1797 return SPIRV::OpPtrEqual;
1798 case CmpInst::ICMP_NE:
1799 return SPIRV::OpPtrNotEqual;
1800 default:
1801 llvm_unreachable("Unknown predicate type for pointer comparison");
1802 }
1803}
1804
1805// Return the logical operation, or abort if none exists.
1806static unsigned getBoolCmpOpcode(unsigned PredNum) {
1807 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1808 switch (Pred) {
1809 case CmpInst::ICMP_EQ:
1810 return SPIRV::OpLogicalEqual;
1811 case CmpInst::ICMP_NE:
1812 return SPIRV::OpLogicalNotEqual;
1813 default:
1814 llvm_unreachable("Unknown predicate type for Bool comparison");
1815 }
1816}
1817
1818static APFloat getZeroFP(const Type *LLVMFloatTy) {
1819 if (!LLVMFloatTy)
1820 return APFloat::getZero(Sem: APFloat::IEEEsingle());
1821 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1822 case Type::HalfTyID:
1823 return APFloat::getZero(Sem: APFloat::IEEEhalf());
1824 default:
1825 case Type::FloatTyID:
1826 return APFloat::getZero(Sem: APFloat::IEEEsingle());
1827 case Type::DoubleTyID:
1828 return APFloat::getZero(Sem: APFloat::IEEEdouble());
1829 }
1830}
1831
1832static APFloat getOneFP(const Type *LLVMFloatTy) {
1833 if (!LLVMFloatTy)
1834 return APFloat::getOne(Sem: APFloat::IEEEsingle());
1835 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1836 case Type::HalfTyID:
1837 return APFloat::getOne(Sem: APFloat::IEEEhalf());
1838 default:
1839 case Type::FloatTyID:
1840 return APFloat::getOne(Sem: APFloat::IEEEsingle());
1841 case Type::DoubleTyID:
1842 return APFloat::getOne(Sem: APFloat::IEEEdouble());
1843 }
1844}
1845
1846bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
1847 const SPIRVType *ResType,
1848 MachineInstr &I,
1849 unsigned OpAnyOrAll) const {
1850 assert(I.getNumOperands() == 3);
1851 assert(I.getOperand(2).isReg());
1852 MachineBasicBlock &BB = *I.getParent();
1853 Register InputRegister = I.getOperand(i: 2).getReg();
1854 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
1855
1856 if (!InputType)
1857 report_fatal_error(reason: "Input Type could not be determined.");
1858
1859 bool IsBoolTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeBool);
1860 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
1861 if (IsBoolTy && !IsVectorTy) {
1862 assert(ResVReg == I.getOperand(0).getReg());
1863 return BuildCOPY(DestReg: ResVReg, SrcReg: InputRegister, I);
1864 }
1865
1866 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
1867 unsigned SpirvNotEqualId =
1868 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
1869 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
1870 SPIRVType *SpvBoolTy = SpvBoolScalarTy;
1871 Register NotEqualReg = ResVReg;
1872
1873 if (IsVectorTy) {
1874 NotEqualReg =
1875 IsBoolTy ? InputRegister
1876 : createVirtualRegister(SpvType: SpvBoolTy, GR: &GR, MRI, MF: MRI->getMF());
1877 const unsigned NumElts = InputType->getOperand(i: 2).getImm();
1878 SpvBoolTy = GR.getOrCreateSPIRVVectorType(BaseType: SpvBoolTy, NumElements: NumElts, I, TII);
1879 }
1880
1881 bool Result = true;
1882 if (!IsBoolTy) {
1883 Register ConstZeroReg =
1884 IsFloatTy ? buildZerosValF(ResType: InputType, I) : buildZerosVal(ResType: InputType, I);
1885
1886 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SpirvNotEqualId))
1887 .addDef(RegNo: NotEqualReg)
1888 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolTy))
1889 .addUse(RegNo: InputRegister)
1890 .addUse(RegNo: ConstZeroReg)
1891 .constrainAllUses(TII, TRI, RBI);
1892 }
1893
1894 if (!IsVectorTy)
1895 return Result;
1896
1897 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpAnyOrAll))
1898 .addDef(RegNo: ResVReg)
1899 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolScalarTy))
1900 .addUse(RegNo: NotEqualReg)
1901 .constrainAllUses(TII, TRI, RBI);
1902}
1903
1904bool SPIRVInstructionSelector::selectAll(Register ResVReg,
1905 const SPIRVType *ResType,
1906 MachineInstr &I) const {
1907 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAll);
1908}
1909
1910bool SPIRVInstructionSelector::selectAny(Register ResVReg,
1911 const SPIRVType *ResType,
1912 MachineInstr &I) const {
1913 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAny);
1914}
1915
1916// Select the OpDot instruction for the given float dot
1917bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
1918 const SPIRVType *ResType,
1919 MachineInstr &I) const {
1920 assert(I.getNumOperands() == 4);
1921 assert(I.getOperand(2).isReg());
1922 assert(I.getOperand(3).isReg());
1923
1924 [[maybe_unused]] SPIRVType *VecType =
1925 GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 2).getReg());
1926
1927 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
1928 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
1929 "dot product requires a vector of at least 2 components");
1930
1931 [[maybe_unused]] SPIRVType *EltType =
1932 GR.getSPIRVTypeForVReg(VReg: VecType->getOperand(i: 1).getReg());
1933
1934 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
1935
1936 MachineBasicBlock &BB = *I.getParent();
1937 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDot))
1938 .addDef(RegNo: ResVReg)
1939 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1940 .addUse(RegNo: I.getOperand(i: 2).getReg())
1941 .addUse(RegNo: I.getOperand(i: 3).getReg())
1942 .constrainAllUses(TII, TRI, RBI);
1943}
1944
1945bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
1946 const SPIRVType *ResType,
1947 MachineInstr &I,
1948 bool Signed) const {
1949 assert(I.getNumOperands() == 4);
1950 assert(I.getOperand(2).isReg());
1951 assert(I.getOperand(3).isReg());
1952 MachineBasicBlock &BB = *I.getParent();
1953
1954 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
1955 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
1956 .addDef(RegNo: ResVReg)
1957 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1958 .addUse(RegNo: I.getOperand(i: 2).getReg())
1959 .addUse(RegNo: I.getOperand(i: 3).getReg())
1960 .constrainAllUses(TII, TRI, RBI);
1961}
1962
1963// Since pre-1.6 SPIRV has no integer dot implementation,
1964// expand by piecewise multiplying and adding the results
1965bool SPIRVInstructionSelector::selectIntegerDotExpansion(
1966 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
1967 assert(I.getNumOperands() == 4);
1968 assert(I.getOperand(2).isReg());
1969 assert(I.getOperand(3).isReg());
1970 MachineBasicBlock &BB = *I.getParent();
1971
1972 // Multiply the vectors, then sum the results
1973 Register Vec0 = I.getOperand(i: 2).getReg();
1974 Register Vec1 = I.getOperand(i: 3).getReg();
1975 Register TmpVec = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
1976 SPIRVType *VecType = GR.getSPIRVTypeForVReg(VReg: Vec0);
1977
1978 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulV))
1979 .addDef(RegNo: TmpVec)
1980 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VecType))
1981 .addUse(RegNo: Vec0)
1982 .addUse(RegNo: Vec1)
1983 .constrainAllUses(TII, TRI, RBI);
1984
1985 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
1986 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
1987 "dot product requires a vector of at least 2 components");
1988
1989 Register Res = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
1990 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1991 .addDef(RegNo: Res)
1992 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1993 .addUse(RegNo: TmpVec)
1994 .addImm(Val: 0)
1995 .constrainAllUses(TII, TRI, RBI);
1996
1997 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(Type: VecType); i++) {
1998 Register Elt = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
1999
2000 Result &=
2001 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2002 .addDef(RegNo: Elt)
2003 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2004 .addUse(RegNo: TmpVec)
2005 .addImm(Val: i)
2006 .constrainAllUses(TII, TRI, RBI);
2007
2008 Register Sum = i < GR.getScalarOrVectorComponentCount(Type: VecType) - 1
2009 ? MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType))
2010 : ResVReg;
2011
2012 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2013 .addDef(RegNo: Sum)
2014 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2015 .addUse(RegNo: Res)
2016 .addUse(RegNo: Elt)
2017 .constrainAllUses(TII, TRI, RBI);
2018 Res = Sum;
2019 }
2020
2021 return Result;
2022}
2023
2024template <bool Signed>
2025bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
2026 const SPIRVType *ResType,
2027 MachineInstr &I) const {
2028 assert(I.getNumOperands() == 5);
2029 assert(I.getOperand(2).isReg());
2030 assert(I.getOperand(3).isReg());
2031 assert(I.getOperand(4).isReg());
2032 MachineBasicBlock &BB = *I.getParent();
2033
2034 Register Acc = I.getOperand(i: 2).getReg();
2035 Register X = I.getOperand(i: 3).getReg();
2036 Register Y = I.getOperand(i: 4).getReg();
2037
2038 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2039 Register Dot = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2040 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2041 .addDef(RegNo: Dot)
2042 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2043 .addUse(RegNo: X)
2044 .addUse(RegNo: Y)
2045 .constrainAllUses(TII, TRI, RBI);
2046
2047 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2048 .addDef(RegNo: ResVReg)
2049 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2050 .addUse(RegNo: Dot)
2051 .addUse(RegNo: Acc)
2052 .constrainAllUses(TII, TRI, RBI);
2053}
2054
2055// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
2056// extract the elements of the packed inputs, multiply them and add the result
2057// to the accumulator.
2058template <bool Signed>
2059bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
2060 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2061 assert(I.getNumOperands() == 5);
2062 assert(I.getOperand(2).isReg());
2063 assert(I.getOperand(3).isReg());
2064 assert(I.getOperand(4).isReg());
2065 MachineBasicBlock &BB = *I.getParent();
2066
2067 bool Result = true;
2068
2069 Register Acc = I.getOperand(i: 2).getReg();
2070 Register X = I.getOperand(i: 3).getReg();
2071 Register Y = I.getOperand(i: 4).getReg();
2072
2073 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII);
2074 auto ExtractOp =
2075 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
2076
2077 bool ZeroAsNull = !STI.isShader();
2078 // Extract the i8 element, multiply and add it to the accumulator
2079 for (unsigned i = 0; i < 4; i++) {
2080 // A[i]
2081 Register AElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2082 Result &=
2083 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2084 .addDef(RegNo: AElt)
2085 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2086 .addUse(RegNo: X)
2087 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2088 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2089 .constrainAllUses(TII, TRI, RBI);
2090
2091 // B[i]
2092 Register BElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2093 Result &=
2094 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2095 .addDef(RegNo: BElt)
2096 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2097 .addUse(RegNo: Y)
2098 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2099 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2100 .constrainAllUses(TII, TRI, RBI);
2101
2102 // A[i] * B[i]
2103 Register Mul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2104 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulS))
2105 .addDef(RegNo: Mul)
2106 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2107 .addUse(RegNo: AElt)
2108 .addUse(RegNo: BElt)
2109 .constrainAllUses(TII, TRI, RBI);
2110
2111 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
2112 Register MaskMul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2113 Result &=
2114 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2115 .addDef(RegNo: MaskMul)
2116 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2117 .addUse(RegNo: Mul)
2118 .addUse(RegNo: GR.getOrCreateConstInt(Val: 0, I, SpvType: EltType, TII, ZeroAsNull))
2119 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2120 .constrainAllUses(TII, TRI, RBI);
2121
2122 // Acc = Acc + A[i] * B[i]
2123 Register Sum =
2124 i < 3 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass) : ResVReg;
2125 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2126 .addDef(RegNo: Sum)
2127 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2128 .addUse(RegNo: Acc)
2129 .addUse(RegNo: MaskMul)
2130 .constrainAllUses(TII, TRI, RBI);
2131
2132 Acc = Sum;
2133 }
2134
2135 return Result;
2136}
2137
2138/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
2139/// does not have a saturate builtin.
2140bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
2141 const SPIRVType *ResType,
2142 MachineInstr &I) const {
2143 assert(I.getNumOperands() == 3);
2144 assert(I.getOperand(2).isReg());
2145 MachineBasicBlock &BB = *I.getParent();
2146 Register VZero = buildZerosValF(ResType, I);
2147 Register VOne = buildOnesValF(ResType, I);
2148
2149 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
2150 .addDef(RegNo: ResVReg)
2151 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2152 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2153 .addImm(Val: GL::FClamp)
2154 .addUse(RegNo: I.getOperand(i: 2).getReg())
2155 .addUse(RegNo: VZero)
2156 .addUse(RegNo: VOne)
2157 .constrainAllUses(TII, TRI, RBI);
2158}
2159
2160bool SPIRVInstructionSelector::selectSign(Register ResVReg,
2161 const SPIRVType *ResType,
2162 MachineInstr &I) const {
2163 assert(I.getNumOperands() == 3);
2164 assert(I.getOperand(2).isReg());
2165 MachineBasicBlock &BB = *I.getParent();
2166 Register InputRegister = I.getOperand(i: 2).getReg();
2167 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2168 auto &DL = I.getDebugLoc();
2169
2170 if (!InputType)
2171 report_fatal_error(reason: "Input Type could not be determined.");
2172
2173 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2174
2175 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(Type: InputType);
2176 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2177
2178 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
2179
2180 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
2181 Register SignReg = NeedsConversion
2182 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass)
2183 : ResVReg;
2184
2185 bool Result =
2186 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpExtInst))
2187 .addDef(RegNo: SignReg)
2188 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: InputType))
2189 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2190 .addImm(Val: SignOpcode)
2191 .addUse(RegNo: InputRegister)
2192 .constrainAllUses(TII, TRI, RBI);
2193
2194 if (NeedsConversion) {
2195 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
2196 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: ConvertOpcode))
2197 .addDef(RegNo: ResVReg)
2198 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2199 .addUse(RegNo: SignReg)
2200 .constrainAllUses(TII, TRI, RBI);
2201 }
2202
2203 return Result;
2204}
2205
2206bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
2207 const SPIRVType *ResType,
2208 MachineInstr &I,
2209 unsigned Opcode) const {
2210 MachineBasicBlock &BB = *I.getParent();
2211 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2212
2213 auto BMI = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2214 .addDef(RegNo: ResVReg)
2215 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2216 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I,
2217 SpvType: IntTy, TII, ZeroAsNull: !STI.isShader()));
2218
2219 for (unsigned J = 2; J < I.getNumOperands(); J++) {
2220 BMI.addUse(RegNo: I.getOperand(i: J).getReg());
2221 }
2222
2223 return BMI.constrainAllUses(TII, TRI, RBI);
2224}
2225
2226bool SPIRVInstructionSelector::selectWaveActiveCountBits(
2227 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2228
2229 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2230 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(BaseType: IntTy, NumElements: 4, I, TII);
2231 Register BallotReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BallotType));
2232 bool Result = selectWaveOpInst(ResVReg: BallotReg, ResType: BallotType, I,
2233 Opcode: SPIRV::OpGroupNonUniformBallot);
2234
2235 MachineBasicBlock &BB = *I.getParent();
2236 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(),
2237 MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2238 .addDef(RegNo: ResVReg)
2239 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2240 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy,
2241 TII, ZeroAsNull: !STI.isShader()))
2242 .addImm(Val: SPIRV::GroupOperation::Reduce)
2243 .addUse(RegNo: BallotReg)
2244 .constrainAllUses(TII, TRI, RBI);
2245
2246 return Result;
2247}
2248
2249bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
2250 const SPIRVType *ResType,
2251 MachineInstr &I,
2252 bool IsUnsigned) const {
2253 assert(I.getNumOperands() == 3);
2254 assert(I.getOperand(2).isReg());
2255 MachineBasicBlock &BB = *I.getParent();
2256 Register InputRegister = I.getOperand(i: 2).getReg();
2257 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2258
2259 if (!InputType)
2260 report_fatal_error(reason: "Input Type could not be determined.");
2261
2262 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2263 // Retreive the operation to use based on input type
2264 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2265 auto IntegerOpcodeType =
2266 IsUnsigned ? SPIRV::OpGroupNonUniformUMax : SPIRV::OpGroupNonUniformSMax;
2267 auto Opcode = IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntegerOpcodeType;
2268 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2269 .addDef(RegNo: ResVReg)
2270 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2271 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2272 ZeroAsNull: !STI.isShader()))
2273 .addImm(Val: SPIRV::GroupOperation::Reduce)
2274 .addUse(RegNo: I.getOperand(i: 2).getReg())
2275 .constrainAllUses(TII, TRI, RBI);
2276}
2277
2278bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
2279 const SPIRVType *ResType,
2280 MachineInstr &I) const {
2281 assert(I.getNumOperands() == 3);
2282 assert(I.getOperand(2).isReg());
2283 MachineBasicBlock &BB = *I.getParent();
2284 Register InputRegister = I.getOperand(i: 2).getReg();
2285 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2286
2287 if (!InputType)
2288 report_fatal_error(reason: "Input Type could not be determined.");
2289
2290 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2291 // Retreive the operation to use based on input type
2292 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2293 auto Opcode =
2294 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd : SPIRV::OpGroupNonUniformIAdd;
2295 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2296 .addDef(RegNo: ResVReg)
2297 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2298 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2299 ZeroAsNull: !STI.isShader()))
2300 .addImm(Val: SPIRV::GroupOperation::Reduce)
2301 .addUse(RegNo: I.getOperand(i: 2).getReg());
2302}
2303
2304bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
2305 const SPIRVType *ResType,
2306 MachineInstr &I) const {
2307 MachineBasicBlock &BB = *I.getParent();
2308 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitReverse))
2309 .addDef(RegNo: ResVReg)
2310 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2311 .addUse(RegNo: I.getOperand(i: 1).getReg())
2312 .constrainAllUses(TII, TRI, RBI);
2313}
2314
2315bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
2316 const SPIRVType *ResType,
2317 MachineInstr &I) const {
2318 // There is no way to implement `freeze` correctly without support on SPIR-V
2319 // standard side, but we may at least address a simple (static) case when
2320 // undef/poison value presence is obvious. The main benefit of even
2321 // incomplete `freeze` support is preventing of translation from crashing due
2322 // to lack of support on legalization and instruction selection steps.
2323 if (!I.getOperand(i: 0).isReg() || !I.getOperand(i: 1).isReg())
2324 return false;
2325 Register OpReg = I.getOperand(i: 1).getReg();
2326 if (MachineInstr *Def = MRI->getVRegDef(Reg: OpReg)) {
2327 if (Def->getOpcode() == TargetOpcode::COPY)
2328 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
2329 Register Reg;
2330 switch (Def->getOpcode()) {
2331 case SPIRV::ASSIGN_TYPE:
2332 if (MachineInstr *AssignToDef =
2333 MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())) {
2334 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
2335 Reg = Def->getOperand(i: 2).getReg();
2336 }
2337 break;
2338 case SPIRV::OpUndef:
2339 Reg = Def->getOperand(i: 1).getReg();
2340 break;
2341 }
2342 unsigned DestOpCode;
2343 if (Reg.isValid()) {
2344 DestOpCode = SPIRV::OpConstantNull;
2345 } else {
2346 DestOpCode = TargetOpcode::COPY;
2347 Reg = OpReg;
2348 }
2349 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DestOpCode))
2350 .addDef(RegNo: I.getOperand(i: 0).getReg())
2351 .addUse(RegNo: Reg)
2352 .constrainAllUses(TII, TRI, RBI);
2353 }
2354 return false;
2355}
2356
2357bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
2358 const SPIRVType *ResType,
2359 MachineInstr &I) const {
2360 unsigned N = 0;
2361 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2362 N = GR.getScalarOrVectorComponentCount(Type: ResType);
2363 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2364 N = getArrayComponentCount(MRI, ResType);
2365 else
2366 report_fatal_error(reason: "Cannot select G_BUILD_VECTOR with a non-vector result");
2367 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
2368 report_fatal_error(reason: "G_BUILD_VECTOR and the result type are inconsistent");
2369
2370 // check if we may construct a constant vector
2371 bool IsConst = true;
2372 for (unsigned i = I.getNumExplicitDefs();
2373 i < I.getNumExplicitOperands() && IsConst; ++i)
2374 if (!isConstReg(MRI, OpReg: I.getOperand(i).getReg()))
2375 IsConst = false;
2376
2377 if (!IsConst && N < 2)
2378 report_fatal_error(
2379 reason: "There must be at least two constituent operands in a vector");
2380
2381 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
2382 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2383 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
2384 : SPIRV::OpCompositeConstruct))
2385 .addDef(RegNo: ResVReg)
2386 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2387 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
2388 MIB.addUse(RegNo: I.getOperand(i).getReg());
2389 return MIB.constrainAllUses(TII, TRI, RBI);
2390}
2391
2392bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
2393 const SPIRVType *ResType,
2394 MachineInstr &I) const {
2395 unsigned N = 0;
2396 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2397 N = GR.getScalarOrVectorComponentCount(Type: ResType);
2398 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2399 N = getArrayComponentCount(MRI, ResType);
2400 else
2401 report_fatal_error(reason: "Cannot select G_SPLAT_VECTOR with a non-vector result");
2402
2403 unsigned OpIdx = I.getNumExplicitDefs();
2404 if (!I.getOperand(i: OpIdx).isReg())
2405 report_fatal_error(reason: "Unexpected argument in G_SPLAT_VECTOR");
2406
2407 // check if we may construct a constant vector
2408 Register OpReg = I.getOperand(i: OpIdx).getReg();
2409 bool IsConst = isConstReg(MRI, OpReg);
2410
2411 if (!IsConst && N < 2)
2412 report_fatal_error(
2413 reason: "There must be at least two constituent operands in a vector");
2414
2415 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
2416 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2417 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
2418 : SPIRV::OpCompositeConstruct))
2419 .addDef(RegNo: ResVReg)
2420 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2421 for (unsigned i = 0; i < N; ++i)
2422 MIB.addUse(RegNo: OpReg);
2423 return MIB.constrainAllUses(TII, TRI, RBI);
2424}
2425
2426bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
2427 const SPIRVType *ResType,
2428 MachineInstr &I) const {
2429
2430 unsigned Opcode;
2431
2432 if (STI.canUseExtension(
2433 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
2434 STI.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6))) {
2435 Opcode = SPIRV::OpDemoteToHelperInvocation;
2436 } else {
2437 Opcode = SPIRV::OpKill;
2438 // OpKill must be the last operation of any basic block.
2439 if (MachineInstr *NextI = I.getNextNode()) {
2440 GR.invalidateMachineInstr(MI: NextI);
2441 NextI->removeFromParent();
2442 }
2443 }
2444
2445 MachineBasicBlock &BB = *I.getParent();
2446 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2447 .constrainAllUses(TII, TRI, RBI);
2448}
2449
2450bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
2451 const SPIRVType *ResType,
2452 unsigned CmpOpc,
2453 MachineInstr &I) const {
2454 Register Cmp0 = I.getOperand(i: 2).getReg();
2455 Register Cmp1 = I.getOperand(i: 3).getReg();
2456 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
2457 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
2458 "CMP operands should have the same type");
2459 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CmpOpc))
2460 .addDef(RegNo: ResVReg)
2461 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2462 .addUse(RegNo: Cmp0)
2463 .addUse(RegNo: Cmp1)
2464 .constrainAllUses(TII, TRI, RBI);
2465}
2466
2467bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
2468 const SPIRVType *ResType,
2469 MachineInstr &I) const {
2470 auto Pred = I.getOperand(i: 1).getPredicate();
2471 unsigned CmpOpc;
2472
2473 Register CmpOperand = I.getOperand(i: 2).getReg();
2474 if (GR.isScalarOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypePointer))
2475 CmpOpc = getPtrCmpOpcode(Pred);
2476 else if (GR.isScalarOrVectorOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypeBool))
2477 CmpOpc = getBoolCmpOpcode(PredNum: Pred);
2478 else
2479 CmpOpc = getICmpOpcode(PredNum: Pred);
2480 return selectCmp(ResVReg, ResType, CmpOpc, I);
2481}
2482
2483std::pair<Register, bool>
2484SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
2485 const SPIRVType *ResType) const {
2486 Type *LLVMTy = IntegerType::get(C&: GR.CurMF->getFunction().getContext(), NumBits: 32);
2487 const SPIRVType *SpvI32Ty =
2488 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2489 // Find a constant in DT or build a new one.
2490 auto ConstInt = ConstantInt::get(Ty: LLVMTy, V: Val);
2491 Register NewReg = GR.find(V: ConstInt, MF: GR.CurMF);
2492 bool Result = true;
2493 if (!NewReg.isValid()) {
2494 NewReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2495 MachineBasicBlock &BB = *I.getParent();
2496 MachineInstr *MI =
2497 Val == 0
2498 ? BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
2499 .addDef(RegNo: NewReg)
2500 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
2501 : BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantI))
2502 .addDef(RegNo: NewReg)
2503 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
2504 .addImm(Val: APInt(32, Val).getZExtValue());
2505 Result &= constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
2506 GR.add(V: ConstInt, MI);
2507 }
2508 return {NewReg, Result};
2509}
2510
2511bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
2512 const SPIRVType *ResType,
2513 MachineInstr &I) const {
2514 unsigned CmpOp = getFCmpOpcode(PredNum: I.getOperand(i: 1).getPredicate());
2515 return selectCmp(ResVReg, ResType, CmpOpc: CmpOp, I);
2516}
2517
2518Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
2519 MachineInstr &I) const {
2520 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2521 bool ZeroAsNull = !STI.isShader();
2522 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2523 return GR.getOrCreateConstVector(Val: 0UL, I, SpvType: ResType, TII, ZeroAsNull);
2524 return GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
2525}
2526
2527Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
2528 MachineInstr &I) const {
2529 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2530 bool ZeroAsNull = !STI.isShader();
2531 APFloat VZero = getZeroFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
2532 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2533 return GR.getOrCreateConstVector(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
2534 return GR.getOrCreateConstFP(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
2535}
2536
2537Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType,
2538 MachineInstr &I) const {
2539 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2540 bool ZeroAsNull = !STI.isShader();
2541 APFloat VOne = getOneFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
2542 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2543 return GR.getOrCreateConstVector(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
2544 return GR.getOrCreateConstFP(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
2545}
2546
2547Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
2548 const SPIRVType *ResType,
2549 MachineInstr &I) const {
2550 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2551 APInt One =
2552 AllOnes ? APInt::getAllOnes(numBits: BitWidth) : APInt::getOneBitSet(numBits: BitWidth, BitNo: 0);
2553 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2554 return GR.getOrCreateConstVector(Val: One.getZExtValue(), I, SpvType: ResType, TII);
2555 return GR.getOrCreateConstInt(Val: One.getZExtValue(), I, SpvType: ResType, TII);
2556}
2557
2558bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
2559 const SPIRVType *ResType,
2560 MachineInstr &I,
2561 bool IsSigned) const {
2562 // To extend a bool, we need to use OpSelect between constants.
2563 Register ZeroReg = buildZerosVal(ResType, I);
2564 Register OneReg = buildOnesVal(AllOnes: IsSigned, ResType, I);
2565 bool IsScalarBool =
2566 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
2567 unsigned Opcode =
2568 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
2569 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2570 .addDef(RegNo: ResVReg)
2571 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2572 .addUse(RegNo: I.getOperand(i: 1).getReg())
2573 .addUse(RegNo: OneReg)
2574 .addUse(RegNo: ZeroReg)
2575 .constrainAllUses(TII, TRI, RBI);
2576}
2577
2578bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
2579 const SPIRVType *ResType,
2580 MachineInstr &I, bool IsSigned,
2581 unsigned Opcode) const {
2582 Register SrcReg = I.getOperand(i: 1).getReg();
2583 // We can convert bool value directly to float type without OpConvert*ToF,
2584 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
2585 if (GR.isScalarOrVectorOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool)) {
2586 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2587 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
2588 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
2589 const unsigned NumElts = ResType->getOperand(i: 2).getImm();
2590 TmpType = GR.getOrCreateSPIRVVectorType(BaseType: TmpType, NumElements: NumElts, I, TII);
2591 }
2592 SrcReg = createVirtualRegister(SpvType: TmpType, GR: &GR, MRI, MF: MRI->getMF());
2593 selectSelect(ResVReg: SrcReg, ResType: TmpType, I, IsSigned: false);
2594 }
2595 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {SrcReg}, Opcode);
2596}
2597
2598bool SPIRVInstructionSelector::selectExt(Register ResVReg,
2599 const SPIRVType *ResType,
2600 MachineInstr &I, bool IsSigned) const {
2601 Register SrcReg = I.getOperand(i: 1).getReg();
2602 if (GR.isScalarOrVectorOfType(VReg: SrcReg, TypeOpcode: SPIRV::OpTypeBool))
2603 return selectSelect(ResVReg, ResType, I, IsSigned);
2604
2605 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
2606 if (SrcType == ResType)
2607 return BuildCOPY(DestReg: ResVReg, SrcReg, I);
2608
2609 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
2610 return selectUnOp(ResVReg, ResType, I, Opcode);
2611}
2612
2613bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
2614 const SPIRVType *ResType,
2615 MachineInstr &I,
2616 bool IsSigned) const {
2617 MachineIRBuilder MIRBuilder(I);
2618 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2619 MachineBasicBlock &BB = *I.getParent();
2620 // Ensure we have bool.
2621 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2622 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
2623 if (N > 1)
2624 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
2625 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
2626 // Build less-than-equal and less-than.
2627 // TODO: replace with one-liner createVirtualRegister() from
2628 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
2629 Register IsLessEqReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2630 MRI->setType(VReg: IsLessEqReg, Ty: LLT::scalar(SizeInBits: 64));
2631 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessEqReg, MF: MIRBuilder.getMF());
2632 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(),
2633 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThanEqual
2634 : SPIRV::OpULessThanEqual))
2635 .addDef(RegNo: IsLessEqReg)
2636 .addUse(RegNo: BoolTypeReg)
2637 .addUse(RegNo: I.getOperand(i: 1).getReg())
2638 .addUse(RegNo: I.getOperand(i: 2).getReg())
2639 .constrainAllUses(TII, TRI, RBI);
2640 Register IsLessReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2641 MRI->setType(VReg: IsLessReg, Ty: LLT::scalar(SizeInBits: 64));
2642 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessReg, MF: MIRBuilder.getMF());
2643 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(),
2644 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
2645 .addDef(RegNo: IsLessReg)
2646 .addUse(RegNo: BoolTypeReg)
2647 .addUse(RegNo: I.getOperand(i: 1).getReg())
2648 .addUse(RegNo: I.getOperand(i: 2).getReg())
2649 .constrainAllUses(TII, TRI, RBI);
2650 // Build selects.
2651 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
2652 Register NegOneOrZeroReg =
2653 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2654 MRI->setType(VReg: NegOneOrZeroReg, Ty: LLT::scalar(SizeInBits: 64));
2655 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: NegOneOrZeroReg, MF: MIRBuilder.getMF());
2656 unsigned SelectOpcode =
2657 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
2658 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
2659 .addDef(RegNo: NegOneOrZeroReg)
2660 .addUse(RegNo: ResTypeReg)
2661 .addUse(RegNo: IsLessReg)
2662 .addUse(RegNo: buildOnesVal(AllOnes: true, ResType, I)) // -1
2663 .addUse(RegNo: buildZerosVal(ResType, I))
2664 .constrainAllUses(TII, TRI, RBI);
2665 return Result & BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
2666 .addDef(RegNo: ResVReg)
2667 .addUse(RegNo: ResTypeReg)
2668 .addUse(RegNo: IsLessEqReg)
2669 .addUse(RegNo: NegOneOrZeroReg) // -1 or 0
2670 .addUse(RegNo: buildOnesVal(AllOnes: false, ResType, I))
2671 .constrainAllUses(TII, TRI, RBI);
2672}
2673
2674bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
2675 Register ResVReg,
2676 MachineInstr &I,
2677 const SPIRVType *IntTy,
2678 const SPIRVType *BoolTy) const {
2679 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
2680 Register BitIntReg = createVirtualRegister(SpvType: IntTy, GR: &GR, MRI, MF: MRI->getMF());
2681 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
2682 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
2683 Register Zero = buildZerosVal(ResType: IntTy, I);
2684 Register One = buildOnesVal(AllOnes: false, ResType: IntTy, I);
2685 MachineBasicBlock &BB = *I.getParent();
2686 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2687 .addDef(RegNo: BitIntReg)
2688 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntTy))
2689 .addUse(RegNo: IntReg)
2690 .addUse(RegNo: One)
2691 .constrainAllUses(TII, TRI, RBI);
2692 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
2693 .addDef(RegNo: ResVReg)
2694 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
2695 .addUse(RegNo: BitIntReg)
2696 .addUse(RegNo: Zero)
2697 .constrainAllUses(TII, TRI, RBI);
2698}
2699
2700bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
2701 const SPIRVType *ResType,
2702 MachineInstr &I) const {
2703 Register IntReg = I.getOperand(i: 1).getReg();
2704 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(VReg: IntReg);
2705 if (GR.isScalarOrVectorOfType(VReg: ResVReg, TypeOpcode: SPIRV::OpTypeBool))
2706 return selectIntToBool(IntReg, ResVReg, I, IntTy: ArgType, BoolTy: ResType);
2707 if (ArgType == ResType)
2708 return BuildCOPY(DestReg: ResVReg, SrcReg: IntReg, I);
2709 bool IsSigned = GR.isScalarOrVectorSigned(Type: ResType);
2710 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
2711 return selectUnOp(ResVReg, ResType, I, Opcode);
2712}
2713
2714bool SPIRVInstructionSelector::selectConst(Register ResVReg,
2715 const SPIRVType *ResType,
2716 MachineInstr &I) const {
2717 unsigned Opcode = I.getOpcode();
2718 unsigned TpOpcode = ResType->getOpcode();
2719 Register Reg;
2720 if (TpOpcode == SPIRV::OpTypePointer || TpOpcode == SPIRV::OpTypeEvent) {
2721 assert(Opcode == TargetOpcode::G_CONSTANT &&
2722 I.getOperand(1).getCImm()->isZero());
2723 MachineBasicBlock &DepMBB = I.getMF()->front();
2724 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
2725 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, SpvType: ResType);
2726 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
2727 Reg = GR.getOrCreateConstFP(Val: I.getOperand(i: 1).getFPImm()->getValue(), I,
2728 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
2729 } else {
2730 Reg = GR.getOrCreateConstInt(Val: I.getOperand(i: 1).getCImm()->getZExtValue(), I,
2731 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
2732 }
2733 return Reg == ResVReg ? true : BuildCOPY(DestReg: ResVReg, SrcReg: Reg, I);
2734}
2735
2736bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
2737 const SPIRVType *ResType,
2738 MachineInstr &I) const {
2739 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
2740 .addDef(RegNo: ResVReg)
2741 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2742 .constrainAllUses(TII, TRI, RBI);
2743}
2744
2745bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
2746 const SPIRVType *ResType,
2747 MachineInstr &I) const {
2748 MachineBasicBlock &BB = *I.getParent();
2749 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2750 .addDef(RegNo: ResVReg)
2751 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2752 // object to insert
2753 .addUse(RegNo: I.getOperand(i: 3).getReg())
2754 // composite to insert into
2755 .addUse(RegNo: I.getOperand(i: 2).getReg());
2756 for (unsigned i = 4; i < I.getNumOperands(); i++)
2757 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
2758 return MIB.constrainAllUses(TII, TRI, RBI);
2759}
2760
2761bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
2762 const SPIRVType *ResType,
2763 MachineInstr &I) const {
2764 MachineBasicBlock &BB = *I.getParent();
2765 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2766 .addDef(RegNo: ResVReg)
2767 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2768 .addUse(RegNo: I.getOperand(i: 2).getReg());
2769 for (unsigned i = 3; i < I.getNumOperands(); i++)
2770 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
2771 return MIB.constrainAllUses(TII, TRI, RBI);
2772}
2773
2774bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
2775 const SPIRVType *ResType,
2776 MachineInstr &I) const {
2777 if (getImm(MO: I.getOperand(i: 4), MRI))
2778 return selectInsertVal(ResVReg, ResType, I);
2779 MachineBasicBlock &BB = *I.getParent();
2780 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorInsertDynamic))
2781 .addDef(RegNo: ResVReg)
2782 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2783 .addUse(RegNo: I.getOperand(i: 2).getReg())
2784 .addUse(RegNo: I.getOperand(i: 3).getReg())
2785 .addUse(RegNo: I.getOperand(i: 4).getReg())
2786 .constrainAllUses(TII, TRI, RBI);
2787}
2788
2789bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
2790 const SPIRVType *ResType,
2791 MachineInstr &I) const {
2792 if (getImm(MO: I.getOperand(i: 3), MRI))
2793 return selectExtractVal(ResVReg, ResType, I);
2794 MachineBasicBlock &BB = *I.getParent();
2795 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorExtractDynamic))
2796 .addDef(RegNo: ResVReg)
2797 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2798 .addUse(RegNo: I.getOperand(i: 2).getReg())
2799 .addUse(RegNo: I.getOperand(i: 3).getReg())
2800 .constrainAllUses(TII, TRI, RBI);
2801}
2802
2803bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
2804 const SPIRVType *ResType,
2805 MachineInstr &I) const {
2806 const bool IsGEPInBounds = I.getOperand(i: 2).getImm();
2807
2808 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
2809 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
2810 // we have to use Op[InBounds]AccessChain.
2811 const unsigned Opcode = STI.isLogicalSPIRV()
2812 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
2813 : SPIRV::OpAccessChain)
2814 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
2815 : SPIRV::OpPtrAccessChain);
2816
2817 auto Res = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2818 .addDef(RegNo: ResVReg)
2819 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2820 // Object to get a pointer to.
2821 .addUse(RegNo: I.getOperand(i: 3).getReg());
2822 // Adding indices.
2823 const unsigned StartingIndex =
2824 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
2825 ? 5
2826 : 4;
2827 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
2828 Res.addUse(RegNo: I.getOperand(i).getReg());
2829 return Res.constrainAllUses(TII, TRI, RBI);
2830}
2831
2832// Maybe wrap a value into OpSpecConstantOp
2833bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
2834 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
2835 bool Result = true;
2836 unsigned Lim = I.getNumExplicitOperands();
2837 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
2838 Register OpReg = I.getOperand(i).getReg();
2839 MachineInstr *OpDefine = MRI->getVRegDef(Reg: OpReg);
2840 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
2841 SmallPtrSet<SPIRVType *, 4> Visited;
2842 if (!OpDefine || !OpType || isConstReg(MRI, OpDef: OpDefine, Visited) ||
2843 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
2844 GR.isAggregateType(Type: OpType)) {
2845 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
2846 // by selectAddrSpaceCast()
2847 CompositeArgs.push_back(Elt: OpReg);
2848 continue;
2849 }
2850 MachineFunction *MF = I.getMF();
2851 Register WrapReg = GR.find(MI: OpDefine, MF);
2852 if (WrapReg.isValid()) {
2853 CompositeArgs.push_back(Elt: WrapReg);
2854 continue;
2855 }
2856 // Create a new register for the wrapper
2857 WrapReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: OpType));
2858 CompositeArgs.push_back(Elt: WrapReg);
2859 // Decorate the wrapper register and generate a new instruction
2860 MRI->setType(VReg: WrapReg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
2861 GR.assignSPIRVTypeToVReg(Type: OpType, VReg: WrapReg, MF: *MF);
2862 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2863 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
2864 .addDef(RegNo: WrapReg)
2865 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: OpType))
2866 .addImm(Val: static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
2867 .addUse(RegNo: OpReg);
2868 GR.add(Obj: OpDefine, MI: MIB);
2869 Result = MIB.constrainAllUses(TII, TRI, RBI);
2870 if (!Result)
2871 break;
2872 }
2873 return Result;
2874}
2875
2876bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
2877 const SPIRVType *ResType,
2878 MachineInstr &I) const {
2879 MachineBasicBlock &BB = *I.getParent();
2880 Intrinsic::ID IID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2881 switch (IID) {
2882 case Intrinsic::spv_load:
2883 return selectLoad(ResVReg, ResType, I);
2884 case Intrinsic::spv_store:
2885 return selectStore(I);
2886 case Intrinsic::spv_extractv:
2887 return selectExtractVal(ResVReg, ResType, I);
2888 case Intrinsic::spv_insertv:
2889 return selectInsertVal(ResVReg, ResType, I);
2890 case Intrinsic::spv_extractelt:
2891 return selectExtractElt(ResVReg, ResType, I);
2892 case Intrinsic::spv_insertelt:
2893 return selectInsertElt(ResVReg, ResType, I);
2894 case Intrinsic::spv_gep:
2895 return selectGEP(ResVReg, ResType, I);
2896 case Intrinsic::spv_unref_global:
2897 case Intrinsic::spv_init_global: {
2898 MachineInstr *MI = MRI->getVRegDef(Reg: I.getOperand(i: 1).getReg());
2899 MachineInstr *Init = I.getNumExplicitOperands() > 2
2900 ? MRI->getVRegDef(Reg: I.getOperand(i: 2).getReg())
2901 : nullptr;
2902 assert(MI);
2903 Register GVarVReg = MI->getOperand(i: 0).getReg();
2904 bool Res = selectGlobalValue(ResVReg: GVarVReg, I&: *MI, Init);
2905 // We violate SSA form by inserting OpVariable and still having a gMIR
2906 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
2907 // the duplicated definition.
2908 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
2909 GR.invalidateMachineInstr(MI);
2910 MI->removeFromParent();
2911 }
2912 return Res;
2913 }
2914 case Intrinsic::spv_undef: {
2915 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
2916 .addDef(RegNo: ResVReg)
2917 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2918 return MIB.constrainAllUses(TII, TRI, RBI);
2919 }
2920 case Intrinsic::spv_const_composite: {
2921 // If no values are attached, the composite is null constant.
2922 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
2923 SmallVector<Register> CompositeArgs;
2924 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
2925
2926 // skip type MD node we already used when generated assign.type for this
2927 if (!IsNull) {
2928 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
2929 return false;
2930 MachineIRBuilder MIR(I);
2931 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
2932 MIRBuilder&: MIR, Opcode: SPIRV::OpConstantComposite, MinWC: 3,
2933 ContinuedOpcode: SPIRV::OpConstantCompositeContinuedINTEL, Args: CompositeArgs, ReturnRegister: ResVReg,
2934 TypeID: GR.getSPIRVTypeID(SpirvType: ResType));
2935 for (auto *Instr : Instructions) {
2936 Instr->setDebugLoc(I.getDebugLoc());
2937 if (!constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI))
2938 return false;
2939 }
2940 return true;
2941 } else {
2942 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
2943 .addDef(RegNo: ResVReg)
2944 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2945 return MIB.constrainAllUses(TII, TRI, RBI);
2946 }
2947 }
2948 case Intrinsic::spv_assign_name: {
2949 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName));
2950 MIB.addUse(RegNo: I.getOperand(i: I.getNumExplicitDefs() + 1).getReg());
2951 for (unsigned i = I.getNumExplicitDefs() + 2;
2952 i < I.getNumExplicitOperands(); ++i) {
2953 MIB.addImm(Val: I.getOperand(i).getImm());
2954 }
2955 return MIB.constrainAllUses(TII, TRI, RBI);
2956 }
2957 case Intrinsic::spv_switch: {
2958 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSwitch));
2959 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
2960 if (I.getOperand(i).isReg())
2961 MIB.addReg(RegNo: I.getOperand(i).getReg());
2962 else if (I.getOperand(i).isCImm())
2963 addNumImm(Imm: I.getOperand(i).getCImm()->getValue(), MIB);
2964 else if (I.getOperand(i).isMBB())
2965 MIB.addMBB(MBB: I.getOperand(i).getMBB());
2966 else
2967 llvm_unreachable("Unexpected OpSwitch operand");
2968 }
2969 return MIB.constrainAllUses(TII, TRI, RBI);
2970 }
2971 case Intrinsic::spv_loop_merge: {
2972 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopMerge));
2973 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
2974 if (I.getOperand(i).isMBB())
2975 MIB.addMBB(MBB: I.getOperand(i).getMBB());
2976 else
2977 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
2978 }
2979 return MIB.constrainAllUses(TII, TRI, RBI);
2980 }
2981 case Intrinsic::spv_selection_merge: {
2982 auto MIB =
2983 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSelectionMerge));
2984 assert(I.getOperand(1).isMBB() &&
2985 "operand 1 to spv_selection_merge must be a basic block");
2986 MIB.addMBB(MBB: I.getOperand(i: 1).getMBB());
2987 MIB.addImm(Val: getSelectionOperandForImm(Imm: I.getOperand(i: 2).getImm()));
2988 return MIB.constrainAllUses(TII, TRI, RBI);
2989 }
2990 case Intrinsic::spv_cmpxchg:
2991 return selectAtomicCmpXchg(ResVReg, ResType, I);
2992 case Intrinsic::spv_unreachable:
2993 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUnreachable))
2994 .constrainAllUses(TII, TRI, RBI);
2995 case Intrinsic::spv_alloca:
2996 return selectFrameIndex(ResVReg, ResType, I);
2997 case Intrinsic::spv_alloca_array:
2998 return selectAllocaArray(ResVReg, ResType, I);
2999 case Intrinsic::spv_assume:
3000 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
3001 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAssumeTrueKHR))
3002 .addUse(RegNo: I.getOperand(i: 1).getReg())
3003 .constrainAllUses(TII, TRI, RBI);
3004 break;
3005 case Intrinsic::spv_expect:
3006 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
3007 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExpectKHR))
3008 .addDef(RegNo: ResVReg)
3009 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3010 .addUse(RegNo: I.getOperand(i: 2).getReg())
3011 .addUse(RegNo: I.getOperand(i: 3).getReg())
3012 .constrainAllUses(TII, TRI, RBI);
3013 break;
3014 case Intrinsic::arithmetic_fence:
3015 if (STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
3016 return BuildMI(BB, I, MIMD: I.getDebugLoc(),
3017 MCID: TII.get(Opcode: SPIRV::OpArithmeticFenceEXT))
3018 .addDef(RegNo: ResVReg)
3019 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3020 .addUse(RegNo: I.getOperand(i: 2).getReg())
3021 .constrainAllUses(TII, TRI, RBI);
3022 else
3023 return BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 2).getReg(), I);
3024 break;
3025 case Intrinsic::spv_thread_id:
3026 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
3027 // intrinsic in LLVM IR for SPIR-V backend.
3028 //
3029 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
3030 // `GlobalInvocationId` builtin variable
3031 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
3032 ResType, I);
3033 case Intrinsic::spv_thread_id_in_group:
3034 // The HLSL SV_GroupThreadId semantic is lowered to
3035 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
3036 //
3037 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
3038 // translated to a `LocalInvocationId` builtin variable
3039 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationId, ResVReg,
3040 ResType, I);
3041 case Intrinsic::spv_group_id:
3042 // The HLSL SV_GroupId semantic is lowered to
3043 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
3044 //
3045 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
3046 // builtin variable
3047 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
3048 I);
3049 case Intrinsic::spv_flattened_thread_id_in_group:
3050 // The HLSL SV_GroupIndex semantic is lowered to
3051 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
3052 // backend.
3053 //
3054 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
3055 // a `LocalInvocationIndex` builtin variable
3056 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
3057 ResType, I);
3058 case Intrinsic::spv_fdot:
3059 return selectFloatDot(ResVReg, ResType, I);
3060 case Intrinsic::spv_udot:
3061 case Intrinsic::spv_sdot:
3062 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3063 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3064 return selectIntegerDot(ResVReg, ResType, I,
3065 /*Signed=*/IID == Intrinsic::spv_sdot);
3066 return selectIntegerDotExpansion(ResVReg, ResType, I);
3067 case Intrinsic::spv_dot4add_i8packed:
3068 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3069 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3070 return selectDot4AddPacked<true>(ResVReg, ResType, I);
3071 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
3072 case Intrinsic::spv_dot4add_u8packed:
3073 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3074 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3075 return selectDot4AddPacked<false>(ResVReg, ResType, I);
3076 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
3077 case Intrinsic::spv_all:
3078 return selectAll(ResVReg, ResType, I);
3079 case Intrinsic::spv_any:
3080 return selectAny(ResVReg, ResType, I);
3081 case Intrinsic::spv_cross:
3082 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cross, GLInst: GL::Cross);
3083 case Intrinsic::spv_distance:
3084 return selectExtInst(ResVReg, ResType, I, CLInst: CL::distance, GLInst: GL::Distance);
3085 case Intrinsic::spv_lerp:
3086 return selectExtInst(ResVReg, ResType, I, CLInst: CL::mix, GLInst: GL::FMix);
3087 case Intrinsic::spv_length:
3088 return selectExtInst(ResVReg, ResType, I, CLInst: CL::length, GLInst: GL::Length);
3089 case Intrinsic::spv_degrees:
3090 return selectExtInst(ResVReg, ResType, I, CLInst: CL::degrees, GLInst: GL::Degrees);
3091 case Intrinsic::spv_faceforward:
3092 return selectExtInst(ResVReg, ResType, I, GLInst: GL::FaceForward);
3093 case Intrinsic::spv_frac:
3094 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fract, GLInst: GL::Fract);
3095 case Intrinsic::spv_normalize:
3096 return selectExtInst(ResVReg, ResType, I, CLInst: CL::normalize, GLInst: GL::Normalize);
3097 case Intrinsic::spv_reflect:
3098 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Reflect);
3099 case Intrinsic::spv_rsqrt:
3100 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rsqrt, GLInst: GL::InverseSqrt);
3101 case Intrinsic::spv_sign:
3102 return selectSign(ResVReg, ResType, I);
3103 case Intrinsic::spv_smoothstep:
3104 return selectExtInst(ResVReg, ResType, I, CLInst: CL::smoothstep, GLInst: GL::SmoothStep);
3105 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
3106 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
3107 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
3108 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
3109 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
3110 return selectFirstBitLow(ResVReg, ResType, I);
3111 case Intrinsic::spv_group_memory_barrier_with_group_sync: {
3112 bool Result = true;
3113 auto MemSemConstant =
3114 buildI32Constant(Val: SPIRV::MemorySemantics::SequentiallyConsistent, I);
3115 Register MemSemReg = MemSemConstant.first;
3116 Result &= MemSemConstant.second;
3117 auto ScopeConstant = buildI32Constant(Val: SPIRV::Scope::Workgroup, I);
3118 Register ScopeReg = ScopeConstant.first;
3119 Result &= ScopeConstant.second;
3120 MachineBasicBlock &BB = *I.getParent();
3121 return Result &&
3122 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpControlBarrier))
3123 .addUse(RegNo: ScopeReg)
3124 .addUse(RegNo: ScopeReg)
3125 .addUse(RegNo: MemSemReg)
3126 .constrainAllUses(TII, TRI, RBI);
3127 }
3128 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
3129 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 1).getReg();
3130 SPIRV::StorageClass::StorageClass ResSC =
3131 GR.getPointerStorageClass(Type: ResType);
3132 if (!isGenericCastablePtr(SC: ResSC))
3133 report_fatal_error(reason: "The target storage class is not castable from the "
3134 "Generic storage class");
3135 return BuildMI(BB, I, MIMD: I.getDebugLoc(),
3136 MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtrExplicit))
3137 .addDef(RegNo: ResVReg)
3138 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3139 .addUse(RegNo: PtrReg)
3140 .addImm(Val: ResSC)
3141 .constrainAllUses(TII, TRI, RBI);
3142 }
3143 case Intrinsic::spv_lifetime_start:
3144 case Intrinsic::spv_lifetime_end: {
3145 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
3146 : SPIRV::OpLifetimeStop;
3147 int64_t Size = I.getOperand(i: I.getNumExplicitDefs() + 1).getImm();
3148 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 2).getReg();
3149 if (Size == -1)
3150 Size = 0;
3151 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Op))
3152 .addUse(RegNo: PtrReg)
3153 .addImm(Val: Size)
3154 .constrainAllUses(TII, TRI, RBI);
3155 }
3156 case Intrinsic::spv_saturate:
3157 return selectSaturate(ResVReg, ResType, I);
3158 case Intrinsic::spv_nclamp:
3159 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fclamp, GLInst: GL::NClamp);
3160 case Intrinsic::spv_uclamp:
3161 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_clamp, GLInst: GL::UClamp);
3162 case Intrinsic::spv_sclamp:
3163 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_clamp, GLInst: GL::SClamp);
3164 case Intrinsic::spv_wave_active_countbits:
3165 return selectWaveActiveCountBits(ResVReg, ResType, I);
3166 case Intrinsic::spv_wave_all:
3167 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAll);
3168 case Intrinsic::spv_wave_any:
3169 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAny);
3170 case Intrinsic::spv_wave_is_first_lane:
3171 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformElect);
3172 case Intrinsic::spv_wave_reduce_umax:
3173 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
3174 case Intrinsic::spv_wave_reduce_max:
3175 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
3176 case Intrinsic::spv_wave_reduce_sum:
3177 return selectWaveReduceSum(ResVReg, ResType, I);
3178 case Intrinsic::spv_wave_readlane:
3179 return selectWaveOpInst(ResVReg, ResType, I,
3180 Opcode: SPIRV::OpGroupNonUniformShuffle);
3181 case Intrinsic::spv_step:
3182 return selectExtInst(ResVReg, ResType, I, CLInst: CL::step, GLInst: GL::Step);
3183 case Intrinsic::spv_radians:
3184 return selectExtInst(ResVReg, ResType, I, CLInst: CL::radians, GLInst: GL::Radians);
3185 // Discard intrinsics which we do not expect to actually represent code after
3186 // lowering or intrinsics which are not implemented but should not crash when
3187 // found in a customer's LLVM IR input.
3188 case Intrinsic::instrprof_increment:
3189 case Intrinsic::instrprof_increment_step:
3190 case Intrinsic::instrprof_value_profile:
3191 break;
3192 // Discard internal intrinsics.
3193 case Intrinsic::spv_value_md:
3194 break;
3195 case Intrinsic::spv_resource_handlefrombinding: {
3196 return selectHandleFromBinding(ResVReg, ResType, I);
3197 }
3198 case Intrinsic::spv_resource_store_typedbuffer: {
3199 return selectImageWriteIntrinsic(I);
3200 }
3201 case Intrinsic::spv_resource_load_typedbuffer: {
3202 return selectReadImageIntrinsic(ResVReg, ResType, I);
3203 }
3204 case Intrinsic::spv_resource_getpointer: {
3205 return selectResourceGetPointer(ResVReg, ResType, I);
3206 }
3207 case Intrinsic::spv_discard: {
3208 return selectDiscard(ResVReg, ResType, I);
3209 }
3210 default: {
3211 std::string DiagMsg;
3212 raw_string_ostream OS(DiagMsg);
3213 I.print(OS);
3214 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
3215 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
3216 }
3217 }
3218 return true;
3219}
3220
3221bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
3222 const SPIRVType *ResType,
3223 MachineInstr &I) const {
3224 // The images need to be loaded in the same basic block as their use. We defer
3225 // loading the image to the intrinsic that uses it.
3226 if (ResType->getOpcode() == SPIRV::OpTypeImage)
3227 return true;
3228
3229 return loadHandleBeforePosition(HandleReg&: ResVReg, ResType: GR.getSPIRVTypeForVReg(VReg: ResVReg),
3230 HandleDef&: *cast<GIntrinsic>(Val: &I), Pos&: I);
3231}
3232
3233bool SPIRVInstructionSelector::selectReadImageIntrinsic(
3234 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
3235
3236 // If the load of the image is in a different basic block, then
3237 // this will generate invalid code. A proper solution is to move
3238 // the OpLoad from selectHandleFromBinding here. However, to do
3239 // that we will need to change the return type of the intrinsic.
3240 // We will do that when we can, but for now trying to move forward with other
3241 // issues.
3242 Register ImageReg = I.getOperand(i: 2).getReg();
3243 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
3244 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
3245 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
3246 HandleDef&: *ImageDef, Pos&: I)) {
3247 return false;
3248 }
3249
3250 Register IdxReg = I.getOperand(i: 3).getReg();
3251 DebugLoc Loc = I.getDebugLoc();
3252 MachineInstr &Pos = I;
3253
3254 return generateImageRead(ResVReg, ResType, ImageReg: NewImageReg, IdxReg, Loc, Pos);
3255}
3256
3257bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg,
3258 const SPIRVType *ResType,
3259 Register ImageReg,
3260 Register IdxReg, DebugLoc Loc,
3261 MachineInstr &Pos) const {
3262 SPIRVType *ImageType = GR.getSPIRVTypeForVReg(VReg: ImageReg);
3263 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
3264 "ImageReg is not an image type.");
3265 bool IsSignedInteger =
3266 sampledTypeIsSignedInteger(HandleType: GR.getTypeForSPIRVType(Ty: ImageType));
3267
3268 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
3269 if (ResultSize == 4) {
3270 auto BMI = BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpImageRead))
3271 .addDef(RegNo: ResVReg)
3272 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3273 .addUse(RegNo: ImageReg)
3274 .addUse(RegNo: IdxReg);
3275
3276 if (IsSignedInteger)
3277 BMI.addImm(Val: 0x1000); // SignExtend
3278 return BMI.constrainAllUses(TII, TRI, RBI);
3279 }
3280
3281 SPIRVType *ReadType = widenTypeToVec4(Type: ResType, I&: Pos);
3282 Register ReadReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ReadType));
3283 auto BMI = BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpImageRead))
3284 .addDef(RegNo: ReadReg)
3285 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ReadType))
3286 .addUse(RegNo: ImageReg)
3287 .addUse(RegNo: IdxReg);
3288 if (IsSignedInteger)
3289 BMI.addImm(Val: 0x1000); // SignExtend
3290 bool Succeed = BMI.constrainAllUses(TII, TRI, RBI);
3291 if (!Succeed)
3292 return false;
3293
3294 if (ResultSize == 1) {
3295 return BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
3296 MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
3297 .addDef(RegNo: ResVReg)
3298 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3299 .addUse(RegNo: ReadReg)
3300 .addImm(Val: 0)
3301 .constrainAllUses(TII, TRI, RBI);
3302 }
3303 return extractSubvector(ResVReg, ResType, ReadReg, InsertionPoint&: Pos);
3304}
3305
3306bool SPIRVInstructionSelector::selectResourceGetPointer(
3307 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
3308 Register ResourcePtr = I.getOperand(i: 2).getReg();
3309 SPIRVType *RegType = GR.getSPIRVTypeForVReg(VReg: ResourcePtr, MF: I.getMF());
3310 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
3311 // For texel buffers, the index into the image is part of the OpImageRead or
3312 // OpImageWrite instructions. So we will do nothing in this case. This
3313 // intrinsic will be combined with the load or store when selecting the load
3314 // or store.
3315 return true;
3316 }
3317
3318 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
3319 MachineIRBuilder MIRBuilder(I);
3320
3321 Register IndexReg = I.getOperand(i: 3).getReg();
3322 Register ZeroReg =
3323 buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
3324 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3325 MCID: TII.get(Opcode: SPIRV::OpAccessChain))
3326 .addDef(RegNo: ResVReg)
3327 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3328 .addUse(RegNo: ResourcePtr)
3329 .addUse(RegNo: ZeroReg)
3330 .addUse(RegNo: IndexReg)
3331 .constrainAllUses(TII, TRI, RBI);
3332}
3333
3334bool SPIRVInstructionSelector::extractSubvector(
3335 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg,
3336 MachineInstr &InsertionPoint) const {
3337 SPIRVType *InputType = GR.getResultType(VReg: ReadReg);
3338 [[maybe_unused]] uint64_t InputSize =
3339 GR.getScalarOrVectorComponentCount(Type: InputType);
3340 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
3341 assert(InputSize > 1 && "The input must be a vector.");
3342 assert(ResultSize > 1 && "The result must be a vector.");
3343 assert(ResultSize < InputSize &&
3344 "Cannot extract more element than there are in the input.");
3345 SmallVector<Register> ComponentRegisters;
3346 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(Type: ResType);
3347 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(SpvType: ScalarType);
3348 for (uint64_t I = 0; I < ResultSize; I++) {
3349 Register ComponentReg = MRI->createVirtualRegister(RegClass: ScalarRegClass);
3350 bool Succeed = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
3351 MIMD: InsertionPoint.getDebugLoc(),
3352 MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
3353 .addDef(RegNo: ComponentReg)
3354 .addUse(RegNo: ScalarType->getOperand(i: 0).getReg())
3355 .addUse(RegNo: ReadReg)
3356 .addImm(Val: I)
3357 .constrainAllUses(TII, TRI, RBI);
3358 if (!Succeed)
3359 return false;
3360 ComponentRegisters.emplace_back(Args&: ComponentReg);
3361 }
3362
3363 MachineInstrBuilder MIB = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
3364 MIMD: InsertionPoint.getDebugLoc(),
3365 MCID: TII.get(Opcode: SPIRV::OpCompositeConstruct))
3366 .addDef(RegNo: ResVReg)
3367 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3368
3369 for (Register ComponentReg : ComponentRegisters)
3370 MIB.addUse(RegNo: ComponentReg);
3371 return MIB.constrainAllUses(TII, TRI, RBI);
3372}
3373
3374bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
3375 MachineInstr &I) const {
3376 // If the load of the image is in a different basic block, then
3377 // this will generate invalid code. A proper solution is to move
3378 // the OpLoad from selectHandleFromBinding here. However, to do
3379 // that we will need to change the return type of the intrinsic.
3380 // We will do that when we can, but for now trying to move forward with other
3381 // issues.
3382 Register ImageReg = I.getOperand(i: 1).getReg();
3383 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
3384 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
3385 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
3386 HandleDef&: *ImageDef, Pos&: I)) {
3387 return false;
3388 }
3389
3390 Register CoordinateReg = I.getOperand(i: 2).getReg();
3391 Register DataReg = I.getOperand(i: 3).getReg();
3392 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
3393 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
3394 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3395 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
3396 .addUse(RegNo: NewImageReg)
3397 .addUse(RegNo: CoordinateReg)
3398 .addUse(RegNo: DataReg)
3399 .constrainAllUses(TII, TRI, RBI);
3400}
3401
3402Register SPIRVInstructionSelector::buildPointerToResource(
3403 const SPIRVType *SpirvResType, SPIRV::StorageClass::StorageClass SC,
3404 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
3405 bool IsNonUniform, StringRef Name, MachineIRBuilder MIRBuilder) const {
3406 const Type *ResType = GR.getTypeForSPIRVType(Ty: SpirvResType);
3407 if (ArraySize == 1) {
3408 SPIRVType *PtrType =
3409 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
3410 assert(GR.getPointeeType(PtrType) == SpirvResType &&
3411 "SpirvResType did not have an explicit layout.");
3412 return GR.getOrCreateGlobalVariableWithBinding(VarType: PtrType, Set, Binding, Name,
3413 MIRBuilder);
3414 }
3415
3416 const Type *VarType = ArrayType::get(ElementType: const_cast<Type *>(ResType), NumElements: ArraySize);
3417 SPIRVType *VarPointerType =
3418 GR.getOrCreateSPIRVPointerType(BaseType: VarType, MIRBuilder, SC);
3419 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
3420 VarType: VarPointerType, Set, Binding, Name, MIRBuilder);
3421
3422 SPIRVType *ResPointerType =
3423 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
3424
3425 Register AcReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResPointerType));
3426 if (IsNonUniform) {
3427 // It is unclear which value needs to be marked an non-uniform, so both
3428 // the index and the access changed are decorated as non-uniform.
3429 buildOpDecorate(Reg: IndexReg, MIRBuilder, Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
3430 buildOpDecorate(Reg: AcReg, MIRBuilder, Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
3431 }
3432
3433 MIRBuilder.buildInstr(Opcode: SPIRV::OpAccessChain)
3434 .addDef(RegNo: AcReg)
3435 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResPointerType))
3436 .addUse(RegNo: VarReg)
3437 .addUse(RegNo: IndexReg);
3438
3439 return AcReg;
3440}
3441
3442bool SPIRVInstructionSelector::selectFirstBitSet16(
3443 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3444 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
3445 Register ExtReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3446 bool Result = selectOpWithSrcs(ResVReg: ExtReg, ResType, I, Srcs: {I.getOperand(i: 2).getReg()},
3447 Opcode: ExtendOpcode);
3448
3449 return Result &&
3450 selectFirstBitSet32(ResVReg, ResType, I, SrcReg: ExtReg, BitSetOpcode);
3451}
3452
3453bool SPIRVInstructionSelector::selectFirstBitSet32(
3454 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3455 Register SrcReg, unsigned BitSetOpcode) const {
3456 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
3457 .addDef(RegNo: ResVReg)
3458 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3459 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3460 .addImm(Val: BitSetOpcode)
3461 .addUse(RegNo: SrcReg)
3462 .constrainAllUses(TII, TRI, RBI);
3463}
3464
3465bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
3466 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3467 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
3468
3469 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
3470 // requires creating a param register and return register with an invalid
3471 // vector size. If that is resolved, then this function can be used for
3472 // vectors of any component size.
3473 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
3474 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
3475
3476 MachineIRBuilder MIRBuilder(I);
3477 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
3478 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 64, MIRBuilder);
3479 SPIRVType *I64x2Type =
3480 GR.getOrCreateSPIRVVectorType(BaseType: I64Type, NumElements: 2, MIRBuilder, EmitIR: false);
3481 SPIRVType *Vec2ResType =
3482 GR.getOrCreateSPIRVVectorType(BaseType, NumElements: 2, MIRBuilder, EmitIR: false);
3483
3484 std::vector<Register> PartialRegs;
3485
3486 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
3487 unsigned CurrentComponent = 0;
3488 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
3489 // This register holds the firstbitX result for each of the i64x2 vectors
3490 // extracted from SrcReg
3491 Register BitSetResult =
3492 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64x2Type));
3493
3494 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3495 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
3496 .addDef(RegNo: BitSetResult)
3497 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: I64x2Type))
3498 .addUse(RegNo: SrcReg)
3499 .addUse(RegNo: SrcReg)
3500 .addImm(Val: CurrentComponent)
3501 .addImm(Val: CurrentComponent + 1);
3502
3503 if (!MIB.constrainAllUses(TII, TRI, RBI))
3504 return false;
3505
3506 Register SubVecBitSetReg =
3507 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Vec2ResType));
3508
3509 if (!selectFirstBitSet64(ResVReg: SubVecBitSetReg, ResType: Vec2ResType, I, SrcReg: BitSetResult,
3510 BitSetOpcode, SwapPrimarySide))
3511 return false;
3512
3513 PartialRegs.push_back(x: SubVecBitSetReg);
3514 }
3515
3516 // On odd component counts we need to handle one more component
3517 if (CurrentComponent != ComponentCount) {
3518 bool ZeroAsNull = !STI.isShader();
3519 Register FinalElemReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64Type));
3520 Register ConstIntLastIdx = GR.getOrCreateConstInt(
3521 Val: ComponentCount - 1, I, SpvType: BaseType, TII, ZeroAsNull);
3522
3523 if (!selectOpWithSrcs(ResVReg: FinalElemReg, ResType: I64Type, I, Srcs: {SrcReg, ConstIntLastIdx},
3524 Opcode: SPIRV::OpVectorExtractDynamic))
3525 return false;
3526
3527 Register FinalElemBitSetReg =
3528 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BaseType));
3529
3530 if (!selectFirstBitSet64(ResVReg: FinalElemBitSetReg, ResType: BaseType, I, SrcReg: FinalElemReg,
3531 BitSetOpcode, SwapPrimarySide))
3532 return false;
3533
3534 PartialRegs.push_back(x: FinalElemBitSetReg);
3535 }
3536
3537 // Join all the resulting registers back into the return type in order
3538 // (ie i32x2, i32x2, i32x1 -> i32x5)
3539 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: PartialRegs,
3540 Opcode: SPIRV::OpCompositeConstruct);
3541}
3542
3543bool SPIRVInstructionSelector::selectFirstBitSet64(
3544 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3545 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
3546 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
3547 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
3548 bool ZeroAsNull = !STI.isShader();
3549 Register ConstIntZero =
3550 GR.getOrCreateConstInt(Val: 0, I, SpvType: BaseType, TII, ZeroAsNull);
3551 Register ConstIntOne =
3552 GR.getOrCreateConstInt(Val: 1, I, SpvType: BaseType, TII, ZeroAsNull);
3553
3554 // SPIRV doesn't support vectors with more than 4 components. Since the
3555 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
3556 // operate on vectors with 2 or less components. When largers vectors are
3557 // seen. Split them, recurse, then recombine them.
3558 if (ComponentCount > 2) {
3559 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
3560 BitSetOpcode, SwapPrimarySide);
3561 }
3562
3563 // 1. Split int64 into 2 pieces using a bitcast
3564 MachineIRBuilder MIRBuilder(I);
3565 SPIRVType *PostCastType = GR.getOrCreateSPIRVVectorType(
3566 BaseType, NumElements: 2 * ComponentCount, MIRBuilder, EmitIR: false);
3567 Register BitcastReg =
3568 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
3569
3570 if (!selectOpWithSrcs(ResVReg: BitcastReg, ResType: PostCastType, I, Srcs: {SrcReg},
3571 Opcode: SPIRV::OpBitcast))
3572 return false;
3573
3574 // 2. Find the first set bit from the primary side for all the pieces in #1
3575 Register FBSReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
3576 if (!selectFirstBitSet32(ResVReg: FBSReg, ResType: PostCastType, I, SrcReg: BitcastReg, BitSetOpcode))
3577 return false;
3578
3579 // 3. Split result vector into high bits and low bits
3580 Register HighReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3581 Register LowReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3582
3583 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
3584 if (IsScalarRes) {
3585 // if scalar do a vector extract
3586 if (!selectOpWithSrcs(ResVReg: HighReg, ResType, I, Srcs: {FBSReg, ConstIntZero},
3587 Opcode: SPIRV::OpVectorExtractDynamic))
3588 return false;
3589 if (!selectOpWithSrcs(ResVReg: LowReg, ResType, I, Srcs: {FBSReg, ConstIntOne},
3590 Opcode: SPIRV::OpVectorExtractDynamic))
3591 return false;
3592 } else {
3593 // if vector do a shufflevector
3594 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3595 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
3596 .addDef(RegNo: HighReg)
3597 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3598 .addUse(RegNo: FBSReg)
3599 // Per the spec, repeat the vector if only one vec is needed
3600 .addUse(RegNo: FBSReg);
3601
3602 // high bits are stored in even indexes. Extract them from FBSReg
3603 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
3604 MIB.addImm(Val: J);
3605 }
3606
3607 if (!MIB.constrainAllUses(TII, TRI, RBI))
3608 return false;
3609
3610 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3611 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
3612 .addDef(RegNo: LowReg)
3613 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3614 .addUse(RegNo: FBSReg)
3615 // Per the spec, repeat the vector if only one vec is needed
3616 .addUse(RegNo: FBSReg);
3617
3618 // low bits are stored in odd indexes. Extract them from FBSReg
3619 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
3620 MIB.addImm(Val: J);
3621 }
3622 if (!MIB.constrainAllUses(TII, TRI, RBI))
3623 return false;
3624 }
3625
3626 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
3627 // primary
3628 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
3629 Register NegOneReg;
3630 Register Reg0;
3631 Register Reg32;
3632 unsigned SelectOp;
3633 unsigned AddOp;
3634
3635 if (IsScalarRes) {
3636 NegOneReg =
3637 GR.getOrCreateConstInt(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
3638 Reg0 = GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
3639 Reg32 = GR.getOrCreateConstInt(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
3640 SelectOp = SPIRV::OpSelectSISCond;
3641 AddOp = SPIRV::OpIAddS;
3642 } else {
3643 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: ComponentCount,
3644 MIRBuilder, EmitIR: false);
3645 NegOneReg =
3646 GR.getOrCreateConstVector(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
3647 Reg0 = GR.getOrCreateConstVector(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
3648 Reg32 = GR.getOrCreateConstVector(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
3649 SelectOp = SPIRV::OpSelectVIVCond;
3650 AddOp = SPIRV::OpIAddV;
3651 }
3652
3653 Register PrimaryReg = HighReg;
3654 Register SecondaryReg = LowReg;
3655 Register PrimaryShiftReg = Reg32;
3656 Register SecondaryShiftReg = Reg0;
3657
3658 // By default the emitted opcodes check for the set bit from the MSB side.
3659 // Setting SwapPrimarySide checks the set bit from the LSB side
3660 if (SwapPrimarySide) {
3661 PrimaryReg = LowReg;
3662 SecondaryReg = HighReg;
3663 PrimaryShiftReg = Reg0;
3664 SecondaryShiftReg = Reg32;
3665 }
3666
3667 // Check if the primary bits are == -1
3668 Register BReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BoolType));
3669 if (!selectOpWithSrcs(ResVReg: BReg, ResType: BoolType, I, Srcs: {PrimaryReg, NegOneReg},
3670 Opcode: SPIRV::OpIEqual))
3671 return false;
3672
3673 // Select secondary bits if true in BReg, otherwise primary bits
3674 Register TmpReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3675 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {BReg, SecondaryReg, PrimaryReg},
3676 Opcode: SelectOp))
3677 return false;
3678
3679 // 5. Add 32 when high bits are used, otherwise 0 for low bits
3680 Register ValReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3681 if (!selectOpWithSrcs(ResVReg: ValReg, ResType, I,
3682 Srcs: {BReg, SecondaryShiftReg, PrimaryShiftReg}, Opcode: SelectOp))
3683 return false;
3684
3685 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {ValReg, TmpReg}, Opcode: AddOp);
3686}
3687
3688bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
3689 const SPIRVType *ResType,
3690 MachineInstr &I,
3691 bool IsSigned) const {
3692 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
3693 Register OpReg = I.getOperand(i: 2).getReg();
3694 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3695 // zero or sign extend
3696 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3697 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
3698
3699 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
3700 case 16:
3701 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
3702 case 32:
3703 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
3704 case 64:
3705 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
3706 /*SwapPrimarySide=*/false);
3707 default:
3708 report_fatal_error(
3709 reason: "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
3710 }
3711}
3712
3713bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
3714 const SPIRVType *ResType,
3715 MachineInstr &I) const {
3716 // FindILsb intrinsic only supports 32 bit integers
3717 Register OpReg = I.getOperand(i: 2).getReg();
3718 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3719 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
3720 // to an unsigned i32. As this leaves all the least significant bits unchanged
3721 // so the first set bit from the LSB side doesn't change.
3722 unsigned ExtendOpcode = SPIRV::OpUConvert;
3723 unsigned BitSetOpcode = GL::FindILsb;
3724
3725 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
3726 case 16:
3727 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
3728 case 32:
3729 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
3730 case 64:
3731 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
3732 /*SwapPrimarySide=*/true);
3733 default:
3734 report_fatal_error(reason: "spv_firstbitlow only supports 16,32,64 bits.");
3735 }
3736}
3737
3738bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
3739 const SPIRVType *ResType,
3740 MachineInstr &I) const {
3741 // there was an allocation size parameter to the allocation instruction
3742 // that is not 1
3743 MachineBasicBlock &BB = *I.getParent();
3744 bool Res = BuildMI(BB, I, MIMD: I.getDebugLoc(),
3745 MCID: TII.get(Opcode: SPIRV::OpVariableLengthArrayINTEL))
3746 .addDef(RegNo: ResVReg)
3747 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3748 .addUse(RegNo: I.getOperand(i: 2).getReg())
3749 .constrainAllUses(TII, TRI, RBI);
3750 if (!STI.isShader()) {
3751 unsigned Alignment = I.getOperand(i: 3).getImm();
3752 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::Alignment, DecArgs: {Alignment});
3753 }
3754 return Res;
3755}
3756
3757bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
3758 const SPIRVType *ResType,
3759 MachineInstr &I) const {
3760 // Change order of instructions if needed: all OpVariable instructions in a
3761 // function must be the first instructions in the first block
3762 auto It = getOpVariableMBBIt(I);
3763 bool Res = BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(),
3764 MCID: TII.get(Opcode: SPIRV::OpVariable))
3765 .addDef(RegNo: ResVReg)
3766 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3767 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
3768 .constrainAllUses(TII, TRI, RBI);
3769 if (!STI.isShader()) {
3770 unsigned Alignment = I.getOperand(i: 2).getImm();
3771 buildOpDecorate(Reg: ResVReg, I&: *It, TII, Dec: SPIRV::Decoration::Alignment,
3772 DecArgs: {Alignment});
3773 }
3774 return Res;
3775}
3776
3777bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
3778 // InstructionSelector walks backwards through the instructions. We can use
3779 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
3780 // first, so can generate an OpBranchConditional here. If there is no
3781 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
3782 const MachineInstr *PrevI = I.getPrevNode();
3783 MachineBasicBlock &MBB = *I.getParent();
3784 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
3785 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
3786 .addUse(RegNo: PrevI->getOperand(i: 0).getReg())
3787 .addMBB(MBB: PrevI->getOperand(i: 1).getMBB())
3788 .addMBB(MBB: I.getOperand(i: 0).getMBB())
3789 .constrainAllUses(TII, TRI, RBI);
3790 }
3791 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranch))
3792 .addMBB(MBB: I.getOperand(i: 0).getMBB())
3793 .constrainAllUses(TII, TRI, RBI);
3794}
3795
3796bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
3797 // InstructionSelector walks backwards through the instructions. For an
3798 // explicit conditional branch with no fallthrough, we use both a G_BR and a
3799 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
3800 // generate the OpBranchConditional in selectBranch above.
3801 //
3802 // If an OpBranchConditional has been generated, we simply return, as the work
3803 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
3804 // implicit fallthrough to the next basic block, so we need to create an
3805 // OpBranchConditional with an explicit "false" argument pointing to the next
3806 // basic block that LLVM would fall through to.
3807 const MachineInstr *NextI = I.getNextNode();
3808 // Check if this has already been successfully selected.
3809 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
3810 return true;
3811 // Must be relying on implicit block fallthrough, so generate an
3812 // OpBranchConditional with the "next" basic block as the "false" target.
3813 MachineBasicBlock &MBB = *I.getParent();
3814 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
3815 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(N: NextMBBNum);
3816 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
3817 .addUse(RegNo: I.getOperand(i: 0).getReg())
3818 .addMBB(MBB: I.getOperand(i: 1).getMBB())
3819 .addMBB(MBB: NextMBB)
3820 .constrainAllUses(TII, TRI, RBI);
3821}
3822
3823bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
3824 const SPIRVType *ResType,
3825 MachineInstr &I) const {
3826 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpPhi))
3827 .addDef(RegNo: ResVReg)
3828 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3829 const unsigned NumOps = I.getNumOperands();
3830 for (unsigned i = 1; i < NumOps; i += 2) {
3831 MIB.addUse(RegNo: I.getOperand(i: i + 0).getReg());
3832 MIB.addMBB(MBB: I.getOperand(i: i + 1).getMBB());
3833 }
3834 bool Res = MIB.constrainAllUses(TII, TRI, RBI);
3835 MIB->setDesc(TII.get(Opcode: TargetOpcode::PHI));
3836 MIB->removeOperand(OpNo: 1);
3837 return Res;
3838}
3839
3840bool SPIRVInstructionSelector::selectGlobalValue(
3841 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
3842 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
3843 MachineIRBuilder MIRBuilder(I);
3844 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
3845 Type *GVType = toTypedPointer(Ty: GR.getDeducedGlobalValueType(Global: GV));
3846
3847 std::string GlobalIdent;
3848 if (!GV->hasName()) {
3849 unsigned &ID = UnnamedGlobalIDs[GV];
3850 if (ID == 0)
3851 ID = UnnamedGlobalIDs.size();
3852 GlobalIdent = "__unnamed_" + Twine(ID).str();
3853 } else {
3854 GlobalIdent = GV->getName();
3855 }
3856
3857 // Behaviour of functions as operands depends on availability of the
3858 // corresponding extension (SPV_INTEL_function_pointers):
3859 // - If there is an extension to operate with functions as operands:
3860 // We create a proper constant operand and evaluate a correct type for a
3861 // function pointer.
3862 // - Without the required extension:
3863 // We have functions as operands in tests with blocks of instruction e.g. in
3864 // transcoding/global_block.ll. These operands are not used and should be
3865 // substituted by zero constants. Their type is expected to be always
3866 // OpTypePointer Function %uchar.
3867 if (isa<Function>(Val: GV)) {
3868 const Constant *ConstVal = GV;
3869 MachineBasicBlock &BB = *I.getParent();
3870 Register NewReg = GR.find(V: ConstVal, MF: GR.CurMF);
3871 if (!NewReg.isValid()) {
3872 Register NewReg = ResVReg;
3873 const Function *GVFun =
3874 STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
3875 ? dyn_cast<Function>(Val: GV)
3876 : nullptr;
3877 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
3878 BaseType: GVType, I,
3879 SC: GVFun ? SPIRV::StorageClass::CodeSectionINTEL
3880 : addressSpaceToStorageClass(AddrSpace: GV->getAddressSpace(), STI));
3881 if (GVFun) {
3882 // References to a function via function pointers generate virtual
3883 // registers without a definition. We will resolve it later, during
3884 // module analysis stage.
3885 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
3886 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3887 Register FuncVReg =
3888 MRI->createGenericVirtualRegister(Ty: GR.getRegType(SpvType: ResType));
3889 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::pIDRegClass);
3890 MachineInstrBuilder MIB1 =
3891 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3892 .addDef(RegNo: FuncVReg)
3893 .addUse(RegNo: ResTypeReg);
3894 MachineInstrBuilder MIB2 =
3895 BuildMI(BB, I, MIMD: I.getDebugLoc(),
3896 MCID: TII.get(Opcode: SPIRV::OpConstantFunctionPointerINTEL))
3897 .addDef(RegNo: NewReg)
3898 .addUse(RegNo: ResTypeReg)
3899 .addUse(RegNo: FuncVReg);
3900 GR.add(V: ConstVal, MI: MIB2);
3901 // mapping the function pointer to the used Function
3902 GR.recordFunctionPointer(MO: &MIB2.getInstr()->getOperand(i: 2), F: GVFun);
3903 return MIB1.constrainAllUses(TII, TRI, RBI) &&
3904 MIB2.constrainAllUses(TII, TRI, RBI);
3905 }
3906 MachineInstrBuilder MIB3 =
3907 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3908 .addDef(RegNo: NewReg)
3909 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3910 GR.add(V: ConstVal, MI: MIB3);
3911 return MIB3.constrainAllUses(TII, TRI, RBI);
3912 }
3913 assert(NewReg != ResVReg);
3914 return BuildCOPY(DestReg: ResVReg, SrcReg: NewReg, I);
3915 }
3916 auto GlobalVar = cast<GlobalVariable>(Val: GV);
3917 assert(GlobalVar->getName() != "llvm.global.annotations");
3918
3919 // Skip empty declaration for GVs with initializers till we get the decl with
3920 // passed initializer.
3921 if (hasInitializer(GV: GlobalVar) && !Init)
3922 return true;
3923
3924 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage() &&
3925 !GV->hasHiddenVisibility();
3926 SPIRV::LinkageType::LinkageType LnkType =
3927 GV->isDeclarationForLinker()
3928 ? SPIRV::LinkageType::Import
3929 : (GV->hasLinkOnceODRLinkage() &&
3930 STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr)
3931 ? SPIRV::LinkageType::LinkOnceODR
3932 : SPIRV::LinkageType::Export);
3933
3934 const unsigned AddrSpace = GV->getAddressSpace();
3935 SPIRV::StorageClass::StorageClass StorageClass =
3936 addressSpaceToStorageClass(AddrSpace, STI);
3937 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(BaseType: GVType, I, SC: StorageClass);
3938 Register Reg = GR.buildGlobalVariable(
3939 Reg: ResVReg, BaseType: ResType, Name: GlobalIdent, GV, Storage: StorageClass, Init,
3940 IsConst: GlobalVar->isConstant(), HasLinkageTy: HasLnkTy, LinkageType: LnkType, MIRBuilder, IsInstSelector: true);
3941 return Reg.isValid();
3942}
3943
3944bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
3945 const SPIRVType *ResType,
3946 MachineInstr &I) const {
3947 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
3948 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log10);
3949 }
3950
3951 // There is no log10 instruction in the GLSL Extended Instruction set, so it
3952 // is implemented as:
3953 // log10(x) = log2(x) * (1 / log2(10))
3954 // = log2(x) * 0.30103
3955
3956 MachineIRBuilder MIRBuilder(I);
3957 MachineBasicBlock &BB = *I.getParent();
3958
3959 // Build log2(x).
3960 Register VarReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3961 bool Result =
3962 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
3963 .addDef(RegNo: VarReg)
3964 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3965 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3966 .addImm(Val: GL::Log2)
3967 .add(MO: I.getOperand(i: 1))
3968 .constrainAllUses(TII, TRI, RBI);
3969
3970 // Build 0.30103.
3971 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
3972 ResType->getOpcode() == SPIRV::OpTypeFloat);
3973 // TODO: Add matrix implementation once supported by the HLSL frontend.
3974 const SPIRVType *SpirvScalarType =
3975 ResType->getOpcode() == SPIRV::OpTypeVector
3976 ? GR.getSPIRVTypeForVReg(VReg: ResType->getOperand(i: 1).getReg())
3977 : ResType;
3978 Register ScaleReg =
3979 GR.buildConstantFP(Val: APFloat(0.30103f), MIRBuilder, SpvType: SpirvScalarType);
3980
3981 // Multiply log2(x) by 0.30103 to get log10(x) result.
3982 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
3983 ? SPIRV::OpVectorTimesScalar
3984 : SPIRV::OpFMulS;
3985 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3986 .addDef(RegNo: ResVReg)
3987 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3988 .addUse(RegNo: VarReg)
3989 .addUse(RegNo: ScaleReg)
3990 .constrainAllUses(TII, TRI, RBI);
3991}
3992
3993// Generate the instructions to load 3-element vector builtin input
3994// IDs/Indices.
3995// Like: GlobalInvocationId, LocalInvocationId, etc....
3996bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
3997 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
3998 const SPIRVType *ResType, MachineInstr &I) const {
3999 MachineIRBuilder MIRBuilder(I);
4000 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
4001 const SPIRVType *Vec3Ty =
4002 GR.getOrCreateSPIRVVectorType(BaseType: U32Type, NumElements: 3, MIRBuilder, EmitIR: false);
4003 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
4004 BaseType: Vec3Ty, MIRBuilder, SC: SPIRV::StorageClass::Input);
4005
4006 // Create new register for the input ID builtin variable.
4007 Register NewRegister =
4008 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
4009 MIRBuilder.getMRI()->setType(VReg: NewRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
4010 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
4011
4012 // Build global variable with the necessary decorations for the input ID
4013 // builtin variable.
4014 Register Variable = GR.buildGlobalVariable(
4015 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
4016 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, HasLinkageTy: false,
4017 LinkageType: SPIRV::LinkageType::Import, MIRBuilder, IsInstSelector: false);
4018
4019 // Create new register for loading value.
4020 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4021 Register LoadedRegister = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
4022 MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
4023 GR.assignSPIRVTypeToVReg(Type: Vec3Ty, VReg: LoadedRegister, MF: MIRBuilder.getMF());
4024
4025 // Load v3uint value from the global variable.
4026 bool Result =
4027 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
4028 .addDef(RegNo: LoadedRegister)
4029 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: Vec3Ty))
4030 .addUse(RegNo: Variable);
4031
4032 // Get the input ID index. Expecting operand is a constant immediate value,
4033 // wrapped in a type assignment.
4034 assert(I.getOperand(2).isReg());
4035 const uint32_t ThreadId = foldImm(MO: I.getOperand(i: 2), MRI);
4036
4037 // Extract the input ID from the loaded vector value.
4038 MachineBasicBlock &BB = *I.getParent();
4039 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
4040 .addDef(RegNo: ResVReg)
4041 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4042 .addUse(RegNo: LoadedRegister)
4043 .addImm(Val: ThreadId);
4044 return Result && MIB.constrainAllUses(TII, TRI, RBI);
4045}
4046
4047// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
4048// Like LocalInvocationIndex
4049bool SPIRVInstructionSelector::loadBuiltinInputID(
4050 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
4051 const SPIRVType *ResType, MachineInstr &I) const {
4052 MachineIRBuilder MIRBuilder(I);
4053 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
4054 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Input);
4055
4056 // Create new register for the input ID builtin variable.
4057 Register NewRegister =
4058 MIRBuilder.getMRI()->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PtrType));
4059 MIRBuilder.getMRI()->setType(
4060 VReg: NewRegister,
4061 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Input),
4062 SizeInBits: GR.getPointerSize()));
4063 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
4064
4065 // Build global variable with the necessary decorations for the input ID
4066 // builtin variable.
4067 Register Variable = GR.buildGlobalVariable(
4068 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
4069 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, HasLinkageTy: false,
4070 LinkageType: SPIRV::LinkageType::Import, MIRBuilder, IsInstSelector: false);
4071
4072 // Load uint value from the global variable.
4073 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
4074 .addDef(RegNo: ResVReg)
4075 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4076 .addUse(RegNo: Variable);
4077
4078 return MIB.constrainAllUses(TII, TRI, RBI);
4079}
4080
4081SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type,
4082 MachineInstr &I) const {
4083 MachineIRBuilder MIRBuilder(I);
4084 if (Type->getOpcode() != SPIRV::OpTypeVector)
4085 return GR.getOrCreateSPIRVVectorType(BaseType: Type, NumElements: 4, MIRBuilder, EmitIR: false);
4086
4087 uint64_t VectorSize = Type->getOperand(i: 2).getImm();
4088 if (VectorSize == 4)
4089 return Type;
4090
4091 Register ScalarTypeReg = Type->getOperand(i: 1).getReg();
4092 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(VReg: ScalarTypeReg);
4093 return GR.getOrCreateSPIRVVectorType(BaseType: ScalarType, NumElements: 4, MIRBuilder, EmitIR: false);
4094}
4095
4096bool SPIRVInstructionSelector::loadHandleBeforePosition(
4097 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef,
4098 MachineInstr &Pos) const {
4099
4100 assert(HandleDef.getIntrinsicID() ==
4101 Intrinsic::spv_resource_handlefrombinding);
4102 uint32_t Set = foldImm(MO: HandleDef.getOperand(i: 2), MRI);
4103 uint32_t Binding = foldImm(MO: HandleDef.getOperand(i: 3), MRI);
4104 uint32_t ArraySize = foldImm(MO: HandleDef.getOperand(i: 4), MRI);
4105 Register IndexReg = HandleDef.getOperand(i: 5).getReg();
4106 bool IsNonUniform = ArraySize > 1 && foldImm(MO: HandleDef.getOperand(i: 6), MRI);
4107 std::string Name =
4108 getStringValueFromReg(Reg: HandleDef.getOperand(i: 7).getReg(), MRI&: *MRI);
4109
4110 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
4111 MachineIRBuilder MIRBuilder(HandleDef);
4112 SPIRVType *VarType = ResType;
4113 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
4114
4115 if (IsStructuredBuffer) {
4116 VarType = GR.getPointeeType(PtrType: ResType);
4117 SC = GR.getPointerStorageClass(Type: ResType);
4118 }
4119
4120 Register VarReg =
4121 buildPointerToResource(SpirvResType: VarType, SC, Set, Binding, ArraySize, IndexReg,
4122 IsNonUniform, Name, MIRBuilder);
4123
4124 if (IsNonUniform)
4125 buildOpDecorate(Reg: HandleReg, I&: HandleDef, TII, Dec: SPIRV::Decoration::NonUniformEXT,
4126 DecArgs: {});
4127
4128 // The handle for the buffer is the pointer to the resource. For an image, the
4129 // handle is the image object. So images get an extra load.
4130 uint32_t LoadOpcode =
4131 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
4132 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: HandleReg, MF: *Pos.getMF());
4133 return BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: HandleDef.getDebugLoc(),
4134 MCID: TII.get(Opcode: LoadOpcode))
4135 .addDef(RegNo: HandleReg)
4136 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4137 .addUse(RegNo: VarReg)
4138 .constrainAllUses(TII, TRI, RBI);
4139}
4140
4141namespace llvm {
4142InstructionSelector *
4143createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
4144 const SPIRVSubtarget &Subtarget,
4145 const RegisterBankInfo &RBI) {
4146 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
4147}
4148} // namespace llvm
4149