1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVUtils.h"
23#include "llvm/ADT/APFloat.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/Register.h"
31#include "llvm/CodeGen/TargetOpcodes.h"
32#include "llvm/IR/IntrinsicsSPIRV.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35
36#define DEBUG_TYPE "spirv-isel"
37
38using namespace llvm;
39namespace CL = SPIRV::OpenCLExtInst;
40namespace GL = SPIRV::GLSLExtInst;
41
42using ExtInstList =
43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
44
45namespace {
46
47llvm::SPIRV::SelectionControl::SelectionControl
48getSelectionOperandForImm(int Imm) {
49 if (Imm == 2)
50 return SPIRV::SelectionControl::Flatten;
51 if (Imm == 1)
52 return SPIRV::SelectionControl::DontFlatten;
53 if (Imm == 0)
54 return SPIRV::SelectionControl::None;
55 llvm_unreachable("Invalid immediate");
56}
57
58#define GET_GLOBALISEL_PREDICATE_BITSET
59#include "SPIRVGenGlobalISel.inc"
60#undef GET_GLOBALISEL_PREDICATE_BITSET
61
62class SPIRVInstructionSelector : public InstructionSelector {
63 const SPIRVSubtarget &STI;
64 const SPIRVInstrInfo &TII;
65 const SPIRVRegisterInfo &TRI;
66 const RegisterBankInfo &RBI;
67 SPIRVGlobalRegistry &GR;
68 MachineRegisterInfo *MRI;
69 MachineFunction *HasVRegsReset = nullptr;
70
71 /// We need to keep track of the number we give to anonymous global values to
72 /// generate the same name every time when this is needed.
73 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
74 SmallPtrSet<MachineInstr *, 8> DeadMIs;
75
76public:
77 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
78 const SPIRVSubtarget &ST,
79 const RegisterBankInfo &RBI);
80 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
81 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
82 BlockFrequencyInfo *BFI) override;
83 // Common selection code. Instruction-specific selection occurs in spvSelect.
84 bool select(MachineInstr &I) override;
85 static const char *getName() { return DEBUG_TYPE; }
86
87#define GET_GLOBALISEL_PREDICATES_DECL
88#include "SPIRVGenGlobalISel.inc"
89#undef GET_GLOBALISEL_PREDICATES_DECL
90
91#define GET_GLOBALISEL_TEMPORARIES_DECL
92#include "SPIRVGenGlobalISel.inc"
93#undef GET_GLOBALISEL_TEMPORARIES_DECL
94
95private:
96 void resetVRegsType(MachineFunction &MF);
97 void removeDeadInstruction(MachineInstr &MI) const;
98 void removeOpNamesForDeadMI(MachineInstr &MI) const;
99
100 // tblgen-erated 'select' implementation, used as the initial selector for
101 // the patterns that don't require complex C++.
102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
103
104 // All instruction-specific selection that didn't happen in "select()".
105 // Is basically a large Switch/Case delegating to all other select method.
106 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
107 MachineInstr &I) const;
108
109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType,
110 MachineInstr &I, bool IsSigned) const;
111
112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType,
113 MachineInstr &I) const;
114
115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType,
116 MachineInstr &I, unsigned ExtendOpcode,
117 unsigned BitSetOpcode) const;
118
119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType,
120 MachineInstr &I, Register SrcReg,
121 unsigned BitSetOpcode) const;
122
123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType,
124 MachineInstr &I, Register SrcReg,
125 unsigned BitSetOpcode, bool SwapPrimarySide) const;
126
127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType,
128 MachineInstr &I, Register SrcReg,
129 unsigned BitSetOpcode,
130 bool SwapPrimarySide) const;
131
132 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
133 const MachineInstr *Init = nullptr) const;
134
135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType,
136 MachineInstr &I, std::vector<Register> SrcRegs,
137 unsigned Opcode) const;
138
139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
140 unsigned Opcode) const;
141
142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType,
143 MachineInstr &I) const;
144
145 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
146 MachineInstr &I) const;
147 bool selectStore(MachineInstr &I) const;
148
149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType,
150 MachineInstr &I) const;
151 bool selectStackRestore(MachineInstr &I) const;
152
153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
154 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
155 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
156 bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
157
158 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
159 MachineInstr &I, unsigned NewOpcode,
160 unsigned NegateOpcode = 0) const;
161
162 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
163 MachineInstr &I) const;
164
165 bool selectFence(MachineInstr &I) const;
166
167 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
168 MachineInstr &I) const;
169
170 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
171 MachineInstr &I, unsigned OpType) const;
172
173 bool selectAll(Register ResVReg, const SPIRVType *ResType,
174 MachineInstr &I) const;
175
176 bool selectAny(Register ResVReg, const SPIRVType *ResType,
177 MachineInstr &I) const;
178
179 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
180 MachineInstr &I) const;
181
182 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType,
183 MachineInstr &I) const;
184 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType,
185 MachineInstr &I) const;
186
187 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
188 unsigned comparisonOpcode, MachineInstr &I) const;
189 bool selectDiscard(Register ResVReg, const SPIRVType *ResType,
190 MachineInstr &I) const;
191
192 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
193 MachineInstr &I) const;
194 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
195 MachineInstr &I) const;
196
197 bool selectSign(Register ResVReg, const SPIRVType *ResType,
198 MachineInstr &I) const;
199
200 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType,
201 MachineInstr &I) const;
202
203 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType,
204 MachineInstr &I, unsigned Opcode) const;
205 bool selectDebugTrap(Register ResVReg, const SPIRVType *ResType,
206 MachineInstr &I) const;
207
208 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType,
209 MachineInstr &I, bool Signed) const;
210
211 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType,
212 MachineInstr &I) const;
213
214 bool selectOpIsInf(Register ResVReg, const SPIRVType *ResType,
215 MachineInstr &I) const;
216
217 bool selectOpIsNan(Register ResVReg, const SPIRVType *ResType,
218 MachineInstr &I) const;
219
220 template <bool Signed>
221 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType,
222 MachineInstr &I) const;
223 template <bool Signed>
224 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType,
225 MachineInstr &I) const;
226
227 bool selectWavePrefixBitCount(Register ResVReg, const SPIRVType *ResType,
228 MachineInstr &I) const;
229
230 template <typename PickOpcodeFn>
231 bool selectWaveReduce(Register ResVReg, const SPIRVType *ResType,
232 MachineInstr &I, bool IsUnsigned,
233 PickOpcodeFn &&PickOpcode) const;
234
235 bool selectWaveReduceMax(Register ResVReg, const SPIRVType *ResType,
236 MachineInstr &I, bool IsUnsigned) const;
237
238 bool selectWaveReduceMin(Register ResVReg, const SPIRVType *ResType,
239 MachineInstr &I, bool IsUnsigned) const;
240
241 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType,
242 MachineInstr &I) const;
243
244 template <typename PickOpcodeFn>
245 bool selectWaveExclusiveScan(Register ResVReg, const SPIRVType *ResType,
246 MachineInstr &I, bool IsUnsigned,
247 PickOpcodeFn &&PickOpcode) const;
248
249 bool selectWaveExclusiveScanSum(Register ResVReg, const SPIRVType *ResType,
250 MachineInstr &I) const;
251
252 bool selectConst(Register ResVReg, const SPIRVType *ResType,
253 MachineInstr &I) const;
254
255 bool selectSelect(Register ResVReg, const SPIRVType *ResType,
256 MachineInstr &I) const;
257 bool selectSelectDefaultArgs(Register ResVReg, const SPIRVType *ResType,
258 MachineInstr &I, bool IsSigned) const;
259 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
260 bool IsSigned, unsigned Opcode) const;
261 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
262 bool IsSigned) const;
263
264 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
265 MachineInstr &I) const;
266
267 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
268 bool IsSigned) const;
269
270 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
271 const SPIRVType *intTy, const SPIRVType *boolTy) const;
272
273 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
274 MachineInstr &I) const;
275 bool selectFreeze(Register ResVReg, const SPIRVType *ResType,
276 MachineInstr &I) const;
277 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
278 MachineInstr &I) const;
279 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
280 MachineInstr &I) const;
281 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
282 MachineInstr &I) const;
283 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
284 MachineInstr &I) const;
285 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
286 MachineInstr &I) const;
287 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
288 MachineInstr &I) const;
289
290 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
291 MachineInstr &I) const;
292 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType,
293 MachineInstr &I) const;
294
295 bool selectBranch(MachineInstr &I) const;
296 bool selectBranchCond(MachineInstr &I) const;
297
298 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
299 MachineInstr &I) const;
300
301 bool selectExtInst(Register ResVReg, const SPIRVType *RestType,
302 MachineInstr &I, GL::GLSLExtInst GLInst) const;
303 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
304 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
305 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
306 MachineInstr &I, CL::OpenCLExtInst CLInst,
307 GL::GLSLExtInst GLInst) const;
308 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
309 MachineInstr &I, const ExtInstList &ExtInsts) const;
310 bool selectExtInstForLRound(Register ResVReg, const SPIRVType *ResType,
311 MachineInstr &I, CL::OpenCLExtInst CLInst,
312 GL::GLSLExtInst GLInst) const;
313 bool selectExtInstForLRound(Register ResVReg, const SPIRVType *ResType,
314 MachineInstr &I,
315 const ExtInstList &ExtInsts) const;
316
317 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
318 MachineInstr &I) const;
319
320 bool selectSaturate(Register ResVReg, const SPIRVType *ResType,
321 MachineInstr &I) const;
322
323 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType,
324 MachineInstr &I, unsigned Opcode) const;
325
326 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType,
327 MachineInstr &I) const;
328
329 bool selectUnmergeValues(MachineInstr &I) const;
330
331 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType,
332 MachineInstr &I) const;
333
334 bool selectCounterHandleFromBinding(Register &ResVReg,
335 const SPIRVType *ResType,
336 MachineInstr &I) const;
337
338 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType,
339 MachineInstr &I) const;
340 bool selectSampleIntrinsic(Register &ResVReg, const SPIRVType *ResType,
341 MachineInstr &I) const;
342 bool selectImageWriteIntrinsic(MachineInstr &I) const;
343 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType,
344 MachineInstr &I) const;
345 bool selectPushConstantGetPointer(Register &ResVReg, const SPIRVType *ResType,
346 MachineInstr &I) const;
347 bool selectResourceNonUniformIndex(Register &ResVReg,
348 const SPIRVType *ResType,
349 MachineInstr &I) const;
350 bool selectModf(Register ResVReg, const SPIRVType *ResType,
351 MachineInstr &I) const;
352 bool selectUpdateCounter(Register &ResVReg, const SPIRVType *ResType,
353 MachineInstr &I) const;
354 bool selectFrexp(Register ResVReg, const SPIRVType *ResType,
355 MachineInstr &I) const;
356 bool selectDerivativeInst(Register ResVReg, const SPIRVType *ResType,
357 MachineInstr &I, const unsigned DPdOpCode) const;
358 // Utilities
359 std::pair<Register, bool>
360 buildI32Constant(uint32_t Val, MachineInstr &I,
361 const SPIRVType *ResType = nullptr) const;
362
363 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
364 bool isScalarOrVectorIntConstantZero(Register Reg) const;
365 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
366 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
367 MachineInstr &I) const;
368 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const;
369
370 bool wrapIntoSpecConstantOp(MachineInstr &I,
371 SmallVector<Register> &CompositeArgs) const;
372
373 Register getUcharPtrTypeReg(MachineInstr &I,
374 SPIRV::StorageClass::StorageClass SC) const;
375 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
376 Register Src, Register DestType,
377 uint32_t Opcode) const;
378 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
379 SPIRVType *SrcPtrTy) const;
380 Register buildPointerToResource(const SPIRVType *ResType,
381 SPIRV::StorageClass::StorageClass SC,
382 uint32_t Set, uint32_t Binding,
383 uint32_t ArraySize, Register IndexReg,
384 StringRef Name,
385 MachineIRBuilder MIRBuilder) const;
386 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const;
387 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType,
388 Register &ReadReg, MachineInstr &InsertionPoint) const;
389 bool generateImageReadOrFetch(Register &ResVReg, const SPIRVType *ResType,
390 Register ImageReg, Register IdxReg,
391 DebugLoc Loc, MachineInstr &Pos) const;
392 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
393 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
394 Register ResVReg, const SPIRVType *ResType,
395 MachineInstr &I) const;
396 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
397 Register ResVReg, const SPIRVType *ResType,
398 MachineInstr &I) const;
399 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType,
400 GIntrinsic &HandleDef, MachineInstr &Pos) const;
401 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
402 void errorIfInstrOutsideShader(MachineInstr &I) const;
403};
404
405bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
406 const TargetExtType *TET = cast<TargetExtType>(Val: HandleType);
407 if (TET->getTargetExtName() == "spirv.Image") {
408 return false;
409 }
410 assert(TET->getTargetExtName() == "spirv.SignedImage");
411 return TET->getTypeParameter(i: 0)->isIntegerTy();
412}
413} // end anonymous namespace
414
415#define GET_GLOBALISEL_IMPL
416#include "SPIRVGenGlobalISel.inc"
417#undef GET_GLOBALISEL_IMPL
418
419SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
420 const SPIRVSubtarget &ST,
421 const RegisterBankInfo &RBI)
422 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
423 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
424 MRI(nullptr),
425#define GET_GLOBALISEL_PREDICATES_INIT
426#include "SPIRVGenGlobalISel.inc"
427#undef GET_GLOBALISEL_PREDICATES_INIT
428#define GET_GLOBALISEL_TEMPORARIES_INIT
429#include "SPIRVGenGlobalISel.inc"
430#undef GET_GLOBALISEL_TEMPORARIES_INIT
431{
432}
433
434void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
435 GISelValueTracking *VT,
436 CodeGenCoverage *CoverageInfo,
437 ProfileSummaryInfo *PSI,
438 BlockFrequencyInfo *BFI) {
439 MRI = &MF.getRegInfo();
440 GR.setCurrentFunc(MF);
441 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
442}
443
444// Ensure that register classes correspond to pattern matching rules.
445void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
446 if (HasVRegsReset == &MF)
447 return;
448 HasVRegsReset = &MF;
449
450 MachineRegisterInfo &MRI = MF.getRegInfo();
451 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
452 Register Reg = Register::index2VirtReg(Index: I);
453 LLT RegType = MRI.getType(Reg);
454 if (RegType.isScalar())
455 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: 64));
456 else if (RegType.isPointer())
457 MRI.setType(VReg: Reg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
458 else if (RegType.isVector())
459 MRI.setType(VReg: Reg, Ty: LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::scalar(SizeInBits: 64)));
460 }
461 for (const auto &MBB : MF) {
462 for (const auto &MI : MBB) {
463 if (isPreISelGenericOpcode(Opcode: MI.getOpcode()))
464 GR.erase(MI: &MI);
465 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
466 continue;
467
468 Register DstReg = MI.getOperand(i: 0).getReg();
469 LLT DstType = MRI.getType(Reg: DstReg);
470 Register SrcReg = MI.getOperand(i: 1).getReg();
471 LLT SrcType = MRI.getType(Reg: SrcReg);
472 if (DstType != SrcType)
473 MRI.setType(VReg: DstReg, Ty: MRI.getType(Reg: SrcReg));
474
475 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(Reg: DstReg);
476 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(Reg: SrcReg);
477 if (DstRC != SrcRC && SrcRC)
478 MRI.setRegClass(Reg: DstReg, RC: SrcRC);
479 }
480 }
481}
482
483// Return true if the type represents a constant register
484static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef,
485 SmallPtrSet<SPIRVType *, 4> &Visited) {
486 OpDef = passCopy(Def: OpDef, MRI);
487
488 if (Visited.contains(Ptr: OpDef))
489 return true;
490 Visited.insert(Ptr: OpDef);
491
492 unsigned Opcode = OpDef->getOpcode();
493 switch (Opcode) {
494 case TargetOpcode::G_CONSTANT:
495 case TargetOpcode::G_FCONSTANT:
496 case TargetOpcode::G_IMPLICIT_DEF:
497 return true;
498 case TargetOpcode::G_INTRINSIC:
499 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
500 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
501 return cast<GIntrinsic>(Val&: *OpDef).getIntrinsicID() ==
502 Intrinsic::spv_const_composite;
503 case TargetOpcode::G_BUILD_VECTOR:
504 case TargetOpcode::G_SPLAT_VECTOR: {
505 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands();
506 i++) {
507 MachineInstr *OpNestedDef =
508 OpDef->getOperand(i).isReg()
509 ? MRI->getVRegDef(Reg: OpDef->getOperand(i).getReg())
510 : nullptr;
511 if (OpNestedDef && !isConstReg(MRI, OpDef: OpNestedDef, Visited))
512 return false;
513 }
514 return true;
515 case SPIRV::OpConstantTrue:
516 case SPIRV::OpConstantFalse:
517 case SPIRV::OpConstantI:
518 case SPIRV::OpConstantF:
519 case SPIRV::OpConstantComposite:
520 case SPIRV::OpConstantCompositeContinuedINTEL:
521 case SPIRV::OpConstantSampler:
522 case SPIRV::OpConstantNull:
523 case SPIRV::OpUndef:
524 case SPIRV::OpConstantFunctionPointerINTEL:
525 return true;
526 }
527 }
528 return false;
529}
530
531// Return true if the virtual register represents a constant
532static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
533 SmallPtrSet<SPIRVType *, 4> Visited;
534 if (MachineInstr *OpDef = MRI->getVRegDef(Reg: OpReg))
535 return isConstReg(MRI, OpDef, Visited);
536 return false;
537}
538
539// TODO(168736): We should make this either a flag in tabelgen
540// or reduce our dependence on the global registry, so we can remove this
541// function. It can easily be missed when new intrinsics are added.
542
543// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
544// definition because they are referenced in the global registry. This is a list
545// of intrinsics that have no side effects other than their references in the
546// global registry.
547static bool intrinsicHasSideEffects(Intrinsic::ID ID) {
548 switch (ID) {
549 // This is not an exhaustive list and may need to be updated.
550 case Intrinsic::spv_all:
551 case Intrinsic::spv_alloca:
552 case Intrinsic::spv_any:
553 case Intrinsic::spv_bitcast:
554 case Intrinsic::spv_const_composite:
555 case Intrinsic::spv_cross:
556 case Intrinsic::spv_degrees:
557 case Intrinsic::spv_distance:
558 case Intrinsic::spv_extractelt:
559 case Intrinsic::spv_extractv:
560 case Intrinsic::spv_faceforward:
561 case Intrinsic::spv_fdot:
562 case Intrinsic::spv_firstbitlow:
563 case Intrinsic::spv_firstbitshigh:
564 case Intrinsic::spv_firstbituhigh:
565 case Intrinsic::spv_frac:
566 case Intrinsic::spv_gep:
567 case Intrinsic::spv_global_offset:
568 case Intrinsic::spv_global_size:
569 case Intrinsic::spv_group_id:
570 case Intrinsic::spv_insertelt:
571 case Intrinsic::spv_insertv:
572 case Intrinsic::spv_isinf:
573 case Intrinsic::spv_isnan:
574 case Intrinsic::spv_lerp:
575 case Intrinsic::spv_length:
576 case Intrinsic::spv_normalize:
577 case Intrinsic::spv_num_subgroups:
578 case Intrinsic::spv_num_workgroups:
579 case Intrinsic::spv_ptrcast:
580 case Intrinsic::spv_radians:
581 case Intrinsic::spv_reflect:
582 case Intrinsic::spv_refract:
583 case Intrinsic::spv_resource_getpointer:
584 case Intrinsic::spv_resource_handlefrombinding:
585 case Intrinsic::spv_resource_handlefromimplicitbinding:
586 case Intrinsic::spv_resource_nonuniformindex:
587 case Intrinsic::spv_resource_sample:
588 case Intrinsic::spv_rsqrt:
589 case Intrinsic::spv_saturate:
590 case Intrinsic::spv_sdot:
591 case Intrinsic::spv_sign:
592 case Intrinsic::spv_smoothstep:
593 case Intrinsic::spv_step:
594 case Intrinsic::spv_subgroup_id:
595 case Intrinsic::spv_subgroup_local_invocation_id:
596 case Intrinsic::spv_subgroup_max_size:
597 case Intrinsic::spv_subgroup_size:
598 case Intrinsic::spv_thread_id:
599 case Intrinsic::spv_thread_id_in_group:
600 case Intrinsic::spv_udot:
601 case Intrinsic::spv_undef:
602 case Intrinsic::spv_value_md:
603 case Intrinsic::spv_workgroup_size:
604 return false;
605 default:
606 return true;
607 }
608}
609
610// TODO(168736): We should make this either a flag in tabelgen
611// or reduce our dependence on the global registry, so we can remove this
612// function. It can easily be missed when new intrinsics are added.
613static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
614 switch (Opcode) {
615 case SPIRV::OpTypeVoid:
616 case SPIRV::OpTypeBool:
617 case SPIRV::OpTypeInt:
618 case SPIRV::OpTypeFloat:
619 case SPIRV::OpTypeVector:
620 case SPIRV::OpTypeMatrix:
621 case SPIRV::OpTypeImage:
622 case SPIRV::OpTypeSampler:
623 case SPIRV::OpTypeSampledImage:
624 case SPIRV::OpTypeArray:
625 case SPIRV::OpTypeRuntimeArray:
626 case SPIRV::OpTypeStruct:
627 case SPIRV::OpTypeOpaque:
628 case SPIRV::OpTypePointer:
629 case SPIRV::OpTypeFunction:
630 case SPIRV::OpTypeEvent:
631 case SPIRV::OpTypeDeviceEvent:
632 case SPIRV::OpTypeReserveId:
633 case SPIRV::OpTypeQueue:
634 case SPIRV::OpTypePipe:
635 case SPIRV::OpTypeForwardPointer:
636 case SPIRV::OpTypePipeStorage:
637 case SPIRV::OpTypeNamedBarrier:
638 case SPIRV::OpTypeAccelerationStructureNV:
639 case SPIRV::OpTypeCooperativeMatrixNV:
640 case SPIRV::OpTypeCooperativeMatrixKHR:
641 return true;
642 default:
643 return false;
644 }
645}
646
647bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
648 // If there are no definitions, then assume there is some other
649 // side-effect that makes this instruction live.
650 if (MI.getNumDefs() == 0)
651 return false;
652
653 for (const auto &MO : MI.all_defs()) {
654 Register Reg = MO.getReg();
655 if (Reg.isPhysical()) {
656 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
657 return false;
658 }
659 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
660 if (UseMI.getOpcode() != SPIRV::OpName) {
661 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
662 return false;
663 }
664 }
665 }
666
667 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
668 MI.isLifetimeMarker()) {
669 LLVM_DEBUG(
670 dbgs()
671 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
672 return false;
673 }
674 if (MI.isPHI()) {
675 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
676 return true;
677 }
678
679 // It is possible that the only side effect is that the instruction is
680 // referenced in the global registry. If that is the only side effect, the
681 // intrinsic is dead.
682 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
683 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
684 const auto &Intr = cast<GIntrinsic>(Val: MI);
685 if (!intrinsicHasSideEffects(ID: Intr.getIntrinsicID())) {
686 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
687 return true;
688 }
689 }
690
691 if (MI.mayStore() || MI.isCall() ||
692 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
693 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
694 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
695 return false;
696 }
697
698 if (isPreISelGenericOpcode(Opcode: MI.getOpcode())) {
699 // TODO: Is there a generic way to check if the opcode has side effects?
700 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
701 return true;
702 }
703
704 if (isOpcodeWithNoSideEffects(Opcode: MI.getOpcode())) {
705 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
706 return true;
707 }
708
709 return false;
710}
711
712void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
713 // Delete the OpName that uses the result if there is one.
714 for (const auto &MO : MI.all_defs()) {
715 Register Reg = MO.getReg();
716 if (Reg.isPhysical())
717 continue;
718 SmallVector<MachineInstr *, 4> UselessOpNames;
719 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
720 assert(UseMI.getOpcode() == SPIRV::OpName &&
721 "There is still a use of the dead function.");
722 UselessOpNames.push_back(Elt: &UseMI);
723 }
724 for (MachineInstr *OpNameMI : UselessOpNames) {
725 GR.invalidateMachineInstr(MI: OpNameMI);
726 OpNameMI->eraseFromParent();
727 }
728 }
729}
730
731void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
732 salvageDebugInfo(MRI: *MRI, MI);
733 GR.invalidateMachineInstr(MI: &MI);
734 removeOpNamesForDeadMI(MI);
735 MI.eraseFromParent();
736}
737
738bool SPIRVInstructionSelector::select(MachineInstr &I) {
739 resetVRegsType(MF&: *I.getParent()->getParent());
740
741 assert(I.getParent() && "Instruction should be in a basic block!");
742 assert(I.getParent()->getParent() && "Instruction should be in a function!");
743
744 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
745 if (isDead(MI: I, MRI: *MRI)) {
746 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
747 removeDeadInstruction(MI&: I);
748 return true;
749 }
750
751 Register Opcode = I.getOpcode();
752 // If it's not a GMIR instruction, we've selected it already.
753 if (!isPreISelGenericOpcode(Opcode)) {
754 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
755 Register DstReg = I.getOperand(i: 0).getReg();
756 Register SrcReg = I.getOperand(i: 1).getReg();
757 auto *Def = MRI->getVRegDef(Reg: SrcReg);
758 if (isTypeFoldingSupported(Opcode: Def->getOpcode()) &&
759 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
760 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
761 bool Res = false;
762 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
763 Register SelectDstReg = Def->getOperand(i: 0).getReg();
764 Res = selectSelect(ResVReg: SelectDstReg, ResType: GR.getSPIRVTypeForVReg(VReg: SelectDstReg),
765 I&: *Def);
766 GR.invalidateMachineInstr(MI: Def);
767 Def->removeFromParent();
768 MRI->replaceRegWith(FromReg: DstReg, ToReg: SelectDstReg);
769 GR.invalidateMachineInstr(MI: &I);
770 I.removeFromParent();
771 } else
772 Res = selectImpl(I, CoverageInfo&: *CoverageInfo);
773 LLVM_DEBUG({
774 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
775 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
776 I.print(dbgs());
777 }
778 });
779 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
780 if (Res) {
781 if (!isTriviallyDead(MI: *Def, MRI: *MRI) && isDead(MI: *Def, MRI: *MRI))
782 DeadMIs.insert(Ptr: Def);
783 return Res;
784 }
785 }
786 MRI->setRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg));
787 MRI->replaceRegWith(FromReg: SrcReg, ToReg: DstReg);
788 GR.invalidateMachineInstr(MI: &I);
789 I.removeFromParent();
790 return true;
791 } else if (I.getNumDefs() == 1) {
792 // Make all vregs 64 bits (for SPIR-V IDs).
793 MRI->setType(VReg: I.getOperand(i: 0).getReg(), Ty: LLT::scalar(SizeInBits: 64));
794 }
795 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
796 }
797
798 if (DeadMIs.contains(Ptr: &I)) {
799 // if the instruction has been already made dead by folding it away
800 // erase it
801 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
802 removeDeadInstruction(MI&: I);
803 return true;
804 }
805
806 if (I.getNumOperands() != I.getNumExplicitOperands()) {
807 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
808 return false;
809 }
810
811 // Common code for getting return reg+type, and removing selected instr
812 // from parent occurs here. Instr-specific selection happens in spvSelect().
813 bool HasDefs = I.getNumDefs() > 0;
814 Register ResVReg = HasDefs ? I.getOperand(i: 0).getReg() : Register(0);
815 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(VReg: ResVReg) : nullptr;
816 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
817 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
818 if (spvSelect(ResVReg, ResType, I)) {
819 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
820 for (unsigned i = 0; i < I.getNumDefs(); ++i)
821 MRI->setType(VReg: I.getOperand(i).getReg(), Ty: LLT::scalar(SizeInBits: 64));
822 GR.invalidateMachineInstr(MI: &I);
823 I.removeFromParent();
824 return true;
825 }
826 return false;
827}
828
829static bool mayApplyGenericSelection(unsigned Opcode) {
830 switch (Opcode) {
831 case TargetOpcode::G_CONSTANT:
832 case TargetOpcode::G_FCONSTANT:
833 return false;
834 case TargetOpcode::G_SADDO:
835 case TargetOpcode::G_SSUBO:
836 return true;
837 }
838 return isTypeFoldingSupported(Opcode);
839}
840
841bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
842 MachineInstr &I) const {
843 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: DestReg);
844 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(Reg: SrcReg);
845 if (DstRC != SrcRC && SrcRC)
846 MRI->setRegClass(Reg: DestReg, RC: SrcRC);
847 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
848 MCID: TII.get(Opcode: TargetOpcode::COPY))
849 .addDef(RegNo: DestReg)
850 .addUse(RegNo: SrcReg)
851 .constrainAllUses(TII, TRI, RBI);
852}
853
854bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
855 const SPIRVType *ResType,
856 MachineInstr &I) const {
857 const unsigned Opcode = I.getOpcode();
858 if (mayApplyGenericSelection(Opcode))
859 return selectImpl(I, CoverageInfo&: *CoverageInfo);
860 switch (Opcode) {
861 case TargetOpcode::G_CONSTANT:
862 case TargetOpcode::G_FCONSTANT:
863 return selectConst(ResVReg, ResType, I);
864 case TargetOpcode::G_GLOBAL_VALUE:
865 return selectGlobalValue(ResVReg, I);
866 case TargetOpcode::G_IMPLICIT_DEF:
867 return selectOpUndef(ResVReg, ResType, I);
868 case TargetOpcode::G_FREEZE:
869 return selectFreeze(ResVReg, ResType, I);
870
871 case TargetOpcode::G_INTRINSIC:
872 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
873 case TargetOpcode::G_INTRINSIC_CONVERGENT:
874 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
875 return selectIntrinsic(ResVReg, ResType, I);
876 case TargetOpcode::G_BITREVERSE:
877 return selectBitreverse(ResVReg, ResType, I);
878
879 case TargetOpcode::G_BUILD_VECTOR:
880 return selectBuildVector(ResVReg, ResType, I);
881 case TargetOpcode::G_SPLAT_VECTOR:
882 return selectSplatVector(ResVReg, ResType, I);
883
884 case TargetOpcode::G_SHUFFLE_VECTOR: {
885 MachineBasicBlock &BB = *I.getParent();
886 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
887 .addDef(RegNo: ResVReg)
888 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
889 .addUse(RegNo: I.getOperand(i: 1).getReg())
890 .addUse(RegNo: I.getOperand(i: 2).getReg());
891 for (auto V : I.getOperand(i: 3).getShuffleMask())
892 MIB.addImm(Val: V);
893 return MIB.constrainAllUses(TII, TRI, RBI);
894 }
895 case TargetOpcode::G_MEMMOVE:
896 case TargetOpcode::G_MEMCPY:
897 case TargetOpcode::G_MEMSET:
898 return selectMemOperation(ResVReg, I);
899
900 case TargetOpcode::G_ICMP:
901 return selectICmp(ResVReg, ResType, I);
902 case TargetOpcode::G_FCMP:
903 return selectFCmp(ResVReg, ResType, I);
904
905 case TargetOpcode::G_FRAME_INDEX:
906 return selectFrameIndex(ResVReg, ResType, I);
907
908 case TargetOpcode::G_LOAD:
909 return selectLoad(ResVReg, ResType, I);
910 case TargetOpcode::G_STORE:
911 return selectStore(I);
912
913 case TargetOpcode::G_BR:
914 return selectBranch(I);
915 case TargetOpcode::G_BRCOND:
916 return selectBranchCond(I);
917
918 case TargetOpcode::G_PHI:
919 return selectPhi(ResVReg, ResType, I);
920
921 case TargetOpcode::G_FPTOSI:
922 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
923 case TargetOpcode::G_FPTOUI:
924 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
925
926 case TargetOpcode::G_FPTOSI_SAT:
927 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
928 case TargetOpcode::G_FPTOUI_SAT:
929 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
930
931 case TargetOpcode::G_SITOFP:
932 return selectIToF(ResVReg, ResType, I, IsSigned: true, Opcode: SPIRV::OpConvertSToF);
933 case TargetOpcode::G_UITOFP:
934 return selectIToF(ResVReg, ResType, I, IsSigned: false, Opcode: SPIRV::OpConvertUToF);
935
936 case TargetOpcode::G_CTPOP:
937 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitCount);
938 case TargetOpcode::G_SMIN:
939 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_min, GLInst: GL::SMin);
940 case TargetOpcode::G_UMIN:
941 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_min, GLInst: GL::UMin);
942
943 case TargetOpcode::G_SMAX:
944 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_max, GLInst: GL::SMax);
945 case TargetOpcode::G_UMAX:
946 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_max, GLInst: GL::UMax);
947
948 case TargetOpcode::G_SCMP:
949 return selectSUCmp(ResVReg, ResType, I, IsSigned: true);
950 case TargetOpcode::G_UCMP:
951 return selectSUCmp(ResVReg, ResType, I, IsSigned: false);
952 case TargetOpcode::G_LROUND:
953 case TargetOpcode::G_LLROUND: {
954 Register regForLround =
955 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ResVReg), Name: "lround");
956 MRI->setRegClass(Reg: regForLround, RC: &SPIRV::iIDRegClass);
957 GR.assignSPIRVTypeToVReg(Type: GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg()),
958 VReg: regForLround, MF: *(I.getParent()->getParent()));
959 selectExtInstForLRound(ResVReg: regForLround, ResType: GR.getSPIRVTypeForVReg(VReg: regForLround),
960 I, CLInst: CL::round, GLInst: GL::Round);
961 MachineBasicBlock &BB = *I.getParent();
962 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConvertFToS))
963 .addDef(RegNo: ResVReg)
964 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
965 .addUse(RegNo: regForLround);
966 return MIB.constrainAllUses(TII, TRI, RBI);
967 }
968 case TargetOpcode::G_STRICT_FMA:
969 case TargetOpcode::G_FMA: {
970 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
971 MachineBasicBlock &BB = *I.getParent();
972 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFmaKHR))
973 .addDef(RegNo: ResVReg)
974 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
975 .addUse(RegNo: I.getOperand(i: 1).getReg())
976 .addUse(RegNo: I.getOperand(i: 2).getReg())
977 .addUse(RegNo: I.getOperand(i: 3).getReg())
978 .setMIFlags(I.getFlags());
979 return MIB.constrainAllUses(TII, TRI, RBI);
980 }
981 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fma, GLInst: GL::Fma);
982 }
983
984 case TargetOpcode::G_STRICT_FLDEXP:
985 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ldexp);
986
987 case TargetOpcode::G_FPOW:
988 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pow, GLInst: GL::Pow);
989 case TargetOpcode::G_FPOWI:
990 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pown);
991
992 case TargetOpcode::G_FEXP:
993 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp, GLInst: GL::Exp);
994 case TargetOpcode::G_FEXP2:
995 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp2, GLInst: GL::Exp2);
996 case TargetOpcode::G_FMODF:
997 return selectModf(ResVReg, ResType, I);
998
999 case TargetOpcode::G_FLOG:
1000 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log, GLInst: GL::Log);
1001 case TargetOpcode::G_FLOG2:
1002 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log2, GLInst: GL::Log2);
1003 case TargetOpcode::G_FLOG10:
1004 return selectLog10(ResVReg, ResType, I);
1005
1006 case TargetOpcode::G_FABS:
1007 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fabs, GLInst: GL::FAbs);
1008 case TargetOpcode::G_ABS:
1009 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_abs, GLInst: GL::SAbs);
1010
1011 case TargetOpcode::G_FMINNUM:
1012 case TargetOpcode::G_FMINIMUM:
1013 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmin, GLInst: GL::NMin);
1014 case TargetOpcode::G_FMAXNUM:
1015 case TargetOpcode::G_FMAXIMUM:
1016 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmax, GLInst: GL::NMax);
1017
1018 case TargetOpcode::G_FCOPYSIGN:
1019 return selectExtInst(ResVReg, ResType, I, CLInst: CL::copysign);
1020
1021 case TargetOpcode::G_FCEIL:
1022 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ceil, GLInst: GL::Ceil);
1023 case TargetOpcode::G_FFLOOR:
1024 return selectExtInst(ResVReg, ResType, I, CLInst: CL::floor, GLInst: GL::Floor);
1025
1026 case TargetOpcode::G_FCOS:
1027 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cos, GLInst: GL::Cos);
1028 case TargetOpcode::G_FSIN:
1029 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sin, GLInst: GL::Sin);
1030 case TargetOpcode::G_FTAN:
1031 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tan, GLInst: GL::Tan);
1032 case TargetOpcode::G_FACOS:
1033 return selectExtInst(ResVReg, ResType, I, CLInst: CL::acos, GLInst: GL::Acos);
1034 case TargetOpcode::G_FASIN:
1035 return selectExtInst(ResVReg, ResType, I, CLInst: CL::asin, GLInst: GL::Asin);
1036 case TargetOpcode::G_FATAN:
1037 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan, GLInst: GL::Atan);
1038 case TargetOpcode::G_FATAN2:
1039 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan2, GLInst: GL::Atan2);
1040 case TargetOpcode::G_FCOSH:
1041 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cosh, GLInst: GL::Cosh);
1042 case TargetOpcode::G_FSINH:
1043 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sinh, GLInst: GL::Sinh);
1044 case TargetOpcode::G_FTANH:
1045 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tanh, GLInst: GL::Tanh);
1046
1047 case TargetOpcode::G_STRICT_FSQRT:
1048 case TargetOpcode::G_FSQRT:
1049 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sqrt, GLInst: GL::Sqrt);
1050
1051 case TargetOpcode::G_CTTZ:
1052 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1053 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ctz);
1054 case TargetOpcode::G_CTLZ:
1055 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1056 return selectExtInst(ResVReg, ResType, I, CLInst: CL::clz);
1057
1058 case TargetOpcode::G_INTRINSIC_ROUND:
1059 return selectExtInst(ResVReg, ResType, I, CLInst: CL::round, GLInst: GL::Round);
1060 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1061 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1062 case TargetOpcode::G_INTRINSIC_TRUNC:
1063 return selectExtInst(ResVReg, ResType, I, CLInst: CL::trunc, GLInst: GL::Trunc);
1064 case TargetOpcode::G_FRINT:
1065 case TargetOpcode::G_FNEARBYINT:
1066 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1067
1068 case TargetOpcode::G_SMULH:
1069 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_mul_hi);
1070 case TargetOpcode::G_UMULH:
1071 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_mul_hi);
1072
1073 case TargetOpcode::G_SADDSAT:
1074 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_add_sat);
1075 case TargetOpcode::G_UADDSAT:
1076 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_add_sat);
1077 case TargetOpcode::G_SSUBSAT:
1078 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_sub_sat);
1079 case TargetOpcode::G_USUBSAT:
1080 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_sub_sat);
1081
1082 case TargetOpcode::G_FFREXP:
1083 return selectFrexp(ResVReg, ResType, I);
1084
1085 case TargetOpcode::G_UADDO:
1086 return selectOverflowArith(ResVReg, ResType, I,
1087 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1088 ? SPIRV::OpIAddCarryV
1089 : SPIRV::OpIAddCarryS);
1090 case TargetOpcode::G_USUBO:
1091 return selectOverflowArith(ResVReg, ResType, I,
1092 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1093 ? SPIRV::OpISubBorrowV
1094 : SPIRV::OpISubBorrowS);
1095 case TargetOpcode::G_UMULO:
1096 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpUMulExtended);
1097 case TargetOpcode::G_SMULO:
1098 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpSMulExtended);
1099
1100 case TargetOpcode::G_SEXT:
1101 return selectExt(ResVReg, ResType, I, IsSigned: true);
1102 case TargetOpcode::G_ANYEXT:
1103 case TargetOpcode::G_ZEXT:
1104 return selectExt(ResVReg, ResType, I, IsSigned: false);
1105 case TargetOpcode::G_TRUNC:
1106 return selectTrunc(ResVReg, ResType, I);
1107 case TargetOpcode::G_FPTRUNC:
1108 case TargetOpcode::G_FPEXT:
1109 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpFConvert);
1110
1111 case TargetOpcode::G_PTRTOINT:
1112 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertPtrToU);
1113 case TargetOpcode::G_INTTOPTR:
1114 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertUToPtr);
1115 case TargetOpcode::G_BITCAST:
1116 return selectBitcast(ResVReg, ResType, I);
1117 case TargetOpcode::G_ADDRSPACE_CAST:
1118 return selectAddrSpaceCast(ResVReg, ResType, I);
1119 case TargetOpcode::G_PTR_ADD: {
1120 // Currently, we get G_PTR_ADD only applied to global variables.
1121 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1122 Register GV = I.getOperand(i: 1).getReg();
1123 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(RegNo: GV);
1124 (void)II;
1125 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1126 (*II).getOpcode() == TargetOpcode::COPY ||
1127 (*II).getOpcode() == SPIRV::OpVariable) &&
1128 getImm(I.getOperand(2), MRI));
1129 // It may be the initialization of a global variable.
1130 bool IsGVInit = false;
1131 for (MachineRegisterInfo::use_instr_iterator
1132 UseIt = MRI->use_instr_begin(RegNo: I.getOperand(i: 0).getReg()),
1133 UseEnd = MRI->use_instr_end();
1134 UseIt != UseEnd; UseIt = std::next(x: UseIt)) {
1135 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1136 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1137 (*UseIt).getOpcode() == SPIRV::OpVariable) {
1138 IsGVInit = true;
1139 break;
1140 }
1141 }
1142 MachineBasicBlock &BB = *I.getParent();
1143 if (!IsGVInit) {
1144 SPIRVType *GVType = GR.getSPIRVTypeForVReg(VReg: GV);
1145 SPIRVType *GVPointeeType = GR.getPointeeType(PtrType: GVType);
1146 SPIRVType *ResPointeeType = GR.getPointeeType(PtrType: ResType);
1147 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1148 // Build a new virtual register that is associated with the required
1149 // data type.
1150 Register NewVReg = MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: GV));
1151 MRI->setRegClass(Reg: NewVReg, RC: MRI->getRegClass(Reg: GV));
1152 // Having a correctly typed base we are ready to build the actually
1153 // required GEP. It may not be a constant though, because all Operands
1154 // of OpSpecConstantOp is to originate from other const instructions,
1155 // and only the AccessChain named opcodes accept a global OpVariable
1156 // instruction. We can't use an AccessChain opcode because of the type
1157 // mismatch between result and base types.
1158 if (!GR.isBitcastCompatible(Type1: ResType, Type2: GVType))
1159 report_fatal_error(
1160 reason: "incompatible result and operand types in a bitcast");
1161 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
1162 MachineInstrBuilder MIB =
1163 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitcast))
1164 .addDef(RegNo: NewVReg)
1165 .addUse(RegNo: ResTypeReg)
1166 .addUse(RegNo: GV);
1167 return MIB.constrainAllUses(TII, TRI, RBI) &&
1168 BuildMI(BB, I, MIMD: I.getDebugLoc(),
1169 MCID: TII.get(Opcode: STI.isLogicalSPIRV()
1170 ? SPIRV::OpInBoundsAccessChain
1171 : SPIRV::OpInBoundsPtrAccessChain))
1172 .addDef(RegNo: ResVReg)
1173 .addUse(RegNo: ResTypeReg)
1174 .addUse(RegNo: NewVReg)
1175 .addUse(RegNo: I.getOperand(i: 2).getReg())
1176 .constrainAllUses(TII, TRI, RBI);
1177 } else {
1178 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1179 .addDef(RegNo: ResVReg)
1180 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1181 .addImm(
1182 Val: static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1183 .addUse(RegNo: GV)
1184 .addUse(RegNo: I.getOperand(i: 2).getReg())
1185 .constrainAllUses(TII, TRI, RBI);
1186 }
1187 }
1188 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1189 // initialize a global variable with a constant expression (e.g., the test
1190 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1191 Register Idx = buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
1192 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1193 .addDef(RegNo: ResVReg)
1194 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1195 .addImm(Val: static_cast<uint32_t>(
1196 SPIRV::Opcode::InBoundsPtrAccessChain))
1197 .addUse(RegNo: GV)
1198 .addUse(RegNo: Idx)
1199 .addUse(RegNo: I.getOperand(i: 2).getReg());
1200 return MIB.constrainAllUses(TII, TRI, RBI);
1201 }
1202
1203 case TargetOpcode::G_ATOMICRMW_OR:
1204 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicOr);
1205 case TargetOpcode::G_ATOMICRMW_ADD:
1206 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicIAdd);
1207 case TargetOpcode::G_ATOMICRMW_AND:
1208 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicAnd);
1209 case TargetOpcode::G_ATOMICRMW_MAX:
1210 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMax);
1211 case TargetOpcode::G_ATOMICRMW_MIN:
1212 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMin);
1213 case TargetOpcode::G_ATOMICRMW_SUB:
1214 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicISub);
1215 case TargetOpcode::G_ATOMICRMW_XOR:
1216 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicXor);
1217 case TargetOpcode::G_ATOMICRMW_UMAX:
1218 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMax);
1219 case TargetOpcode::G_ATOMICRMW_UMIN:
1220 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMin);
1221 case TargetOpcode::G_ATOMICRMW_XCHG:
1222 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicExchange);
1223 case TargetOpcode::G_ATOMIC_CMPXCHG:
1224 return selectAtomicCmpXchg(ResVReg, ResType, I);
1225
1226 case TargetOpcode::G_ATOMICRMW_FADD:
1227 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT);
1228 case TargetOpcode::G_ATOMICRMW_FSUB:
1229 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1230 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT,
1231 NegateOpcode: ResType->getOpcode() == SPIRV::OpTypeVector
1232 ? SPIRV::OpFNegateV
1233 : SPIRV::OpFNegate);
1234 case TargetOpcode::G_ATOMICRMW_FMIN:
1235 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMinEXT);
1236 case TargetOpcode::G_ATOMICRMW_FMAX:
1237 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMaxEXT);
1238
1239 case TargetOpcode::G_FENCE:
1240 return selectFence(I);
1241
1242 case TargetOpcode::G_STACKSAVE:
1243 return selectStackSave(ResVReg, ResType, I);
1244 case TargetOpcode::G_STACKRESTORE:
1245 return selectStackRestore(I);
1246
1247 case TargetOpcode::G_UNMERGE_VALUES:
1248 return selectUnmergeValues(I);
1249
1250 // Discard gen opcodes for intrinsics which we do not expect to actually
1251 // represent code after lowering or intrinsics which are not implemented but
1252 // should not crash when found in a customer's LLVM IR input.
1253 case TargetOpcode::G_TRAP:
1254 case TargetOpcode::G_UBSANTRAP:
1255 case TargetOpcode::DBG_LABEL:
1256 return true;
1257 case TargetOpcode::G_DEBUGTRAP:
1258 return selectDebugTrap(ResVReg, ResType, I);
1259
1260 default:
1261 return false;
1262 }
1263}
1264
1265bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1266 const SPIRVType *ResType,
1267 MachineInstr &I) const {
1268 unsigned Opcode = SPIRV::OpNop;
1269 MachineBasicBlock &BB = *I.getParent();
1270 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1271 .constrainAllUses(TII, TRI, RBI);
1272}
1273
1274bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1275 const SPIRVType *ResType,
1276 MachineInstr &I,
1277 GL::GLSLExtInst GLInst) const {
1278 if (!STI.canUseExtInstSet(
1279 E: SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) {
1280 std::string DiagMsg;
1281 raw_string_ostream OS(DiagMsg);
1282 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
1283 DiagMsg += " is only supported with the GLSL extended instruction set.\n";
1284 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1285 }
1286 return selectExtInst(ResVReg, ResType, I,
1287 ExtInsts: {{SPIRV::InstructionSet::GLSL_std_450, GLInst}});
1288}
1289
1290bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1291 const SPIRVType *ResType,
1292 MachineInstr &I,
1293 CL::OpenCLExtInst CLInst) const {
1294 return selectExtInst(ResVReg, ResType, I,
1295 ExtInsts: {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
1296}
1297
1298bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1299 const SPIRVType *ResType,
1300 MachineInstr &I,
1301 CL::OpenCLExtInst CLInst,
1302 GL::GLSLExtInst GLInst) const {
1303 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1304 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1305 return selectExtInst(ResVReg, ResType, I, ExtInsts);
1306}
1307
1308bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1309 const SPIRVType *ResType,
1310 MachineInstr &I,
1311 const ExtInstList &Insts) const {
1312
1313 for (const auto &Ex : Insts) {
1314 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1315 uint32_t Opcode = Ex.second;
1316 if (STI.canUseExtInstSet(E: Set)) {
1317 MachineBasicBlock &BB = *I.getParent();
1318 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1319 .addDef(RegNo: ResVReg)
1320 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1321 .addImm(Val: static_cast<uint32_t>(Set))
1322 .addImm(Val: Opcode)
1323 .setMIFlags(I.getFlags());
1324 const unsigned NumOps = I.getNumOperands();
1325 unsigned Index = 1;
1326 if (Index < NumOps &&
1327 I.getOperand(i: Index).getType() ==
1328 MachineOperand::MachineOperandType::MO_IntrinsicID)
1329 Index = 2;
1330 for (; Index < NumOps; ++Index)
1331 MIB.add(MO: I.getOperand(i: Index));
1332 return MIB.constrainAllUses(TII, TRI, RBI);
1333 }
1334 }
1335 return false;
1336}
1337bool SPIRVInstructionSelector::selectExtInstForLRound(
1338 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
1339 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst) const {
1340 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1341 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1342 return selectExtInstForLRound(ResVReg, ResType, I, ExtInsts);
1343}
1344
1345bool SPIRVInstructionSelector::selectExtInstForLRound(
1346 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
1347 const ExtInstList &Insts) const {
1348 for (const auto &Ex : Insts) {
1349 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1350 uint32_t Opcode = Ex.second;
1351 if (STI.canUseExtInstSet(E: Set)) {
1352 MachineBasicBlock &BB = *I.getParent();
1353 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1354 .addDef(RegNo: ResVReg)
1355 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1356 .addImm(Val: static_cast<uint32_t>(Set))
1357 .addImm(Val: Opcode);
1358 const unsigned NumOps = I.getNumOperands();
1359 unsigned Index = 1;
1360 if (Index < NumOps &&
1361 I.getOperand(i: Index).getType() ==
1362 MachineOperand::MachineOperandType::MO_IntrinsicID)
1363 Index = 2;
1364 for (; Index < NumOps; ++Index)
1365 MIB.add(MO: I.getOperand(i: Index));
1366 MIB.constrainAllUses(TII, TRI, RBI);
1367 return true;
1368 }
1369 }
1370 return false;
1371}
1372
1373bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1374 const SPIRVType *ResType,
1375 MachineInstr &I) const {
1376 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1377 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1378 for (const auto &Ex : ExtInsts) {
1379 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1380 uint32_t Opcode = Ex.second;
1381 if (!STI.canUseExtInstSet(E: Set))
1382 continue;
1383
1384 MachineIRBuilder MIRBuilder(I);
1385 SPIRVType *PointeeTy = GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg());
1386 const SPIRVType *PointerType = GR.getOrCreateSPIRVPointerType(
1387 BaseType: PointeeTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
1388 Register PointerVReg =
1389 createVirtualRegister(SpvType: PointerType, GR: &GR, MRI, MF: MRI->getMF());
1390
1391 auto It = getOpVariableMBBIt(I);
1392 auto MIB = BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(),
1393 MCID: TII.get(Opcode: SPIRV::OpVariable))
1394 .addDef(RegNo: PointerVReg)
1395 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointerType))
1396 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
1397 .constrainAllUses(TII, TRI, RBI);
1398
1399 MIB = MIB &
1400 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1401 .addDef(RegNo: ResVReg)
1402 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1403 .addImm(Val: static_cast<uint32_t>(Ex.first))
1404 .addImm(Val: Opcode)
1405 .add(MO: I.getOperand(i: 2))
1406 .addUse(RegNo: PointerVReg)
1407 .constrainAllUses(TII, TRI, RBI);
1408
1409 MIB = MIB &
1410 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1411 .addDef(RegNo: I.getOperand(i: 1).getReg())
1412 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointeeTy))
1413 .addUse(RegNo: PointerVReg)
1414 .constrainAllUses(TII, TRI, RBI);
1415 return MIB;
1416 }
1417 return false;
1418}
1419
1420bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1421 const SPIRVType *ResType,
1422 MachineInstr &I,
1423 std::vector<Register> Srcs,
1424 unsigned Opcode) const {
1425 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1426 .addDef(RegNo: ResVReg)
1427 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
1428 for (Register SReg : Srcs) {
1429 MIB.addUse(RegNo: SReg);
1430 }
1431 return MIB.constrainAllUses(TII, TRI, RBI);
1432}
1433
1434bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1435 const SPIRVType *ResType,
1436 MachineInstr &I,
1437 unsigned Opcode) const {
1438 if (STI.isPhysicalSPIRV() && I.getOperand(i: 1).isReg()) {
1439 Register SrcReg = I.getOperand(i: 1).getReg();
1440 bool IsGV = false;
1441 for (MachineRegisterInfo::def_instr_iterator DefIt =
1442 MRI->def_instr_begin(RegNo: SrcReg);
1443 DefIt != MRI->def_instr_end(); DefIt = std::next(x: DefIt)) {
1444 unsigned DefOpCode = DefIt->getOpcode();
1445 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1446 // We need special handling to look through the type assignment or the
1447 // COPY pseudo-op and see if this is a constant or a global.
1448 if (auto *VRD = getVRegDef(MRI&: *MRI, Reg: DefIt->getOperand(i: 1).getReg()))
1449 DefOpCode = VRD->getOpcode();
1450 }
1451 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1452 DefOpCode == TargetOpcode::G_CONSTANT ||
1453 DefOpCode == SPIRV::OpVariable || DefOpCode == SPIRV::OpConstantI) {
1454 IsGV = true;
1455 break;
1456 }
1457 }
1458 if (IsGV) {
1459 uint32_t SpecOpcode = 0;
1460 switch (Opcode) {
1461 case SPIRV::OpConvertPtrToU:
1462 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1463 break;
1464 case SPIRV::OpConvertUToPtr:
1465 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1466 break;
1467 }
1468 if (SpecOpcode)
1469 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1470 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1471 .addDef(RegNo: ResVReg)
1472 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1473 .addImm(Val: SpecOpcode)
1474 .addUse(RegNo: SrcReg)
1475 .constrainAllUses(TII, TRI, RBI);
1476 }
1477 }
1478 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {I.getOperand(i: 1).getReg()},
1479 Opcode);
1480}
1481
1482bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1483 const SPIRVType *ResType,
1484 MachineInstr &I) const {
1485 Register OpReg = I.getOperand(i: 1).getReg();
1486 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
1487 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
1488 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
1489 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitcast);
1490}
1491
1492static void addMemoryOperands(MachineMemOperand *MemOp,
1493 MachineInstrBuilder &MIB,
1494 MachineIRBuilder &MIRBuilder,
1495 SPIRVGlobalRegistry &GR) {
1496 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1497 if (MemOp->isVolatile())
1498 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1499 if (MemOp->isNonTemporal())
1500 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1501 if (MemOp->getAlign().value())
1502 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1503
1504 [[maybe_unused]] MachineInstr *AliasList = nullptr;
1505 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
1506 const SPIRVSubtarget *ST =
1507 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1508 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
1509 if (auto *MD = MemOp->getAAInfo().Scope) {
1510 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1511 if (AliasList)
1512 SpvMemOp |=
1513 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
1514 }
1515 if (auto *MD = MemOp->getAAInfo().NoAlias) {
1516 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1517 if (NoAliasList)
1518 SpvMemOp |=
1519 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
1520 }
1521 }
1522
1523 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1524 MIB.addImm(Val: SpvMemOp);
1525 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1526 MIB.addImm(Val: MemOp->getAlign().value());
1527 if (AliasList)
1528 MIB.addUse(RegNo: AliasList->getOperand(i: 0).getReg());
1529 if (NoAliasList)
1530 MIB.addUse(RegNo: NoAliasList->getOperand(i: 0).getReg());
1531 }
1532}
1533
1534static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
1535 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1536 if (Flags & MachineMemOperand::Flags::MOVolatile)
1537 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1538 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1539 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1540
1541 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1542 MIB.addImm(Val: SpvMemOp);
1543}
1544
1545bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1546 const SPIRVType *ResType,
1547 MachineInstr &I) const {
1548 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1549 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1550
1551 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1552 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1553 if (IntPtrDef &&
1554 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1555 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1556 SPIRVType *HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1557 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1558 Register NewHandleReg =
1559 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1560 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1561 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1562 return false;
1563 }
1564
1565 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1566 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewHandleReg, IdxReg,
1567 Loc: I.getDebugLoc(), Pos&: I);
1568 }
1569 }
1570
1571 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1572 .addDef(RegNo: ResVReg)
1573 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1574 .addUse(RegNo: Ptr);
1575 if (!I.getNumMemOperands()) {
1576 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1577 I.getOpcode() ==
1578 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1579 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1580 } else {
1581 MachineIRBuilder MIRBuilder(I);
1582 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1583 }
1584 return MIB.constrainAllUses(TII, TRI, RBI);
1585}
1586
1587bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1588 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1589 Register StoreVal = I.getOperand(i: 0 + OpOffset).getReg();
1590 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1591
1592 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1593 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1594 if (IntPtrDef &&
1595 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1596 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1597 Register NewHandleReg =
1598 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1599 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1600 SPIRVType *HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1601 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1602 return false;
1603 }
1604
1605 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1606 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1607 auto BMI = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1608 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
1609 .addUse(RegNo: NewHandleReg)
1610 .addUse(RegNo: IdxReg)
1611 .addUse(RegNo: StoreVal);
1612
1613 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(Ty: HandleType);
1614 if (sampledTypeIsSignedInteger(HandleType: LLVMHandleType))
1615 BMI.addImm(Val: 0x1000); // SignExtend
1616
1617 return BMI.constrainAllUses(TII, TRI, RBI);
1618 }
1619 }
1620
1621 MachineBasicBlock &BB = *I.getParent();
1622 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpStore))
1623 .addUse(RegNo: Ptr)
1624 .addUse(RegNo: StoreVal);
1625 if (!I.getNumMemOperands()) {
1626 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1627 I.getOpcode() ==
1628 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1629 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1630 } else {
1631 MachineIRBuilder MIRBuilder(I);
1632 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1633 }
1634 return MIB.constrainAllUses(TII, TRI, RBI);
1635}
1636
1637bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
1638 const SPIRVType *ResType,
1639 MachineInstr &I) const {
1640 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1641 report_fatal_error(
1642 reason: "llvm.stacksave intrinsic: this instruction requires the following "
1643 "SPIR-V extension: SPV_INTEL_variable_length_array",
1644 gen_crash_diag: false);
1645 MachineBasicBlock &BB = *I.getParent();
1646 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSaveMemoryINTEL))
1647 .addDef(RegNo: ResVReg)
1648 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1649 .constrainAllUses(TII, TRI, RBI);
1650}
1651
1652bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
1653 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1654 report_fatal_error(
1655 reason: "llvm.stackrestore intrinsic: this instruction requires the following "
1656 "SPIR-V extension: SPV_INTEL_variable_length_array",
1657 gen_crash_diag: false);
1658 if (!I.getOperand(i: 0).isReg())
1659 return false;
1660 MachineBasicBlock &BB = *I.getParent();
1661 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpRestoreMemoryINTEL))
1662 .addUse(RegNo: I.getOperand(i: 0).getReg())
1663 .constrainAllUses(TII, TRI, RBI);
1664}
1665
1666Register
1667SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
1668 MachineIRBuilder MIRBuilder(I);
1669 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1670
1671 // TODO: check if we have such GV, add init, use buildGlobalVariable.
1672 unsigned Num = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1673 Function &CurFunction = GR.CurMF->getFunction();
1674 Type *LLVMArrTy =
1675 ArrayType::get(ElementType: IntegerType::get(C&: CurFunction.getContext(), NumBits: 8), NumElements: Num);
1676 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
1677 true, GlobalValue::InternalLinkage,
1678 Constant::getNullValue(Ty: LLVMArrTy));
1679
1680 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1681 Type *ArrTy = ArrayType::get(ElementType: ValTy, NumElements: Num);
1682 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
1683 BaseType: ArrTy, MIRBuilder, SC: SPIRV::StorageClass::UniformConstant);
1684
1685 SPIRVType *SpvArrTy = GR.getOrCreateSPIRVType(
1686 Type: ArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::None, EmitIR: false);
1687
1688 unsigned Val = getIConstVal(ConstReg: I.getOperand(i: 1).getReg(), MRI);
1689 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvType: SpvArrTy, TII);
1690
1691 Register VarReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1692 auto MIBVar =
1693 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1694 .addDef(RegNo: VarReg)
1695 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VarTy))
1696 .addImm(Val: SPIRV::StorageClass::UniformConstant)
1697 .addUse(RegNo: Const);
1698 if (!MIBVar.constrainAllUses(TII, TRI, RBI))
1699 return Register();
1700
1701 GR.add(V: GV, MI: MIBVar);
1702 GR.addGlobalObject(V: GV, MF: GR.CurMF, R: VarReg);
1703
1704 buildOpDecorate(Reg: VarReg, I, TII, Dec: SPIRV::Decoration::Constant, DecArgs: {});
1705 return VarReg;
1706}
1707
1708bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
1709 Register SrcReg) const {
1710 MachineBasicBlock &BB = *I.getParent();
1711 Register DstReg = I.getOperand(i: 0).getReg();
1712 SPIRVType *DstTy = GR.getSPIRVTypeForVReg(VReg: DstReg);
1713 SPIRVType *SrcTy = GR.getSPIRVTypeForVReg(VReg: SrcReg);
1714 if (GR.getPointeeType(PtrType: DstTy) != GR.getPointeeType(PtrType: SrcTy))
1715 report_fatal_error(reason: "OpCopyMemory requires operands to have the same type");
1716 uint64_t CopySize = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1717 SPIRVType *PointeeTy = GR.getPointeeType(PtrType: DstTy);
1718 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(Ty: PointeeTy);
1719 if (!LLVMPointeeTy)
1720 report_fatal_error(
1721 reason: "Unable to determine pointee type size for OpCopyMemory");
1722 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
1723 if (CopySize != DL.getTypeStoreSize(Ty: const_cast<Type *>(LLVMPointeeTy)))
1724 report_fatal_error(
1725 reason: "OpCopyMemory requires the size to match the pointee type size");
1726 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemory))
1727 .addUse(RegNo: DstReg)
1728 .addUse(RegNo: SrcReg);
1729 if (I.getNumMemOperands()) {
1730 MachineIRBuilder MIRBuilder(I);
1731 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1732 }
1733 return MIB.constrainAllUses(TII, TRI, RBI);
1734}
1735
1736bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
1737 Register SrcReg) const {
1738 MachineBasicBlock &BB = *I.getParent();
1739 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemorySized))
1740 .addUse(RegNo: I.getOperand(i: 0).getReg())
1741 .addUse(RegNo: SrcReg)
1742 .addUse(RegNo: I.getOperand(i: 2).getReg());
1743 if (I.getNumMemOperands()) {
1744 MachineIRBuilder MIRBuilder(I);
1745 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1746 }
1747 return MIB.constrainAllUses(TII, TRI, RBI);
1748}
1749
1750bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
1751 MachineInstr &I) const {
1752 Register SrcReg = I.getOperand(i: 1).getReg();
1753 bool Result = true;
1754 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
1755 Register VarReg = getOrCreateMemSetGlobal(I);
1756 if (!VarReg.isValid())
1757 return false;
1758 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1759 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
1760 BaseType: ValTy, I, SC: SPIRV::StorageClass::UniformConstant);
1761 SrcReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1762 Result &= selectOpWithSrcs(ResVReg: SrcReg, ResType: SourceTy, I, Srcs: {VarReg}, Opcode: SPIRV::OpBitcast);
1763 }
1764 if (STI.isLogicalSPIRV()) {
1765 Result &= selectCopyMemory(I, SrcReg);
1766 } else {
1767 Result &= selectCopyMemorySized(I, SrcReg);
1768 }
1769 if (ResVReg.isValid() && ResVReg != I.getOperand(i: 0).getReg())
1770 Result &= BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 0).getReg(), I);
1771 return Result;
1772}
1773
1774bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
1775 const SPIRVType *ResType,
1776 MachineInstr &I,
1777 unsigned NewOpcode,
1778 unsigned NegateOpcode) const {
1779 bool Result = true;
1780 assert(I.hasOneMemOperand());
1781 const MachineMemOperand *MemOp = *I.memoperands_begin();
1782 uint32_t Scope = static_cast<uint32_t>(getMemScope(
1783 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1784 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1785 Register ScopeReg = ScopeConstant.first;
1786 Result &= ScopeConstant.second;
1787
1788 Register Ptr = I.getOperand(i: 1).getReg();
1789 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
1790 // auto ScSem =
1791 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
1792 AtomicOrdering AO = MemOp->getSuccessOrdering();
1793 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1794 auto MemSemConstant = buildI32Constant(Val: MemSem /*| ScSem*/, I);
1795 Register MemSemReg = MemSemConstant.first;
1796 Result &= MemSemConstant.second;
1797
1798 Register ValueReg = I.getOperand(i: 2).getReg();
1799 if (NegateOpcode != 0) {
1800 // Translation with negative value operand is requested
1801 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: MRI->getMF());
1802 Result &= selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {ValueReg}, Opcode: NegateOpcode);
1803 ValueReg = TmpReg;
1804 }
1805
1806 return Result &&
1807 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: NewOpcode))
1808 .addDef(RegNo: ResVReg)
1809 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1810 .addUse(RegNo: Ptr)
1811 .addUse(RegNo: ScopeReg)
1812 .addUse(RegNo: MemSemReg)
1813 .addUse(RegNo: ValueReg)
1814 .constrainAllUses(TII, TRI, RBI);
1815}
1816
1817bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
1818 unsigned ArgI = I.getNumOperands() - 1;
1819 Register SrcReg =
1820 I.getOperand(i: ArgI).isReg() ? I.getOperand(i: ArgI).getReg() : Register(0);
1821 SPIRVType *SrcType =
1822 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: SrcReg) : nullptr;
1823 if (!SrcType || SrcType->getOpcode() != SPIRV::OpTypeVector)
1824 report_fatal_error(
1825 reason: "cannot select G_UNMERGE_VALUES with a non-vector argument");
1826
1827 SPIRVType *ScalarType =
1828 GR.getSPIRVTypeForVReg(VReg: SrcType->getOperand(i: 1).getReg());
1829 MachineBasicBlock &BB = *I.getParent();
1830 bool Res = false;
1831 unsigned CurrentIndex = 0;
1832 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1833 Register ResVReg = I.getOperand(i).getReg();
1834 SPIRVType *ResType = GR.getSPIRVTypeForVReg(VReg: ResVReg);
1835 if (!ResType) {
1836 LLT ResLLT = MRI->getType(Reg: ResVReg);
1837 assert(ResLLT.isValid());
1838 if (ResLLT.isVector()) {
1839 ResType = GR.getOrCreateSPIRVVectorType(
1840 BaseType: ScalarType, NumElements: ResLLT.getNumElements(), I, TII);
1841 } else {
1842 ResType = ScalarType;
1843 }
1844 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
1845 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *GR.CurMF);
1846 }
1847
1848 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1849 Register UndefReg = GR.getOrCreateUndef(I, SpvType: SrcType, TII);
1850 auto MIB =
1851 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
1852 .addDef(RegNo: ResVReg)
1853 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1854 .addUse(RegNo: SrcReg)
1855 .addUse(RegNo: UndefReg);
1856 unsigned NumElements = GR.getScalarOrVectorComponentCount(Type: ResType);
1857 for (unsigned j = 0; j < NumElements; ++j) {
1858 MIB.addImm(Val: CurrentIndex + j);
1859 }
1860 CurrentIndex += NumElements;
1861 Res |= MIB.constrainAllUses(TII, TRI, RBI);
1862 } else {
1863 auto MIB =
1864 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1865 .addDef(RegNo: ResVReg)
1866 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1867 .addUse(RegNo: SrcReg)
1868 .addImm(Val: CurrentIndex);
1869 CurrentIndex++;
1870 Res |= MIB.constrainAllUses(TII, TRI, RBI);
1871 }
1872 }
1873 return Res;
1874}
1875
1876bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
1877 AtomicOrdering AO = AtomicOrdering(I.getOperand(i: 0).getImm());
1878 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1879 auto MemSemConstant = buildI32Constant(Val: MemSem, I);
1880 Register MemSemReg = MemSemConstant.first;
1881 bool Result = MemSemConstant.second;
1882 SyncScope::ID Ord = SyncScope::ID(I.getOperand(i: 1).getImm());
1883 uint32_t Scope = static_cast<uint32_t>(
1884 getMemScope(Ctx&: GR.CurMF->getFunction().getContext(), Id: Ord));
1885 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1886 Register ScopeReg = ScopeConstant.first;
1887 Result &= ScopeConstant.second;
1888 MachineBasicBlock &BB = *I.getParent();
1889 return Result &&
1890 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMemoryBarrier))
1891 .addUse(RegNo: ScopeReg)
1892 .addUse(RegNo: MemSemReg)
1893 .constrainAllUses(TII, TRI, RBI);
1894}
1895
1896bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
1897 const SPIRVType *ResType,
1898 MachineInstr &I,
1899 unsigned Opcode) const {
1900 Type *ResTy = nullptr;
1901 StringRef ResName;
1902 if (!GR.findValueAttrs(Key: &I, Ty&: ResTy, Name&: ResName))
1903 report_fatal_error(
1904 reason: "Not enough info to select the arithmetic with overflow instruction");
1905 if (!ResTy || !ResTy->isStructTy())
1906 report_fatal_error(reason: "Expect struct type result for the arithmetic "
1907 "with overflow instruction");
1908 // "Result Type must be from OpTypeStruct. The struct must have two members,
1909 // and the two members must be the same type."
1910 Type *ResElemTy = cast<StructType>(Val: ResTy)->getElementType(N: 0);
1911 ResTy = StructType::get(elt1: ResElemTy, elts: ResElemTy);
1912 // Build SPIR-V types and constant(s) if needed.
1913 MachineIRBuilder MIRBuilder(I);
1914 SPIRVType *StructType = GR.getOrCreateSPIRVType(
1915 Type: ResTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
1916 assert(I.getNumDefs() > 1 && "Not enought operands");
1917 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
1918 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
1919 if (N > 1)
1920 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
1921 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
1922 Register ZeroReg = buildZerosVal(ResType, I);
1923 // A new virtual register to store the result struct.
1924 Register StructVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1925 MRI->setRegClass(Reg: StructVReg, RC: &SPIRV::IDRegClass);
1926 // Build the result name if needed.
1927 if (ResName.size() > 0)
1928 buildOpName(Target: StructVReg, Name: ResName, MIRBuilder);
1929 // Build the arithmetic with overflow instruction.
1930 MachineBasicBlock &BB = *I.getParent();
1931 auto MIB =
1932 BuildMI(BB, I: MIRBuilder.getInsertPt(), MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1933 .addDef(RegNo: StructVReg)
1934 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: StructType));
1935 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
1936 MIB.addUse(RegNo: I.getOperand(i).getReg());
1937 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
1938 // Build instructions to extract fields of the instruction's result.
1939 // A new virtual register to store the higher part of the result struct.
1940 Register HigherVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1941 MRI->setRegClass(Reg: HigherVReg, RC: &SPIRV::iIDRegClass);
1942 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1943 auto MIB =
1944 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1945 .addDef(RegNo: i == 1 ? HigherVReg : I.getOperand(i).getReg())
1946 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1947 .addUse(RegNo: StructVReg)
1948 .addImm(Val: i);
1949 Result &= MIB.constrainAllUses(TII, TRI, RBI);
1950 }
1951 // Build boolean value from the higher part.
1952 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
1953 .addDef(RegNo: I.getOperand(i: 1).getReg())
1954 .addUse(RegNo: BoolTypeReg)
1955 .addUse(RegNo: HigherVReg)
1956 .addUse(RegNo: ZeroReg)
1957 .constrainAllUses(TII, TRI, RBI);
1958}
1959
1960bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
1961 const SPIRVType *ResType,
1962 MachineInstr &I) const {
1963 bool Result = true;
1964 Register ScopeReg;
1965 Register MemSemEqReg;
1966 Register MemSemNeqReg;
1967 Register Ptr = I.getOperand(i: 2).getReg();
1968 if (!isa<GIntrinsic>(Val: I)) {
1969 assert(I.hasOneMemOperand());
1970 const MachineMemOperand *MemOp = *I.memoperands_begin();
1971 unsigned Scope = static_cast<uint32_t>(getMemScope(
1972 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1973 auto ScopeConstant = buildI32Constant(Val: Scope, I);
1974 ScopeReg = ScopeConstant.first;
1975 Result &= ScopeConstant.second;
1976
1977 unsigned ScSem = static_cast<uint32_t>(
1978 getMemSemanticsForStorageClass(SC: GR.getPointerStorageClass(VReg: Ptr)));
1979 AtomicOrdering AO = MemOp->getSuccessOrdering();
1980 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(Ord: AO)) | ScSem;
1981 auto MemSemEqConstant = buildI32Constant(Val: MemSemEq, I);
1982 MemSemEqReg = MemSemEqConstant.first;
1983 Result &= MemSemEqConstant.second;
1984 AtomicOrdering FO = MemOp->getFailureOrdering();
1985 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(Ord: FO)) | ScSem;
1986 if (MemSemEq == MemSemNeq)
1987 MemSemNeqReg = MemSemEqReg;
1988 else {
1989 auto MemSemNeqConstant = buildI32Constant(Val: MemSemEq, I);
1990 MemSemNeqReg = MemSemNeqConstant.first;
1991 Result &= MemSemNeqConstant.second;
1992 }
1993 } else {
1994 ScopeReg = I.getOperand(i: 5).getReg();
1995 MemSemEqReg = I.getOperand(i: 6).getReg();
1996 MemSemNeqReg = I.getOperand(i: 7).getReg();
1997 }
1998
1999 Register Cmp = I.getOperand(i: 3).getReg();
2000 Register Val = I.getOperand(i: 4).getReg();
2001 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(VReg: Val);
2002 Register ACmpRes = createVirtualRegister(SpvType: SpvValTy, GR: &GR, MRI, MF: *I.getMF());
2003 const DebugLoc &DL = I.getDebugLoc();
2004 Result &=
2005 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpAtomicCompareExchange))
2006 .addDef(RegNo: ACmpRes)
2007 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvValTy))
2008 .addUse(RegNo: Ptr)
2009 .addUse(RegNo: ScopeReg)
2010 .addUse(RegNo: MemSemEqReg)
2011 .addUse(RegNo: MemSemNeqReg)
2012 .addUse(RegNo: Val)
2013 .addUse(RegNo: Cmp)
2014 .constrainAllUses(TII, TRI, RBI);
2015 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2016 Register CmpSuccReg = createVirtualRegister(SpvType: BoolTy, GR: &GR, MRI, MF: *I.getMF());
2017 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpIEqual))
2018 .addDef(RegNo: CmpSuccReg)
2019 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
2020 .addUse(RegNo: ACmpRes)
2021 .addUse(RegNo: Cmp)
2022 .constrainAllUses(TII, TRI, RBI);
2023 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: *I.getMF());
2024 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2025 .addDef(RegNo: TmpReg)
2026 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2027 .addUse(RegNo: ACmpRes)
2028 .addUse(RegNo: GR.getOrCreateUndef(I, SpvType: ResType, TII))
2029 .addImm(Val: 0)
2030 .constrainAllUses(TII, TRI, RBI);
2031 return Result &&
2032 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2033 .addDef(RegNo: ResVReg)
2034 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2035 .addUse(RegNo: CmpSuccReg)
2036 .addUse(RegNo: TmpReg)
2037 .addImm(Val: 1)
2038 .constrainAllUses(TII, TRI, RBI);
2039}
2040
2041static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2042 switch (SC) {
2043 case SPIRV::StorageClass::DeviceOnlyINTEL:
2044 case SPIRV::StorageClass::HostOnlyINTEL:
2045 return true;
2046 default:
2047 return false;
2048 }
2049}
2050
2051// Returns true ResVReg is referred only from global vars and OpName's.
2052static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
2053 bool IsGRef = false;
2054 bool IsAllowedRefs =
2055 llvm::all_of(Range: MRI->use_instructions(Reg: ResVReg), P: [&IsGRef](auto const &It) {
2056 unsigned Opcode = It.getOpcode();
2057 if (Opcode == SPIRV::OpConstantComposite ||
2058 Opcode == SPIRV::OpVariable ||
2059 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2060 return IsGRef = true;
2061 return Opcode == SPIRV::OpName;
2062 });
2063 return IsAllowedRefs && IsGRef;
2064}
2065
2066Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2067 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2068 return GR.getSPIRVTypeID(SpirvType: GR.getOrCreateSPIRVPointerType(
2069 BaseType: Type::getInt8Ty(C&: I.getMF()->getFunction().getContext()), I, SC));
2070}
2071
2072MachineInstrBuilder
2073SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2074 Register Src, Register DestType,
2075 uint32_t Opcode) const {
2076 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2077 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
2078 .addDef(RegNo: Dest)
2079 .addUse(RegNo: DestType)
2080 .addImm(Val: Opcode)
2081 .addUse(RegNo: Src);
2082}
2083
2084MachineInstrBuilder
2085SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2086 SPIRVType *SrcPtrTy) const {
2087 SPIRVType *GenericPtrTy =
2088 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2089 Register Tmp = MRI->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
2090 MRI->setType(VReg: Tmp, Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(
2091 SC: SPIRV::StorageClass::Generic),
2092 SizeInBits: GR.getPointerSize()));
2093 MachineFunction *MF = I.getParent()->getParent();
2094 GR.assignSPIRVTypeToVReg(Type: GenericPtrTy, VReg: Tmp, MF: *MF);
2095 MachineInstrBuilder MIB = buildSpecConstantOp(
2096 I, Dest: Tmp, Src: SrcPtr, DestType: GR.getSPIRVTypeID(SpirvType: GenericPtrTy),
2097 Opcode: static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
2098 GR.add(Obj: MIB.getInstr(), MI: MIB);
2099 return MIB;
2100}
2101
2102// In SPIR-V address space casting can only happen to and from the Generic
2103// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
2104// pointers to and from Generic pointers. As such, we can convert e.g. from
2105// Workgroup to Function by going via a Generic pointer as an intermediary. All
2106// other combinations can only be done by a bitcast, and are probably not safe.
2107bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
2108 const SPIRVType *ResType,
2109 MachineInstr &I) const {
2110 MachineBasicBlock &BB = *I.getParent();
2111 const DebugLoc &DL = I.getDebugLoc();
2112
2113 Register SrcPtr = I.getOperand(i: 1).getReg();
2114 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(VReg: SrcPtr);
2115
2116 // don't generate a cast for a null that may be represented by OpTypeInt
2117 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
2118 ResType->getOpcode() != SPIRV::OpTypePointer)
2119 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2120
2121 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(Type: SrcPtrTy);
2122 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(Type: ResType);
2123
2124 if (isASCastInGVar(MRI, ResVReg)) {
2125 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
2126 // are expressed by OpSpecConstantOp with an Opcode.
2127 // TODO: maybe insert a check whether the Kernel capability was declared and
2128 // so PtrCastToGeneric/GenericCastToPtr are available.
2129 unsigned SpecOpcode =
2130 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC)
2131 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
2132 : (SrcSC == SPIRV::StorageClass::Generic &&
2133 isGenericCastablePtr(SC: DstSC)
2134 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
2135 : 0);
2136 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
2137 // correct value of ResType and use general i8* instead. Maybe this should
2138 // be addressed in the emit-intrinsic step to infer a correct
2139 // OpConstantComposite type.
2140 if (SpecOpcode) {
2141 return buildSpecConstantOp(I, Dest: ResVReg, Src: SrcPtr,
2142 DestType: getUcharPtrTypeReg(I, SC: DstSC), Opcode: SpecOpcode)
2143 .constrainAllUses(TII, TRI, RBI);
2144 } else if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2145 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
2146 return MIB.constrainAllUses(TII, TRI, RBI) &&
2147 buildSpecConstantOp(
2148 I, Dest: ResVReg, Src: MIB->getOperand(i: 0).getReg(),
2149 DestType: getUcharPtrTypeReg(I, SC: DstSC),
2150 Opcode: static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
2151 .constrainAllUses(TII, TRI, RBI);
2152 }
2153 }
2154
2155 // don't generate a cast between identical storage classes
2156 if (SrcSC == DstSC)
2157 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2158
2159 if ((SrcSC == SPIRV::StorageClass::Function &&
2160 DstSC == SPIRV::StorageClass::Private) ||
2161 (DstSC == SPIRV::StorageClass::Function &&
2162 SrcSC == SPIRV::StorageClass::Private))
2163 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2164
2165 // Casting from an eligible pointer to Generic.
2166 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC))
2167 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2168 // Casting from Generic to an eligible pointer.
2169 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: DstSC))
2170 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2171 // Casting between 2 eligible pointers using Generic as an intermediary.
2172 if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2173 SPIRVType *GenericPtrTy =
2174 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2175 Register Tmp = createVirtualRegister(SpvType: GenericPtrTy, GR: &GR, MRI, MF: MRI->getMF());
2176 bool Result = BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpPtrCastToGeneric))
2177 .addDef(RegNo: Tmp)
2178 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: GenericPtrTy))
2179 .addUse(RegNo: SrcPtr)
2180 .constrainAllUses(TII, TRI, RBI);
2181 return Result && BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtr))
2182 .addDef(RegNo: ResVReg)
2183 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2184 .addUse(RegNo: Tmp)
2185 .constrainAllUses(TII, TRI, RBI);
2186 }
2187
2188 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
2189 // be applied
2190 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
2191 return selectUnOp(ResVReg, ResType, I,
2192 Opcode: SPIRV::OpPtrCastToCrossWorkgroupINTEL);
2193 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(SC: DstSC))
2194 return selectUnOp(ResVReg, ResType, I,
2195 Opcode: SPIRV::OpCrossWorkgroupCastToPtrINTEL);
2196 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::Generic)
2197 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2198 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(SC: DstSC))
2199 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2200
2201 // Bitcast for pointers requires that the address spaces must match
2202 return false;
2203}
2204
2205static unsigned getFCmpOpcode(unsigned PredNum) {
2206 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2207 switch (Pred) {
2208 case CmpInst::FCMP_OEQ:
2209 return SPIRV::OpFOrdEqual;
2210 case CmpInst::FCMP_OGE:
2211 return SPIRV::OpFOrdGreaterThanEqual;
2212 case CmpInst::FCMP_OGT:
2213 return SPIRV::OpFOrdGreaterThan;
2214 case CmpInst::FCMP_OLE:
2215 return SPIRV::OpFOrdLessThanEqual;
2216 case CmpInst::FCMP_OLT:
2217 return SPIRV::OpFOrdLessThan;
2218 case CmpInst::FCMP_ONE:
2219 return SPIRV::OpFOrdNotEqual;
2220 case CmpInst::FCMP_ORD:
2221 return SPIRV::OpOrdered;
2222 case CmpInst::FCMP_UEQ:
2223 return SPIRV::OpFUnordEqual;
2224 case CmpInst::FCMP_UGE:
2225 return SPIRV::OpFUnordGreaterThanEqual;
2226 case CmpInst::FCMP_UGT:
2227 return SPIRV::OpFUnordGreaterThan;
2228 case CmpInst::FCMP_ULE:
2229 return SPIRV::OpFUnordLessThanEqual;
2230 case CmpInst::FCMP_ULT:
2231 return SPIRV::OpFUnordLessThan;
2232 case CmpInst::FCMP_UNE:
2233 return SPIRV::OpFUnordNotEqual;
2234 case CmpInst::FCMP_UNO:
2235 return SPIRV::OpUnordered;
2236 default:
2237 llvm_unreachable("Unknown predicate type for FCmp");
2238 }
2239}
2240
2241static unsigned getICmpOpcode(unsigned PredNum) {
2242 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2243 switch (Pred) {
2244 case CmpInst::ICMP_EQ:
2245 return SPIRV::OpIEqual;
2246 case CmpInst::ICMP_NE:
2247 return SPIRV::OpINotEqual;
2248 case CmpInst::ICMP_SGE:
2249 return SPIRV::OpSGreaterThanEqual;
2250 case CmpInst::ICMP_SGT:
2251 return SPIRV::OpSGreaterThan;
2252 case CmpInst::ICMP_SLE:
2253 return SPIRV::OpSLessThanEqual;
2254 case CmpInst::ICMP_SLT:
2255 return SPIRV::OpSLessThan;
2256 case CmpInst::ICMP_UGE:
2257 return SPIRV::OpUGreaterThanEqual;
2258 case CmpInst::ICMP_UGT:
2259 return SPIRV::OpUGreaterThan;
2260 case CmpInst::ICMP_ULE:
2261 return SPIRV::OpULessThanEqual;
2262 case CmpInst::ICMP_ULT:
2263 return SPIRV::OpULessThan;
2264 default:
2265 llvm_unreachable("Unknown predicate type for ICmp");
2266 }
2267}
2268
2269static unsigned getPtrCmpOpcode(unsigned Pred) {
2270 switch (static_cast<CmpInst::Predicate>(Pred)) {
2271 case CmpInst::ICMP_EQ:
2272 return SPIRV::OpPtrEqual;
2273 case CmpInst::ICMP_NE:
2274 return SPIRV::OpPtrNotEqual;
2275 default:
2276 llvm_unreachable("Unknown predicate type for pointer comparison");
2277 }
2278}
2279
2280// Return the logical operation, or abort if none exists.
2281static unsigned getBoolCmpOpcode(unsigned PredNum) {
2282 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2283 switch (Pred) {
2284 case CmpInst::ICMP_EQ:
2285 return SPIRV::OpLogicalEqual;
2286 case CmpInst::ICMP_NE:
2287 return SPIRV::OpLogicalNotEqual;
2288 default:
2289 llvm_unreachable("Unknown predicate type for Bool comparison");
2290 }
2291}
2292
2293static APFloat getZeroFP(const Type *LLVMFloatTy) {
2294 if (!LLVMFloatTy)
2295 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2296 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2297 case Type::HalfTyID:
2298 return APFloat::getZero(Sem: APFloat::IEEEhalf());
2299 default:
2300 case Type::FloatTyID:
2301 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2302 case Type::DoubleTyID:
2303 return APFloat::getZero(Sem: APFloat::IEEEdouble());
2304 }
2305}
2306
2307static APFloat getOneFP(const Type *LLVMFloatTy) {
2308 if (!LLVMFloatTy)
2309 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2310 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2311 case Type::HalfTyID:
2312 return APFloat::getOne(Sem: APFloat::IEEEhalf());
2313 default:
2314 case Type::FloatTyID:
2315 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2316 case Type::DoubleTyID:
2317 return APFloat::getOne(Sem: APFloat::IEEEdouble());
2318 }
2319}
2320
2321bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
2322 const SPIRVType *ResType,
2323 MachineInstr &I,
2324 unsigned OpAnyOrAll) const {
2325 assert(I.getNumOperands() == 3);
2326 assert(I.getOperand(2).isReg());
2327 MachineBasicBlock &BB = *I.getParent();
2328 Register InputRegister = I.getOperand(i: 2).getReg();
2329 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2330
2331 if (!InputType)
2332 report_fatal_error(reason: "Input Type could not be determined.");
2333
2334 bool IsBoolTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeBool);
2335 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
2336 if (IsBoolTy && !IsVectorTy) {
2337 assert(ResVReg == I.getOperand(0).getReg());
2338 return BuildCOPY(DestReg: ResVReg, SrcReg: InputRegister, I);
2339 }
2340
2341 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2342 unsigned SpirvNotEqualId =
2343 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
2344 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
2345 SPIRVType *SpvBoolTy = SpvBoolScalarTy;
2346 Register NotEqualReg = ResVReg;
2347
2348 if (IsVectorTy) {
2349 NotEqualReg =
2350 IsBoolTy ? InputRegister
2351 : createVirtualRegister(SpvType: SpvBoolTy, GR: &GR, MRI, MF: MRI->getMF());
2352 const unsigned NumElts = InputType->getOperand(i: 2).getImm();
2353 SpvBoolTy = GR.getOrCreateSPIRVVectorType(BaseType: SpvBoolTy, NumElements: NumElts, I, TII);
2354 }
2355
2356 bool Result = true;
2357 if (!IsBoolTy) {
2358 Register ConstZeroReg =
2359 IsFloatTy ? buildZerosValF(ResType: InputType, I) : buildZerosVal(ResType: InputType, I);
2360
2361 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SpirvNotEqualId))
2362 .addDef(RegNo: NotEqualReg)
2363 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolTy))
2364 .addUse(RegNo: InputRegister)
2365 .addUse(RegNo: ConstZeroReg)
2366 .constrainAllUses(TII, TRI, RBI);
2367 }
2368
2369 if (!IsVectorTy)
2370 return Result;
2371
2372 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpAnyOrAll))
2373 .addDef(RegNo: ResVReg)
2374 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolScalarTy))
2375 .addUse(RegNo: NotEqualReg)
2376 .constrainAllUses(TII, TRI, RBI);
2377}
2378
2379bool SPIRVInstructionSelector::selectAll(Register ResVReg,
2380 const SPIRVType *ResType,
2381 MachineInstr &I) const {
2382 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAll);
2383}
2384
2385bool SPIRVInstructionSelector::selectAny(Register ResVReg,
2386 const SPIRVType *ResType,
2387 MachineInstr &I) const {
2388 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAny);
2389}
2390
2391// Select the OpDot instruction for the given float dot
2392bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
2393 const SPIRVType *ResType,
2394 MachineInstr &I) const {
2395 assert(I.getNumOperands() == 4);
2396 assert(I.getOperand(2).isReg());
2397 assert(I.getOperand(3).isReg());
2398
2399 [[maybe_unused]] SPIRVType *VecType =
2400 GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 2).getReg());
2401
2402 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2403 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2404 "dot product requires a vector of at least 2 components");
2405
2406 [[maybe_unused]] SPIRVType *EltType =
2407 GR.getSPIRVTypeForVReg(VReg: VecType->getOperand(i: 1).getReg());
2408
2409 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
2410
2411 MachineBasicBlock &BB = *I.getParent();
2412 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDot))
2413 .addDef(RegNo: ResVReg)
2414 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2415 .addUse(RegNo: I.getOperand(i: 2).getReg())
2416 .addUse(RegNo: I.getOperand(i: 3).getReg())
2417 .constrainAllUses(TII, TRI, RBI);
2418}
2419
2420bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
2421 const SPIRVType *ResType,
2422 MachineInstr &I,
2423 bool Signed) const {
2424 assert(I.getNumOperands() == 4);
2425 assert(I.getOperand(2).isReg());
2426 assert(I.getOperand(3).isReg());
2427 MachineBasicBlock &BB = *I.getParent();
2428
2429 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2430 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2431 .addDef(RegNo: ResVReg)
2432 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2433 .addUse(RegNo: I.getOperand(i: 2).getReg())
2434 .addUse(RegNo: I.getOperand(i: 3).getReg())
2435 .constrainAllUses(TII, TRI, RBI);
2436}
2437
2438// Since pre-1.6 SPIRV has no integer dot implementation,
2439// expand by piecewise multiplying and adding the results
2440bool SPIRVInstructionSelector::selectIntegerDotExpansion(
2441 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2442 assert(I.getNumOperands() == 4);
2443 assert(I.getOperand(2).isReg());
2444 assert(I.getOperand(3).isReg());
2445 MachineBasicBlock &BB = *I.getParent();
2446
2447 // Multiply the vectors, then sum the results
2448 Register Vec0 = I.getOperand(i: 2).getReg();
2449 Register Vec1 = I.getOperand(i: 3).getReg();
2450 Register TmpVec = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2451 SPIRVType *VecType = GR.getSPIRVTypeForVReg(VReg: Vec0);
2452
2453 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulV))
2454 .addDef(RegNo: TmpVec)
2455 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VecType))
2456 .addUse(RegNo: Vec0)
2457 .addUse(RegNo: Vec1)
2458 .constrainAllUses(TII, TRI, RBI);
2459
2460 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2461 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2462 "dot product requires a vector of at least 2 components");
2463
2464 Register Res = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2465 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2466 .addDef(RegNo: Res)
2467 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2468 .addUse(RegNo: TmpVec)
2469 .addImm(Val: 0)
2470 .constrainAllUses(TII, TRI, RBI);
2471
2472 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(Type: VecType); i++) {
2473 Register Elt = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2474
2475 Result &=
2476 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2477 .addDef(RegNo: Elt)
2478 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2479 .addUse(RegNo: TmpVec)
2480 .addImm(Val: i)
2481 .constrainAllUses(TII, TRI, RBI);
2482
2483 Register Sum = i < GR.getScalarOrVectorComponentCount(Type: VecType) - 1
2484 ? MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType))
2485 : ResVReg;
2486
2487 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2488 .addDef(RegNo: Sum)
2489 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2490 .addUse(RegNo: Res)
2491 .addUse(RegNo: Elt)
2492 .constrainAllUses(TII, TRI, RBI);
2493 Res = Sum;
2494 }
2495
2496 return Result;
2497}
2498
2499bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
2500 const SPIRVType *ResType,
2501 MachineInstr &I) const {
2502 MachineBasicBlock &BB = *I.getParent();
2503 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsInf))
2504 .addDef(RegNo: ResVReg)
2505 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2506 .addUse(RegNo: I.getOperand(i: 2).getReg())
2507 .constrainAllUses(TII, TRI, RBI);
2508}
2509
2510bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
2511 const SPIRVType *ResType,
2512 MachineInstr &I) const {
2513 MachineBasicBlock &BB = *I.getParent();
2514 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsNan))
2515 .addDef(RegNo: ResVReg)
2516 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2517 .addUse(RegNo: I.getOperand(i: 2).getReg())
2518 .constrainAllUses(TII, TRI, RBI);
2519}
2520
2521template <bool Signed>
2522bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
2523 const SPIRVType *ResType,
2524 MachineInstr &I) const {
2525 assert(I.getNumOperands() == 5);
2526 assert(I.getOperand(2).isReg());
2527 assert(I.getOperand(3).isReg());
2528 assert(I.getOperand(4).isReg());
2529 MachineBasicBlock &BB = *I.getParent();
2530
2531 Register Acc = I.getOperand(i: 2).getReg();
2532 Register X = I.getOperand(i: 3).getReg();
2533 Register Y = I.getOperand(i: 4).getReg();
2534
2535 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2536 Register Dot = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2537 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2538 .addDef(RegNo: Dot)
2539 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2540 .addUse(RegNo: X)
2541 .addUse(RegNo: Y)
2542 .constrainAllUses(TII, TRI, RBI);
2543
2544 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2545 .addDef(RegNo: ResVReg)
2546 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2547 .addUse(RegNo: Dot)
2548 .addUse(RegNo: Acc)
2549 .constrainAllUses(TII, TRI, RBI);
2550}
2551
2552// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
2553// extract the elements of the packed inputs, multiply them and add the result
2554// to the accumulator.
2555template <bool Signed>
2556bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
2557 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2558 assert(I.getNumOperands() == 5);
2559 assert(I.getOperand(2).isReg());
2560 assert(I.getOperand(3).isReg());
2561 assert(I.getOperand(4).isReg());
2562 MachineBasicBlock &BB = *I.getParent();
2563
2564 bool Result = true;
2565
2566 Register Acc = I.getOperand(i: 2).getReg();
2567 Register X = I.getOperand(i: 3).getReg();
2568 Register Y = I.getOperand(i: 4).getReg();
2569
2570 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII);
2571 auto ExtractOp =
2572 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
2573
2574 bool ZeroAsNull = !STI.isShader();
2575 // Extract the i8 element, multiply and add it to the accumulator
2576 for (unsigned i = 0; i < 4; i++) {
2577 // A[i]
2578 Register AElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2579 Result &=
2580 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2581 .addDef(RegNo: AElt)
2582 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2583 .addUse(RegNo: X)
2584 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2585 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2586 .constrainAllUses(TII, TRI, RBI);
2587
2588 // B[i]
2589 Register BElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2590 Result &=
2591 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2592 .addDef(RegNo: BElt)
2593 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2594 .addUse(RegNo: Y)
2595 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2596 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2597 .constrainAllUses(TII, TRI, RBI);
2598
2599 // A[i] * B[i]
2600 Register Mul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2601 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulS))
2602 .addDef(RegNo: Mul)
2603 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2604 .addUse(RegNo: AElt)
2605 .addUse(RegNo: BElt)
2606 .constrainAllUses(TII, TRI, RBI);
2607
2608 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
2609 Register MaskMul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2610 Result &=
2611 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2612 .addDef(RegNo: MaskMul)
2613 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2614 .addUse(RegNo: Mul)
2615 .addUse(RegNo: GR.getOrCreateConstInt(Val: 0, I, SpvType: EltType, TII, ZeroAsNull))
2616 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2617 .constrainAllUses(TII, TRI, RBI);
2618
2619 // Acc = Acc + A[i] * B[i]
2620 Register Sum =
2621 i < 3 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass) : ResVReg;
2622 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2623 .addDef(RegNo: Sum)
2624 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2625 .addUse(RegNo: Acc)
2626 .addUse(RegNo: MaskMul)
2627 .constrainAllUses(TII, TRI, RBI);
2628
2629 Acc = Sum;
2630 }
2631
2632 return Result;
2633}
2634
2635/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
2636/// does not have a saturate builtin.
2637bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
2638 const SPIRVType *ResType,
2639 MachineInstr &I) const {
2640 assert(I.getNumOperands() == 3);
2641 assert(I.getOperand(2).isReg());
2642 MachineBasicBlock &BB = *I.getParent();
2643 Register VZero = buildZerosValF(ResType, I);
2644 Register VOne = buildOnesValF(ResType, I);
2645
2646 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
2647 .addDef(RegNo: ResVReg)
2648 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2649 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2650 .addImm(Val: GL::FClamp)
2651 .addUse(RegNo: I.getOperand(i: 2).getReg())
2652 .addUse(RegNo: VZero)
2653 .addUse(RegNo: VOne)
2654 .constrainAllUses(TII, TRI, RBI);
2655}
2656
2657bool SPIRVInstructionSelector::selectSign(Register ResVReg,
2658 const SPIRVType *ResType,
2659 MachineInstr &I) const {
2660 assert(I.getNumOperands() == 3);
2661 assert(I.getOperand(2).isReg());
2662 MachineBasicBlock &BB = *I.getParent();
2663 Register InputRegister = I.getOperand(i: 2).getReg();
2664 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2665 auto &DL = I.getDebugLoc();
2666
2667 if (!InputType)
2668 report_fatal_error(reason: "Input Type could not be determined.");
2669
2670 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2671
2672 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(Type: InputType);
2673 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2674
2675 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
2676
2677 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
2678 Register SignReg = NeedsConversion
2679 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass)
2680 : ResVReg;
2681
2682 bool Result =
2683 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpExtInst))
2684 .addDef(RegNo: SignReg)
2685 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: InputType))
2686 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2687 .addImm(Val: SignOpcode)
2688 .addUse(RegNo: InputRegister)
2689 .constrainAllUses(TII, TRI, RBI);
2690
2691 if (NeedsConversion) {
2692 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
2693 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: ConvertOpcode))
2694 .addDef(RegNo: ResVReg)
2695 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2696 .addUse(RegNo: SignReg)
2697 .constrainAllUses(TII, TRI, RBI);
2698 }
2699
2700 return Result;
2701}
2702
2703bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
2704 const SPIRVType *ResType,
2705 MachineInstr &I,
2706 unsigned Opcode) const {
2707 MachineBasicBlock &BB = *I.getParent();
2708 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2709
2710 auto BMI = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2711 .addDef(RegNo: ResVReg)
2712 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2713 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I,
2714 SpvType: IntTy, TII, ZeroAsNull: !STI.isShader()));
2715
2716 for (unsigned J = 2; J < I.getNumOperands(); J++) {
2717 BMI.addUse(RegNo: I.getOperand(i: J).getReg());
2718 }
2719
2720 return BMI.constrainAllUses(TII, TRI, RBI);
2721}
2722
2723bool SPIRVInstructionSelector::selectWaveActiveCountBits(
2724 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2725
2726 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2727 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(BaseType: IntTy, NumElements: 4, I, TII);
2728 Register BallotReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BallotType));
2729 bool Result = selectWaveOpInst(ResVReg: BallotReg, ResType: BallotType, I,
2730 Opcode: SPIRV::OpGroupNonUniformBallot);
2731
2732 MachineBasicBlock &BB = *I.getParent();
2733 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(),
2734 MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2735 .addDef(RegNo: ResVReg)
2736 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2737 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy,
2738 TII, ZeroAsNull: !STI.isShader()))
2739 .addImm(Val: SPIRV::GroupOperation::Reduce)
2740 .addUse(RegNo: BallotReg)
2741 .constrainAllUses(TII, TRI, RBI);
2742
2743 return Result;
2744}
2745
2746bool SPIRVInstructionSelector::selectWavePrefixBitCount(
2747 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2748
2749 assert(I.getNumOperands() == 3);
2750
2751 auto Op = I.getOperand(i: 2);
2752 assert(Op.isReg());
2753
2754 MachineBasicBlock &BB = *I.getParent();
2755 DebugLoc DL = I.getDebugLoc();
2756
2757 Register InputRegister = Op.getReg();
2758 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2759
2760 if (!InputType)
2761 report_fatal_error(reason: "Input Type could not be determined.");
2762
2763 if (InputType->getOpcode() != SPIRV::OpTypeBool)
2764 report_fatal_error(reason: "WavePrefixBitCount requires boolean input");
2765
2766 // Types
2767 SPIRVType *Int32Ty = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2768
2769 // Ballot result type: vector<uint32>
2770 // Match DXC: %v4uint for Subgroup size
2771 SPIRVType *BallotTy = GR.getOrCreateSPIRVVectorType(BaseType: Int32Ty, NumElements: 4, I, TII);
2772
2773 // Create a vreg for the ballot result
2774 Register BallotVReg = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2775
2776 // 1. OpGroupNonUniformBallot
2777 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallot))
2778 .addDef(RegNo: BallotVReg)
2779 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BallotTy))
2780 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
2781 .addUse(RegNo: InputRegister)
2782 .constrainAllUses(TII, TRI, RBI);
2783
2784 // 2. OpGroupNonUniformBallotBitCount
2785 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2786 .addDef(RegNo: ResVReg)
2787 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2788 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
2789 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
2790 .addUse(RegNo: BallotVReg)
2791 .constrainAllUses(TII, TRI, RBI);
2792
2793 return true;
2794}
2795
2796bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
2797 const SPIRVType *ResType,
2798 MachineInstr &I,
2799 bool IsUnsigned) const {
2800 return selectWaveReduce(
2801 ResVReg, ResType, I, IsUnsigned,
2802 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2803 const bool IsFloatTy =
2804 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2805 const unsigned IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
2806 : SPIRV::OpGroupNonUniformSMax;
2807 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
2808 });
2809}
2810
2811bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
2812 const SPIRVType *ResType,
2813 MachineInstr &I,
2814 bool IsUnsigned) const {
2815 return selectWaveReduce(
2816 ResVReg, ResType, I, IsUnsigned,
2817 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2818 const bool IsFloatTy =
2819 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2820 const unsigned IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
2821 : SPIRV::OpGroupNonUniformSMin;
2822 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
2823 });
2824}
2825
2826bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
2827 const SPIRVType *ResType,
2828 MachineInstr &I) const {
2829 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
2830 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2831 bool IsFloatTy = GR.isScalarOrVectorOfType(
2832 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2833 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
2834 : SPIRV::OpGroupNonUniformIAdd;
2835 });
2836}
2837
2838template <typename PickOpcodeFn>
2839bool SPIRVInstructionSelector::selectWaveReduce(
2840 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
2841 bool IsUnsigned, PickOpcodeFn &&PickOpcode) const {
2842 assert(I.getNumOperands() == 3);
2843 assert(I.getOperand(2).isReg());
2844 MachineBasicBlock &BB = *I.getParent();
2845 Register InputRegister = I.getOperand(i: 2).getReg();
2846 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2847
2848 if (!InputType)
2849 report_fatal_error(reason: "Input Type could not be determined.");
2850
2851 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2852 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
2853 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2854 .addDef(RegNo: ResVReg)
2855 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2856 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2857 ZeroAsNull: !STI.isShader()))
2858 .addImm(Val: SPIRV::GroupOperation::Reduce)
2859 .addUse(RegNo: I.getOperand(i: 2).getReg())
2860 .constrainAllUses(TII, TRI, RBI);
2861}
2862
2863bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
2864 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2865 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
2866 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2867 bool IsFloatTy = GR.isScalarOrVectorOfType(
2868 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2869 return IsFloatTy
2870 ? SPIRV::OpGroupNonUniformFAdd
2871 : SPIRV::OpGroupNonUniformIAdd;
2872 });
2873}
2874
2875template <typename PickOpcodeFn>
2876bool SPIRVInstructionSelector::selectWaveExclusiveScan(
2877 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
2878 bool IsUnsigned, PickOpcodeFn &&PickOpcode) const {
2879 assert(I.getNumOperands() == 3);
2880 assert(I.getOperand(2).isReg());
2881 MachineBasicBlock &BB = *I.getParent();
2882 Register InputRegister = I.getOperand(i: 2).getReg();
2883 SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2884
2885 if (!InputType)
2886 report_fatal_error(reason: "Input Type could not be determined.");
2887
2888 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2889 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
2890 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2891 .addDef(RegNo: ResVReg)
2892 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2893 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2894 ZeroAsNull: !STI.isShader()))
2895 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
2896 .addUse(RegNo: I.getOperand(i: 2).getReg())
2897 .constrainAllUses(TII, TRI, RBI);
2898}
2899
2900bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
2901 const SPIRVType *ResType,
2902 MachineInstr &I) const {
2903 MachineBasicBlock &BB = *I.getParent();
2904 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitReverse))
2905 .addDef(RegNo: ResVReg)
2906 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2907 .addUse(RegNo: I.getOperand(i: 1).getReg())
2908 .constrainAllUses(TII, TRI, RBI);
2909}
2910
2911bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
2912 const SPIRVType *ResType,
2913 MachineInstr &I) const {
2914 // There is no way to implement `freeze` correctly without support on SPIR-V
2915 // standard side, but we may at least address a simple (static) case when
2916 // undef/poison value presence is obvious. The main benefit of even
2917 // incomplete `freeze` support is preventing of translation from crashing due
2918 // to lack of support on legalization and instruction selection steps.
2919 if (!I.getOperand(i: 0).isReg() || !I.getOperand(i: 1).isReg())
2920 return false;
2921 Register OpReg = I.getOperand(i: 1).getReg();
2922 if (MachineInstr *Def = MRI->getVRegDef(Reg: OpReg)) {
2923 if (Def->getOpcode() == TargetOpcode::COPY)
2924 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
2925 Register Reg;
2926 switch (Def->getOpcode()) {
2927 case SPIRV::ASSIGN_TYPE:
2928 if (MachineInstr *AssignToDef =
2929 MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())) {
2930 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
2931 Reg = Def->getOperand(i: 2).getReg();
2932 }
2933 break;
2934 case SPIRV::OpUndef:
2935 Reg = Def->getOperand(i: 1).getReg();
2936 break;
2937 }
2938 unsigned DestOpCode;
2939 if (Reg.isValid()) {
2940 DestOpCode = SPIRV::OpConstantNull;
2941 } else {
2942 DestOpCode = TargetOpcode::COPY;
2943 Reg = OpReg;
2944 }
2945 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DestOpCode))
2946 .addDef(RegNo: I.getOperand(i: 0).getReg())
2947 .addUse(RegNo: Reg)
2948 .constrainAllUses(TII, TRI, RBI);
2949 }
2950 return false;
2951}
2952
2953bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
2954 const SPIRVType *ResType,
2955 MachineInstr &I) const {
2956 unsigned N = 0;
2957 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2958 N = GR.getScalarOrVectorComponentCount(Type: ResType);
2959 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2960 N = getArrayComponentCount(MRI, ResType);
2961 else
2962 report_fatal_error(reason: "Cannot select G_BUILD_VECTOR with a non-vector result");
2963 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
2964 report_fatal_error(reason: "G_BUILD_VECTOR and the result type are inconsistent");
2965
2966 // check if we may construct a constant vector
2967 bool IsConst = true;
2968 for (unsigned i = I.getNumExplicitDefs();
2969 i < I.getNumExplicitOperands() && IsConst; ++i)
2970 if (!isConstReg(MRI, OpReg: I.getOperand(i).getReg()))
2971 IsConst = false;
2972
2973 if (!IsConst && N < 2)
2974 report_fatal_error(
2975 reason: "There must be at least two constituent operands in a vector");
2976
2977 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
2978 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2979 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
2980 : SPIRV::OpCompositeConstruct))
2981 .addDef(RegNo: ResVReg)
2982 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2983 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
2984 MIB.addUse(RegNo: I.getOperand(i).getReg());
2985 return MIB.constrainAllUses(TII, TRI, RBI);
2986}
2987
2988bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
2989 const SPIRVType *ResType,
2990 MachineInstr &I) const {
2991 unsigned N = 0;
2992 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2993 N = GR.getScalarOrVectorComponentCount(Type: ResType);
2994 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2995 N = getArrayComponentCount(MRI, ResType);
2996 else
2997 report_fatal_error(reason: "Cannot select G_SPLAT_VECTOR with a non-vector result");
2998
2999 unsigned OpIdx = I.getNumExplicitDefs();
3000 if (!I.getOperand(i: OpIdx).isReg())
3001 report_fatal_error(reason: "Unexpected argument in G_SPLAT_VECTOR");
3002
3003 // check if we may construct a constant vector
3004 Register OpReg = I.getOperand(i: OpIdx).getReg();
3005 bool IsConst = isConstReg(MRI, OpReg);
3006
3007 if (!IsConst && N < 2)
3008 report_fatal_error(
3009 reason: "There must be at least two constituent operands in a vector");
3010
3011 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3012 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3013 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
3014 : SPIRV::OpCompositeConstruct))
3015 .addDef(RegNo: ResVReg)
3016 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3017 for (unsigned i = 0; i < N; ++i)
3018 MIB.addUse(RegNo: OpReg);
3019 return MIB.constrainAllUses(TII, TRI, RBI);
3020}
3021
3022bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
3023 const SPIRVType *ResType,
3024 MachineInstr &I) const {
3025
3026 unsigned Opcode;
3027
3028 if (STI.canUseExtension(
3029 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
3030 STI.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6))) {
3031 Opcode = SPIRV::OpDemoteToHelperInvocation;
3032 } else {
3033 Opcode = SPIRV::OpKill;
3034 // OpKill must be the last operation of any basic block.
3035 if (MachineInstr *NextI = I.getNextNode()) {
3036 GR.invalidateMachineInstr(MI: NextI);
3037 NextI->removeFromParent();
3038 }
3039 }
3040
3041 MachineBasicBlock &BB = *I.getParent();
3042 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3043 .constrainAllUses(TII, TRI, RBI);
3044}
3045
3046bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
3047 const SPIRVType *ResType,
3048 unsigned CmpOpc,
3049 MachineInstr &I) const {
3050 Register Cmp0 = I.getOperand(i: 2).getReg();
3051 Register Cmp1 = I.getOperand(i: 3).getReg();
3052 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
3053 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
3054 "CMP operands should have the same type");
3055 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CmpOpc))
3056 .addDef(RegNo: ResVReg)
3057 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3058 .addUse(RegNo: Cmp0)
3059 .addUse(RegNo: Cmp1)
3060 .setMIFlags(I.getFlags())
3061 .constrainAllUses(TII, TRI, RBI);
3062}
3063
3064bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
3065 const SPIRVType *ResType,
3066 MachineInstr &I) const {
3067 auto Pred = I.getOperand(i: 1).getPredicate();
3068 unsigned CmpOpc;
3069
3070 Register CmpOperand = I.getOperand(i: 2).getReg();
3071 if (GR.isScalarOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypePointer))
3072 CmpOpc = getPtrCmpOpcode(Pred);
3073 else if (GR.isScalarOrVectorOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypeBool))
3074 CmpOpc = getBoolCmpOpcode(PredNum: Pred);
3075 else
3076 CmpOpc = getICmpOpcode(PredNum: Pred);
3077 return selectCmp(ResVReg, ResType, CmpOpc, I);
3078}
3079
3080std::pair<Register, bool>
3081SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
3082 const SPIRVType *ResType) const {
3083 Type *LLVMTy = IntegerType::get(C&: GR.CurMF->getFunction().getContext(), NumBits: 32);
3084 const SPIRVType *SpvI32Ty =
3085 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3086 // Find a constant in DT or build a new one.
3087 auto ConstInt = ConstantInt::get(Ty: LLVMTy, V: Val);
3088 Register NewReg = GR.find(V: ConstInt, MF: GR.CurMF);
3089 bool Result = true;
3090 if (!NewReg.isValid()) {
3091 NewReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
3092 MachineBasicBlock &BB = *I.getParent();
3093 MachineInstr *MI =
3094 Val == 0
3095 ? BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3096 .addDef(RegNo: NewReg)
3097 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3098 : BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantI))
3099 .addDef(RegNo: NewReg)
3100 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3101 .addImm(Val: APInt(32, Val).getZExtValue());
3102 Result &= constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
3103 GR.add(V: ConstInt, MI);
3104 }
3105 return {NewReg, Result};
3106}
3107
3108bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
3109 const SPIRVType *ResType,
3110 MachineInstr &I) const {
3111 unsigned CmpOp = getFCmpOpcode(PredNum: I.getOperand(i: 1).getPredicate());
3112 return selectCmp(ResVReg, ResType, CmpOpc: CmpOp, I);
3113}
3114
3115Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
3116 MachineInstr &I) const {
3117 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3118 bool ZeroAsNull = !STI.isShader();
3119 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3120 return GR.getOrCreateConstVector(Val: 0UL, I, SpvType: ResType, TII, ZeroAsNull);
3121 return GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
3122}
3123
3124bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
3125 Register Reg) const {
3126 SPIRVType *Type = GR.getSPIRVTypeForVReg(VReg: Reg);
3127 if (!Type)
3128 return false;
3129 SPIRVType *CompType = GR.getScalarOrVectorComponentType(Type);
3130 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
3131 return false;
3132
3133 auto IsZero = [this](Register Reg) {
3134 MachineInstr *Def = getDefInstrMaybeConstant(ConstReg&: Reg, MRI);
3135 if (!Def)
3136 return false;
3137
3138 if (Def->getOpcode() == SPIRV::OpConstantNull)
3139 return true;
3140
3141 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
3142 Def->getOpcode() == SPIRV::OpConstantI)
3143 return getIConstVal(ConstReg: Reg, MRI) == 0;
3144
3145 return false;
3146 };
3147
3148 if (IsZero(Reg))
3149 return true;
3150
3151 MachineInstr *Def = MRI->getVRegDef(Reg);
3152 if (!Def)
3153 return false;
3154
3155 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
3156 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
3157 cast<GIntrinsic>(Val: Def)->getIntrinsicID() ==
3158 Intrinsic::spv_const_composite)) {
3159 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
3160 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
3161 if (!IsZero(Def->getOperand(i).getReg()))
3162 return false;
3163 }
3164 return true;
3165 }
3166
3167 return false;
3168}
3169
3170Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
3171 MachineInstr &I) const {
3172 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3173 bool ZeroAsNull = !STI.isShader();
3174 APFloat VZero = getZeroFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3175 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3176 return GR.getOrCreateConstVector(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3177 return GR.getOrCreateConstFP(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3178}
3179
3180Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType,
3181 MachineInstr &I) const {
3182 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3183 bool ZeroAsNull = !STI.isShader();
3184 APFloat VOne = getOneFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3185 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3186 return GR.getOrCreateConstVector(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3187 return GR.getOrCreateConstFP(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3188}
3189
3190Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
3191 const SPIRVType *ResType,
3192 MachineInstr &I) const {
3193 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3194 APInt One =
3195 AllOnes ? APInt::getAllOnes(numBits: BitWidth) : APInt::getOneBitSet(numBits: BitWidth, BitNo: 0);
3196 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3197 return GR.getOrCreateConstVector(Val: One.getZExtValue(), I, SpvType: ResType, TII);
3198 return GR.getOrCreateConstInt(Val: One.getZExtValue(), I, SpvType: ResType, TII);
3199}
3200
3201bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
3202 const SPIRVType *ResType,
3203 MachineInstr &I) const {
3204 Register SelectFirstArg = I.getOperand(i: 2).getReg();
3205 Register SelectSecondArg = I.getOperand(i: 3).getReg();
3206 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
3207 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
3208
3209 bool IsFloatTy =
3210 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypeFloat);
3211 bool IsPtrTy =
3212 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypePointer);
3213 bool IsVectorTy = GR.getSPIRVTypeForVReg(VReg: SelectFirstArg)->getOpcode() ==
3214 SPIRV::OpTypeVector;
3215
3216 bool IsScalarBool =
3217 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
3218 unsigned Opcode;
3219 if (IsVectorTy) {
3220 if (IsFloatTy) {
3221 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
3222 } else if (IsPtrTy) {
3223 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
3224 } else {
3225 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
3226 }
3227 } else {
3228 if (IsFloatTy) {
3229 Opcode = IsScalarBool ? SPIRV::OpSelectSFSCond : SPIRV::OpSelectVFVCond;
3230 } else if (IsPtrTy) {
3231 Opcode = IsScalarBool ? SPIRV::OpSelectSPSCond : SPIRV::OpSelectVPVCond;
3232 } else {
3233 Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3234 }
3235 }
3236 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3237 .addDef(RegNo: ResVReg)
3238 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3239 .addUse(RegNo: I.getOperand(i: 1).getReg())
3240 .addUse(RegNo: SelectFirstArg)
3241 .addUse(RegNo: SelectSecondArg)
3242 .constrainAllUses(TII, TRI, RBI);
3243}
3244
3245bool SPIRVInstructionSelector::selectSelectDefaultArgs(Register ResVReg,
3246 const SPIRVType *ResType,
3247 MachineInstr &I,
3248 bool IsSigned) const {
3249 // To extend a bool, we need to use OpSelect between constants.
3250 Register ZeroReg = buildZerosVal(ResType, I);
3251 Register OneReg = buildOnesVal(AllOnes: IsSigned, ResType, I);
3252 bool IsScalarBool =
3253 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
3254 unsigned Opcode =
3255 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3256 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3257 .addDef(RegNo: ResVReg)
3258 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3259 .addUse(RegNo: I.getOperand(i: 1).getReg())
3260 .addUse(RegNo: OneReg)
3261 .addUse(RegNo: ZeroReg)
3262 .constrainAllUses(TII, TRI, RBI);
3263}
3264
3265bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
3266 const SPIRVType *ResType,
3267 MachineInstr &I, bool IsSigned,
3268 unsigned Opcode) const {
3269 Register SrcReg = I.getOperand(i: 1).getReg();
3270 // We can convert bool value directly to float type without OpConvert*ToF,
3271 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
3272 if (GR.isScalarOrVectorOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool)) {
3273 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3274 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
3275 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
3276 const unsigned NumElts = ResType->getOperand(i: 2).getImm();
3277 TmpType = GR.getOrCreateSPIRVVectorType(BaseType: TmpType, NumElements: NumElts, I, TII);
3278 }
3279 SrcReg = createVirtualRegister(SpvType: TmpType, GR: &GR, MRI, MF: MRI->getMF());
3280 selectSelectDefaultArgs(ResVReg: SrcReg, ResType: TmpType, I, IsSigned: false);
3281 }
3282 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {SrcReg}, Opcode);
3283}
3284
3285bool SPIRVInstructionSelector::selectExt(Register ResVReg,
3286 const SPIRVType *ResType,
3287 MachineInstr &I, bool IsSigned) const {
3288 Register SrcReg = I.getOperand(i: 1).getReg();
3289 if (GR.isScalarOrVectorOfType(VReg: SrcReg, TypeOpcode: SPIRV::OpTypeBool))
3290 return selectSelectDefaultArgs(ResVReg, ResType, I, IsSigned);
3291
3292 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
3293 if (SrcType == ResType)
3294 return BuildCOPY(DestReg: ResVReg, SrcReg, I);
3295
3296 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3297 return selectUnOp(ResVReg, ResType, I, Opcode);
3298}
3299
3300bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
3301 const SPIRVType *ResType,
3302 MachineInstr &I,
3303 bool IsSigned) const {
3304 MachineIRBuilder MIRBuilder(I);
3305 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3306 MachineBasicBlock &BB = *I.getParent();
3307 // Ensure we have bool.
3308 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
3309 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
3310 if (N > 1)
3311 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
3312 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
3313 // Build less-than-equal and less-than.
3314 // TODO: replace with one-liner createVirtualRegister() from
3315 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
3316 Register IsLessEqReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3317 MRI->setType(VReg: IsLessEqReg, Ty: LLT::scalar(SizeInBits: 64));
3318 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessEqReg, MF: MIRBuilder.getMF());
3319 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(),
3320 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThanEqual
3321 : SPIRV::OpULessThanEqual))
3322 .addDef(RegNo: IsLessEqReg)
3323 .addUse(RegNo: BoolTypeReg)
3324 .addUse(RegNo: I.getOperand(i: 1).getReg())
3325 .addUse(RegNo: I.getOperand(i: 2).getReg())
3326 .constrainAllUses(TII, TRI, RBI);
3327 Register IsLessReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3328 MRI->setType(VReg: IsLessReg, Ty: LLT::scalar(SizeInBits: 64));
3329 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessReg, MF: MIRBuilder.getMF());
3330 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(),
3331 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
3332 .addDef(RegNo: IsLessReg)
3333 .addUse(RegNo: BoolTypeReg)
3334 .addUse(RegNo: I.getOperand(i: 1).getReg())
3335 .addUse(RegNo: I.getOperand(i: 2).getReg())
3336 .constrainAllUses(TII, TRI, RBI);
3337 // Build selects.
3338 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
3339 Register NegOneOrZeroReg =
3340 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3341 MRI->setType(VReg: NegOneOrZeroReg, Ty: LLT::scalar(SizeInBits: 64));
3342 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: NegOneOrZeroReg, MF: MIRBuilder.getMF());
3343 unsigned SelectOpcode =
3344 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
3345 Result &= BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3346 .addDef(RegNo: NegOneOrZeroReg)
3347 .addUse(RegNo: ResTypeReg)
3348 .addUse(RegNo: IsLessReg)
3349 .addUse(RegNo: buildOnesVal(AllOnes: true, ResType, I)) // -1
3350 .addUse(RegNo: buildZerosVal(ResType, I))
3351 .constrainAllUses(TII, TRI, RBI);
3352 return Result & BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3353 .addDef(RegNo: ResVReg)
3354 .addUse(RegNo: ResTypeReg)
3355 .addUse(RegNo: IsLessEqReg)
3356 .addUse(RegNo: NegOneOrZeroReg) // -1 or 0
3357 .addUse(RegNo: buildOnesVal(AllOnes: false, ResType, I))
3358 .constrainAllUses(TII, TRI, RBI);
3359}
3360
3361bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
3362 Register ResVReg,
3363 MachineInstr &I,
3364 const SPIRVType *IntTy,
3365 const SPIRVType *BoolTy) const {
3366 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
3367 Register BitIntReg = createVirtualRegister(SpvType: IntTy, GR: &GR, MRI, MF: MRI->getMF());
3368 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
3369 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
3370 Register Zero = buildZerosVal(ResType: IntTy, I);
3371 Register One = buildOnesVal(AllOnes: false, ResType: IntTy, I);
3372 MachineBasicBlock &BB = *I.getParent();
3373 bool Result = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3374 .addDef(RegNo: BitIntReg)
3375 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntTy))
3376 .addUse(RegNo: IntReg)
3377 .addUse(RegNo: One)
3378 .constrainAllUses(TII, TRI, RBI);
3379 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
3380 .addDef(RegNo: ResVReg)
3381 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
3382 .addUse(RegNo: BitIntReg)
3383 .addUse(RegNo: Zero)
3384 .constrainAllUses(TII, TRI, RBI);
3385}
3386
3387bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
3388 const SPIRVType *ResType,
3389 MachineInstr &I) const {
3390 Register IntReg = I.getOperand(i: 1).getReg();
3391 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(VReg: IntReg);
3392 if (GR.isScalarOrVectorOfType(VReg: ResVReg, TypeOpcode: SPIRV::OpTypeBool))
3393 return selectIntToBool(IntReg, ResVReg, I, IntTy: ArgType, BoolTy: ResType);
3394 if (ArgType == ResType)
3395 return BuildCOPY(DestReg: ResVReg, SrcReg: IntReg, I);
3396 bool IsSigned = GR.isScalarOrVectorSigned(Type: ResType);
3397 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3398 return selectUnOp(ResVReg, ResType, I, Opcode);
3399}
3400
3401bool SPIRVInstructionSelector::selectConst(Register ResVReg,
3402 const SPIRVType *ResType,
3403 MachineInstr &I) const {
3404 unsigned Opcode = I.getOpcode();
3405 unsigned TpOpcode = ResType->getOpcode();
3406 Register Reg;
3407 if (TpOpcode == SPIRV::OpTypePointer || TpOpcode == SPIRV::OpTypeEvent) {
3408 assert(Opcode == TargetOpcode::G_CONSTANT &&
3409 I.getOperand(1).getCImm()->isZero());
3410 MachineBasicBlock &DepMBB = I.getMF()->front();
3411 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
3412 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, SpvType: ResType);
3413 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
3414 Reg = GR.getOrCreateConstFP(Val: I.getOperand(i: 1).getFPImm()->getValue(), I,
3415 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3416 } else {
3417 Reg = GR.getOrCreateConstInt(Val: I.getOperand(i: 1).getCImm()->getZExtValue(), I,
3418 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3419 }
3420 return Reg == ResVReg ? true : BuildCOPY(DestReg: ResVReg, SrcReg: Reg, I);
3421}
3422
3423bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
3424 const SPIRVType *ResType,
3425 MachineInstr &I) const {
3426 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3427 .addDef(RegNo: ResVReg)
3428 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3429 .constrainAllUses(TII, TRI, RBI);
3430}
3431
3432bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
3433 const SPIRVType *ResType,
3434 MachineInstr &I) const {
3435 MachineBasicBlock &BB = *I.getParent();
3436 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
3437 .addDef(RegNo: ResVReg)
3438 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3439 // object to insert
3440 .addUse(RegNo: I.getOperand(i: 3).getReg())
3441 // composite to insert into
3442 .addUse(RegNo: I.getOperand(i: 2).getReg());
3443 for (unsigned i = 4; i < I.getNumOperands(); i++)
3444 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3445 return MIB.constrainAllUses(TII, TRI, RBI);
3446}
3447
3448bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
3449 const SPIRVType *ResType,
3450 MachineInstr &I) const {
3451 Type *MaybeResTy = nullptr;
3452 StringRef ResName;
3453 if (GR.findValueAttrs(Key: &I, Ty&: MaybeResTy, Name&: ResName) &&
3454 MaybeResTy != GR.getTypeForSPIRVType(Ty: ResType)) {
3455 assert(!MaybeResTy ||
3456 MaybeResTy->isAggregateType() &&
3457 "Expected aggregate type for extractv instruction");
3458 ResType = GR.getOrCreateSPIRVType(Type: MaybeResTy, I,
3459 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
3460 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *I.getMF());
3461 }
3462 MachineBasicBlock &BB = *I.getParent();
3463 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
3464 .addDef(RegNo: ResVReg)
3465 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3466 .addUse(RegNo: I.getOperand(i: 2).getReg());
3467 for (unsigned i = 3; i < I.getNumOperands(); i++)
3468 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3469 return MIB.constrainAllUses(TII, TRI, RBI);
3470}
3471
3472bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
3473 const SPIRVType *ResType,
3474 MachineInstr &I) const {
3475 if (getImm(MO: I.getOperand(i: 4), MRI))
3476 return selectInsertVal(ResVReg, ResType, I);
3477 MachineBasicBlock &BB = *I.getParent();
3478 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorInsertDynamic))
3479 .addDef(RegNo: ResVReg)
3480 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3481 .addUse(RegNo: I.getOperand(i: 2).getReg())
3482 .addUse(RegNo: I.getOperand(i: 3).getReg())
3483 .addUse(RegNo: I.getOperand(i: 4).getReg())
3484 .constrainAllUses(TII, TRI, RBI);
3485}
3486
3487bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
3488 const SPIRVType *ResType,
3489 MachineInstr &I) const {
3490 if (getImm(MO: I.getOperand(i: 3), MRI))
3491 return selectExtractVal(ResVReg, ResType, I);
3492 MachineBasicBlock &BB = *I.getParent();
3493 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorExtractDynamic))
3494 .addDef(RegNo: ResVReg)
3495 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3496 .addUse(RegNo: I.getOperand(i: 2).getReg())
3497 .addUse(RegNo: I.getOperand(i: 3).getReg())
3498 .constrainAllUses(TII, TRI, RBI);
3499}
3500
3501bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
3502 const SPIRVType *ResType,
3503 MachineInstr &I) const {
3504 const bool IsGEPInBounds = I.getOperand(i: 2).getImm();
3505
3506 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
3507 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
3508 // we have to use Op[InBounds]AccessChain.
3509 const unsigned Opcode = STI.isLogicalSPIRV()
3510 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
3511 : SPIRV::OpAccessChain)
3512 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
3513 : SPIRV::OpPtrAccessChain);
3514
3515 auto Res = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3516 .addDef(RegNo: ResVReg)
3517 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3518 // Object to get a pointer to.
3519 .addUse(RegNo: I.getOperand(i: 3).getReg());
3520 assert(
3521 (Opcode == SPIRV::OpPtrAccessChain ||
3522 Opcode == SPIRV::OpInBoundsPtrAccessChain ||
3523 (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0)) &&
3524 "Cannot translate GEP to OpAccessChain. First index must be 0.");
3525
3526 // Adding indices.
3527 const unsigned StartingIndex =
3528 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
3529 ? 5
3530 : 4;
3531 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
3532 Res.addUse(RegNo: I.getOperand(i).getReg());
3533 return Res.constrainAllUses(TII, TRI, RBI);
3534}
3535
3536// Maybe wrap a value into OpSpecConstantOp
3537bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
3538 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
3539 bool Result = true;
3540 unsigned Lim = I.getNumExplicitOperands();
3541 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
3542 Register OpReg = I.getOperand(i).getReg();
3543 MachineInstr *OpDefine = MRI->getVRegDef(Reg: OpReg);
3544 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3545 SmallPtrSet<SPIRVType *, 4> Visited;
3546 if (!OpDefine || !OpType || isConstReg(MRI, OpDef: OpDefine, Visited) ||
3547 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
3548 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
3549 GR.isAggregateType(Type: OpType)) {
3550 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
3551 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
3552 CompositeArgs.push_back(Elt: OpReg);
3553 continue;
3554 }
3555 MachineFunction *MF = I.getMF();
3556 Register WrapReg = GR.find(MI: OpDefine, MF);
3557 if (WrapReg.isValid()) {
3558 CompositeArgs.push_back(Elt: WrapReg);
3559 continue;
3560 }
3561 // Create a new register for the wrapper
3562 WrapReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: OpType));
3563 CompositeArgs.push_back(Elt: WrapReg);
3564 // Decorate the wrapper register and generate a new instruction
3565 MRI->setType(VReg: WrapReg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3566 GR.assignSPIRVTypeToVReg(Type: OpType, VReg: WrapReg, MF: *MF);
3567 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3568 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
3569 .addDef(RegNo: WrapReg)
3570 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: OpType))
3571 .addImm(Val: static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
3572 .addUse(RegNo: OpReg);
3573 GR.add(Obj: OpDefine, MI: MIB);
3574 Result = MIB.constrainAllUses(TII, TRI, RBI);
3575 if (!Result)
3576 break;
3577 }
3578 return Result;
3579}
3580
3581bool SPIRVInstructionSelector::selectDerivativeInst(
3582 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3583 const unsigned DPdOpCode) const {
3584 // TODO: This should check specifically for Fragment Execution Model, but STI
3585 // doesn't provide that information yet. See #167562
3586 errorIfInstrOutsideShader(I);
3587
3588 // If the arg/result types are half then we need to wrap the instr in
3589 // conversions to float
3590 // This case occurs because a half arg/result is legal in HLSL but not spirv.
3591 Register SrcReg = I.getOperand(i: 2).getReg();
3592 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
3593 unsigned BitWidth = std::min(a: GR.getScalarOrVectorBitWidth(Type: SrcType),
3594 b: GR.getScalarOrVectorBitWidth(Type: ResType));
3595 if (BitWidth == 32)
3596 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
3597 .addDef(RegNo: ResVReg)
3598 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3599 .addUse(RegNo: I.getOperand(i: 2).getReg());
3600
3601 MachineIRBuilder MIRBuilder(I);
3602 unsigned componentCount = GR.getScalarOrVectorComponentCount(Type: SrcType);
3603 SPIRVType *F32ConvertTy = GR.getOrCreateSPIRVFloatType(BitWidth: 32, I, TII);
3604 if (componentCount != 1)
3605 F32ConvertTy = GR.getOrCreateSPIRVVectorType(BaseType: F32ConvertTy, NumElements: componentCount,
3606 MIRBuilder, EmitIR: false);
3607
3608 const TargetRegisterClass *RegClass = GR.getRegClass(SpvType: SrcType);
3609 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
3610 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
3611
3612 bool Result =
3613 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
3614 .addDef(RegNo: ConvertToVReg)
3615 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
3616 .addUse(RegNo: SrcReg)
3617 .constrainAllUses(TII, TRI, RBI);
3618 Result &= BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
3619 .addDef(RegNo: DpdOpVReg)
3620 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
3621 .addUse(RegNo: ConvertToVReg)
3622 .constrainAllUses(TII, TRI, RBI);
3623 Result &=
3624 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
3625 .addDef(RegNo: ResVReg)
3626 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3627 .addUse(RegNo: DpdOpVReg)
3628 .constrainAllUses(TII, TRI, RBI);
3629 return Result;
3630}
3631
3632bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
3633 const SPIRVType *ResType,
3634 MachineInstr &I) const {
3635 MachineBasicBlock &BB = *I.getParent();
3636 Intrinsic::ID IID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
3637 switch (IID) {
3638 case Intrinsic::spv_load:
3639 return selectLoad(ResVReg, ResType, I);
3640 case Intrinsic::spv_store:
3641 return selectStore(I);
3642 case Intrinsic::spv_extractv:
3643 return selectExtractVal(ResVReg, ResType, I);
3644 case Intrinsic::spv_insertv:
3645 return selectInsertVal(ResVReg, ResType, I);
3646 case Intrinsic::spv_extractelt:
3647 return selectExtractElt(ResVReg, ResType, I);
3648 case Intrinsic::spv_insertelt:
3649 return selectInsertElt(ResVReg, ResType, I);
3650 case Intrinsic::spv_gep:
3651 return selectGEP(ResVReg, ResType, I);
3652 case Intrinsic::spv_bitcast: {
3653 Register OpReg = I.getOperand(i: 2).getReg();
3654 SPIRVType *OpType =
3655 OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
3656 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
3657 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
3658 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {OpReg}, Opcode: SPIRV::OpBitcast);
3659 }
3660 case Intrinsic::spv_unref_global:
3661 case Intrinsic::spv_init_global: {
3662 MachineInstr *MI = MRI->getVRegDef(Reg: I.getOperand(i: 1).getReg());
3663 MachineInstr *Init = I.getNumExplicitOperands() > 2
3664 ? MRI->getVRegDef(Reg: I.getOperand(i: 2).getReg())
3665 : nullptr;
3666 assert(MI);
3667 Register GVarVReg = MI->getOperand(i: 0).getReg();
3668 bool Res = selectGlobalValue(ResVReg: GVarVReg, I&: *MI, Init);
3669 // We violate SSA form by inserting OpVariable and still having a gMIR
3670 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
3671 // the duplicated definition.
3672 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
3673 GR.invalidateMachineInstr(MI);
3674 MI->removeFromParent();
3675 }
3676 return Res;
3677 }
3678 case Intrinsic::spv_undef: {
3679 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3680 .addDef(RegNo: ResVReg)
3681 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3682 return MIB.constrainAllUses(TII, TRI, RBI);
3683 }
3684 case Intrinsic::spv_const_composite: {
3685 // If no values are attached, the composite is null constant.
3686 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
3687 SmallVector<Register> CompositeArgs;
3688 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3689
3690 // skip type MD node we already used when generated assign.type for this
3691 if (!IsNull) {
3692 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
3693 return false;
3694 MachineIRBuilder MIR(I);
3695 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
3696 MIRBuilder&: MIR, Opcode: SPIRV::OpConstantComposite, MinWC: 3,
3697 ContinuedOpcode: SPIRV::OpConstantCompositeContinuedINTEL, Args: CompositeArgs, ReturnRegister: ResVReg,
3698 TypeID: GR.getSPIRVTypeID(SpirvType: ResType));
3699 for (auto *Instr : Instructions) {
3700 Instr->setDebugLoc(I.getDebugLoc());
3701 if (!constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI))
3702 return false;
3703 }
3704 return true;
3705 } else {
3706 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3707 .addDef(RegNo: ResVReg)
3708 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3709 return MIB.constrainAllUses(TII, TRI, RBI);
3710 }
3711 }
3712 case Intrinsic::spv_assign_name: {
3713 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName));
3714 MIB.addUse(RegNo: I.getOperand(i: I.getNumExplicitDefs() + 1).getReg());
3715 for (unsigned i = I.getNumExplicitDefs() + 2;
3716 i < I.getNumExplicitOperands(); ++i) {
3717 MIB.addImm(Val: I.getOperand(i).getImm());
3718 }
3719 return MIB.constrainAllUses(TII, TRI, RBI);
3720 }
3721 case Intrinsic::spv_switch: {
3722 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSwitch));
3723 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
3724 if (I.getOperand(i).isReg())
3725 MIB.addReg(RegNo: I.getOperand(i).getReg());
3726 else if (I.getOperand(i).isCImm())
3727 addNumImm(Imm: I.getOperand(i).getCImm()->getValue(), MIB);
3728 else if (I.getOperand(i).isMBB())
3729 MIB.addMBB(MBB: I.getOperand(i).getMBB());
3730 else
3731 llvm_unreachable("Unexpected OpSwitch operand");
3732 }
3733 return MIB.constrainAllUses(TII, TRI, RBI);
3734 }
3735 case Intrinsic::spv_loop_merge: {
3736 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopMerge));
3737 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
3738 if (I.getOperand(i).isMBB())
3739 MIB.addMBB(MBB: I.getOperand(i).getMBB());
3740 else
3741 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3742 }
3743 return MIB.constrainAllUses(TII, TRI, RBI);
3744 }
3745 case Intrinsic::spv_loop_control_intel: {
3746 auto MIB =
3747 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopControlINTEL));
3748 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
3749 MIB.addImm(Val: foldImm(MO: I.getOperand(i: J), MRI));
3750 return MIB.constrainAllUses(TII, TRI, RBI);
3751 }
3752 case Intrinsic::spv_selection_merge: {
3753 auto MIB =
3754 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSelectionMerge));
3755 assert(I.getOperand(1).isMBB() &&
3756 "operand 1 to spv_selection_merge must be a basic block");
3757 MIB.addMBB(MBB: I.getOperand(i: 1).getMBB());
3758 MIB.addImm(Val: getSelectionOperandForImm(Imm: I.getOperand(i: 2).getImm()));
3759 return MIB.constrainAllUses(TII, TRI, RBI);
3760 }
3761 case Intrinsic::spv_cmpxchg:
3762 return selectAtomicCmpXchg(ResVReg, ResType, I);
3763 case Intrinsic::spv_unreachable:
3764 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUnreachable))
3765 .constrainAllUses(TII, TRI, RBI);
3766 case Intrinsic::spv_alloca:
3767 return selectFrameIndex(ResVReg, ResType, I);
3768 case Intrinsic::spv_alloca_array:
3769 return selectAllocaArray(ResVReg, ResType, I);
3770 case Intrinsic::spv_assume:
3771 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
3772 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAssumeTrueKHR))
3773 .addUse(RegNo: I.getOperand(i: 1).getReg())
3774 .constrainAllUses(TII, TRI, RBI);
3775 break;
3776 case Intrinsic::spv_expect:
3777 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
3778 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExpectKHR))
3779 .addDef(RegNo: ResVReg)
3780 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3781 .addUse(RegNo: I.getOperand(i: 2).getReg())
3782 .addUse(RegNo: I.getOperand(i: 3).getReg())
3783 .constrainAllUses(TII, TRI, RBI);
3784 break;
3785 case Intrinsic::arithmetic_fence:
3786 if (STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
3787 return BuildMI(BB, I, MIMD: I.getDebugLoc(),
3788 MCID: TII.get(Opcode: SPIRV::OpArithmeticFenceEXT))
3789 .addDef(RegNo: ResVReg)
3790 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3791 .addUse(RegNo: I.getOperand(i: 2).getReg())
3792 .constrainAllUses(TII, TRI, RBI);
3793 else
3794 return BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 2).getReg(), I);
3795 break;
3796 case Intrinsic::spv_thread_id:
3797 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
3798 // intrinsic in LLVM IR for SPIR-V backend.
3799 //
3800 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
3801 // `GlobalInvocationId` builtin variable
3802 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
3803 ResType, I);
3804 case Intrinsic::spv_thread_id_in_group:
3805 // The HLSL SV_GroupThreadId semantic is lowered to
3806 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
3807 //
3808 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
3809 // translated to a `LocalInvocationId` builtin variable
3810 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationId, ResVReg,
3811 ResType, I);
3812 case Intrinsic::spv_group_id:
3813 // The HLSL SV_GroupId semantic is lowered to
3814 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
3815 //
3816 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
3817 // builtin variable
3818 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
3819 I);
3820 case Intrinsic::spv_flattened_thread_id_in_group:
3821 // The HLSL SV_GroupIndex semantic is lowered to
3822 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
3823 // backend.
3824 //
3825 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
3826 // a `LocalInvocationIndex` builtin variable
3827 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
3828 ResType, I);
3829 case Intrinsic::spv_workgroup_size:
3830 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupSize, ResVReg,
3831 ResType, I);
3832 case Intrinsic::spv_global_size:
3833 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
3834 I);
3835 case Intrinsic::spv_global_offset:
3836 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalOffset, ResVReg,
3837 ResType, I);
3838 case Intrinsic::spv_num_workgroups:
3839 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumWorkgroups, ResVReg,
3840 ResType, I);
3841 case Intrinsic::spv_subgroup_size:
3842 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
3843 I);
3844 case Intrinsic::spv_num_subgroups:
3845 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
3846 I);
3847 case Intrinsic::spv_subgroup_id:
3848 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
3849 case Intrinsic::spv_subgroup_local_invocation_id:
3850 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupLocalInvocationId,
3851 ResVReg, ResType, I);
3852 case Intrinsic::spv_subgroup_max_size:
3853 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
3854 I);
3855 case Intrinsic::spv_fdot:
3856 return selectFloatDot(ResVReg, ResType, I);
3857 case Intrinsic::spv_udot:
3858 case Intrinsic::spv_sdot:
3859 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3860 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3861 return selectIntegerDot(ResVReg, ResType, I,
3862 /*Signed=*/IID == Intrinsic::spv_sdot);
3863 return selectIntegerDotExpansion(ResVReg, ResType, I);
3864 case Intrinsic::spv_dot4add_i8packed:
3865 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3866 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3867 return selectDot4AddPacked<true>(ResVReg, ResType, I);
3868 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
3869 case Intrinsic::spv_dot4add_u8packed:
3870 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3871 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3872 return selectDot4AddPacked<false>(ResVReg, ResType, I);
3873 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
3874 case Intrinsic::spv_all:
3875 return selectAll(ResVReg, ResType, I);
3876 case Intrinsic::spv_any:
3877 return selectAny(ResVReg, ResType, I);
3878 case Intrinsic::spv_cross:
3879 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cross, GLInst: GL::Cross);
3880 case Intrinsic::spv_distance:
3881 return selectExtInst(ResVReg, ResType, I, CLInst: CL::distance, GLInst: GL::Distance);
3882 case Intrinsic::spv_lerp:
3883 return selectExtInst(ResVReg, ResType, I, CLInst: CL::mix, GLInst: GL::FMix);
3884 case Intrinsic::spv_length:
3885 return selectExtInst(ResVReg, ResType, I, CLInst: CL::length, GLInst: GL::Length);
3886 case Intrinsic::spv_degrees:
3887 return selectExtInst(ResVReg, ResType, I, CLInst: CL::degrees, GLInst: GL::Degrees);
3888 case Intrinsic::spv_faceforward:
3889 return selectExtInst(ResVReg, ResType, I, GLInst: GL::FaceForward);
3890 case Intrinsic::spv_frac:
3891 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fract, GLInst: GL::Fract);
3892 case Intrinsic::spv_isinf:
3893 return selectOpIsInf(ResVReg, ResType, I);
3894 case Intrinsic::spv_isnan:
3895 return selectOpIsNan(ResVReg, ResType, I);
3896 case Intrinsic::spv_normalize:
3897 return selectExtInst(ResVReg, ResType, I, CLInst: CL::normalize, GLInst: GL::Normalize);
3898 case Intrinsic::spv_refract:
3899 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Refract);
3900 case Intrinsic::spv_reflect:
3901 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Reflect);
3902 case Intrinsic::spv_rsqrt:
3903 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rsqrt, GLInst: GL::InverseSqrt);
3904 case Intrinsic::spv_sign:
3905 return selectSign(ResVReg, ResType, I);
3906 case Intrinsic::spv_smoothstep:
3907 return selectExtInst(ResVReg, ResType, I, CLInst: CL::smoothstep, GLInst: GL::SmoothStep);
3908 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
3909 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
3910 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
3911 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
3912 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
3913 return selectFirstBitLow(ResVReg, ResType, I);
3914 case Intrinsic::spv_group_memory_barrier_with_group_sync: {
3915 bool Result = true;
3916 auto MemSemConstant =
3917 buildI32Constant(Val: SPIRV::MemorySemantics::SequentiallyConsistent, I);
3918 Register MemSemReg = MemSemConstant.first;
3919 Result &= MemSemConstant.second;
3920 auto ScopeConstant = buildI32Constant(Val: SPIRV::Scope::Workgroup, I);
3921 Register ScopeReg = ScopeConstant.first;
3922 Result &= ScopeConstant.second;
3923 MachineBasicBlock &BB = *I.getParent();
3924 return Result &&
3925 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpControlBarrier))
3926 .addUse(RegNo: ScopeReg)
3927 .addUse(RegNo: ScopeReg)
3928 .addUse(RegNo: MemSemReg)
3929 .constrainAllUses(TII, TRI, RBI);
3930 }
3931 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
3932 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 1).getReg();
3933 SPIRV::StorageClass::StorageClass ResSC =
3934 GR.getPointerStorageClass(Type: ResType);
3935 if (!isGenericCastablePtr(SC: ResSC))
3936 report_fatal_error(reason: "The target storage class is not castable from the "
3937 "Generic storage class");
3938 return BuildMI(BB, I, MIMD: I.getDebugLoc(),
3939 MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtrExplicit))
3940 .addDef(RegNo: ResVReg)
3941 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3942 .addUse(RegNo: PtrReg)
3943 .addImm(Val: ResSC)
3944 .constrainAllUses(TII, TRI, RBI);
3945 }
3946 case Intrinsic::spv_lifetime_start:
3947 case Intrinsic::spv_lifetime_end: {
3948 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
3949 : SPIRV::OpLifetimeStop;
3950 int64_t Size = I.getOperand(i: I.getNumExplicitDefs() + 1).getImm();
3951 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 2).getReg();
3952 if (Size == -1)
3953 Size = 0;
3954 return BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Op))
3955 .addUse(RegNo: PtrReg)
3956 .addImm(Val: Size)
3957 .constrainAllUses(TII, TRI, RBI);
3958 }
3959 case Intrinsic::spv_saturate:
3960 return selectSaturate(ResVReg, ResType, I);
3961 case Intrinsic::spv_nclamp:
3962 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fclamp, GLInst: GL::NClamp);
3963 case Intrinsic::spv_uclamp:
3964 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_clamp, GLInst: GL::UClamp);
3965 case Intrinsic::spv_sclamp:
3966 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_clamp, GLInst: GL::SClamp);
3967 case Intrinsic::spv_subgroup_prefix_bit_count:
3968 return selectWavePrefixBitCount(ResVReg, ResType, I);
3969 case Intrinsic::spv_wave_active_countbits:
3970 return selectWaveActiveCountBits(ResVReg, ResType, I);
3971 case Intrinsic::spv_wave_all:
3972 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAll);
3973 case Intrinsic::spv_wave_any:
3974 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAny);
3975 case Intrinsic::spv_subgroup_ballot:
3976 return selectWaveOpInst(ResVReg, ResType, I,
3977 Opcode: SPIRV::OpGroupNonUniformBallot);
3978 case Intrinsic::spv_wave_is_first_lane:
3979 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformElect);
3980 case Intrinsic::spv_wave_reduce_umax:
3981 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
3982 case Intrinsic::spv_wave_reduce_max:
3983 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
3984 case Intrinsic::spv_wave_reduce_umin:
3985 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
3986 case Intrinsic::spv_wave_reduce_min:
3987 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
3988 case Intrinsic::spv_wave_reduce_sum:
3989 return selectWaveReduceSum(ResVReg, ResType, I);
3990 case Intrinsic::spv_wave_readlane:
3991 return selectWaveOpInst(ResVReg, ResType, I,
3992 Opcode: SPIRV::OpGroupNonUniformShuffle);
3993 case Intrinsic::spv_wave_prefix_sum:
3994 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
3995 case Intrinsic::spv_step:
3996 return selectExtInst(ResVReg, ResType, I, CLInst: CL::step, GLInst: GL::Step);
3997 case Intrinsic::spv_radians:
3998 return selectExtInst(ResVReg, ResType, I, CLInst: CL::radians, GLInst: GL::Radians);
3999 // Discard intrinsics which we do not expect to actually represent code after
4000 // lowering or intrinsics which are not implemented but should not crash when
4001 // found in a customer's LLVM IR input.
4002 case Intrinsic::instrprof_increment:
4003 case Intrinsic::instrprof_increment_step:
4004 case Intrinsic::instrprof_value_profile:
4005 break;
4006 // Discard internal intrinsics.
4007 case Intrinsic::spv_value_md:
4008 break;
4009 case Intrinsic::spv_resource_handlefrombinding: {
4010 return selectHandleFromBinding(ResVReg, ResType, I);
4011 }
4012 case Intrinsic::spv_resource_counterhandlefrombinding:
4013 return selectCounterHandleFromBinding(ResVReg, ResType, I);
4014 case Intrinsic::spv_resource_updatecounter:
4015 return selectUpdateCounter(ResVReg, ResType, I);
4016 case Intrinsic::spv_resource_store_typedbuffer: {
4017 return selectImageWriteIntrinsic(I);
4018 }
4019 case Intrinsic::spv_resource_load_typedbuffer: {
4020 return selectReadImageIntrinsic(ResVReg, ResType, I);
4021 }
4022 case Intrinsic::spv_resource_sample:
4023 case Intrinsic::spv_resource_sample_clamp: {
4024 return selectSampleIntrinsic(ResVReg, ResType, I);
4025 }
4026 case Intrinsic::spv_resource_getpointer: {
4027 return selectResourceGetPointer(ResVReg, ResType, I);
4028 }
4029 case Intrinsic::spv_pushconstant_getpointer: {
4030 return selectPushConstantGetPointer(ResVReg, ResType, I);
4031 }
4032 case Intrinsic::spv_discard: {
4033 return selectDiscard(ResVReg, ResType, I);
4034 }
4035 case Intrinsic::spv_resource_nonuniformindex: {
4036 return selectResourceNonUniformIndex(ResVReg, ResType, I);
4037 }
4038 case Intrinsic::spv_unpackhalf2x16: {
4039 return selectExtInst(ResVReg, ResType, I, GLInst: GL::UnpackHalf2x16);
4040 }
4041 case Intrinsic::spv_packhalf2x16: {
4042 return selectExtInst(ResVReg, ResType, I, GLInst: GL::PackHalf2x16);
4043 }
4044 case Intrinsic::spv_ddx:
4045 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdx);
4046 case Intrinsic::spv_ddy:
4047 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdy);
4048 case Intrinsic::spv_ddx_coarse:
4049 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxCoarse);
4050 case Intrinsic::spv_ddy_coarse:
4051 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyCoarse);
4052 case Intrinsic::spv_ddx_fine:
4053 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxFine);
4054 case Intrinsic::spv_ddy_fine:
4055 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyFine);
4056 case Intrinsic::spv_fwidth:
4057 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpFwidth);
4058 default: {
4059 std::string DiagMsg;
4060 raw_string_ostream OS(DiagMsg);
4061 I.print(OS);
4062 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
4063 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
4064 }
4065 }
4066 return true;
4067}
4068
4069bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
4070 const SPIRVType *ResType,
4071 MachineInstr &I) const {
4072 // The images need to be loaded in the same basic block as their use. We defer
4073 // loading the image to the intrinsic that uses it.
4074 if (ResType->getOpcode() == SPIRV::OpTypeImage)
4075 return true;
4076
4077 return loadHandleBeforePosition(HandleReg&: ResVReg, ResType: GR.getSPIRVTypeForVReg(VReg: ResVReg),
4078 HandleDef&: *cast<GIntrinsic>(Val: &I), Pos&: I);
4079}
4080
4081bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
4082 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
4083 auto &Intr = cast<GIntrinsic>(Val&: I);
4084 assert(Intr.getIntrinsicID() ==
4085 Intrinsic::spv_resource_counterhandlefrombinding);
4086
4087 // Extract information from the intrinsic call.
4088 Register MainHandleReg = Intr.getOperand(i: 2).getReg();
4089 auto *MainHandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: MainHandleReg));
4090 assert(MainHandleDef->getIntrinsicID() ==
4091 Intrinsic::spv_resource_handlefrombinding);
4092
4093 uint32_t Set = getIConstVal(ConstReg: Intr.getOperand(i: 4).getReg(), MRI);
4094 uint32_t Binding = getIConstVal(ConstReg: Intr.getOperand(i: 3).getReg(), MRI);
4095 uint32_t ArraySize = getIConstVal(ConstReg: MainHandleDef->getOperand(i: 4).getReg(), MRI);
4096 Register IndexReg = MainHandleDef->getOperand(i: 5).getReg();
4097 std::string CounterName =
4098 getStringValueFromReg(Reg: MainHandleDef->getOperand(i: 6).getReg(), MRI&: *MRI) +
4099 ".counter";
4100
4101 // Create the counter variable.
4102 MachineIRBuilder MIRBuilder(I);
4103 Register CounterVarReg = buildPointerToResource(
4104 ResType: GR.getPointeeType(PtrType: ResType), SC: GR.getPointerStorageClass(Type: ResType), Set,
4105 Binding, ArraySize, IndexReg, Name: CounterName, MIRBuilder);
4106
4107 return BuildCOPY(DestReg: ResVReg, SrcReg: CounterVarReg, I);
4108}
4109
4110bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
4111 const SPIRVType *ResType,
4112 MachineInstr &I) const {
4113 auto &Intr = cast<GIntrinsic>(Val&: I);
4114 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
4115
4116 Register CounterHandleReg = Intr.getOperand(i: 2).getReg();
4117 Register IncrReg = Intr.getOperand(i: 3).getReg();
4118
4119 // The counter handle is a pointer to the counter variable (which is a struct
4120 // containing an i32). We need to get a pointer to that i32 member to do the
4121 // atomic operation.
4122#ifndef NDEBUG
4123 SPIRVType *CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
4124 SPIRVType *CounterVarPointeeType = GR.getPointeeType(CounterVarType);
4125 assert(CounterVarPointeeType &&
4126 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
4127 "Counter variable must be a struct");
4128 assert(GR.getPointerStorageClass(CounterVarType) ==
4129 SPIRV::StorageClass::StorageBuffer &&
4130 "Counter variable must be in the storage buffer storage class");
4131 assert(CounterVarPointeeType->getNumOperands() == 2 &&
4132 "Counter variable must have exactly 1 member in the struct");
4133 const SPIRVType *MemberType =
4134 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
4135 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
4136 "Counter variable struct must have a single i32 member");
4137#endif
4138
4139 // The struct has a single i32 member.
4140 MachineIRBuilder MIRBuilder(I);
4141 const Type *LLVMIntType =
4142 Type::getInt32Ty(C&: I.getMF()->getFunction().getContext());
4143
4144 SPIRVType *IntPtrType = GR.getOrCreateSPIRVPointerType(
4145 BaseType: LLVMIntType, MIRBuilder, SC: SPIRV::StorageClass::StorageBuffer);
4146
4147 auto Zero = buildI32Constant(Val: 0, I);
4148 if (!Zero.second)
4149 return false;
4150
4151 Register PtrToCounter =
4152 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: IntPtrType));
4153 if (!BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4154 MCID: TII.get(Opcode: SPIRV::OpAccessChain))
4155 .addDef(RegNo: PtrToCounter)
4156 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntPtrType))
4157 .addUse(RegNo: CounterHandleReg)
4158 .addUse(RegNo: Zero.first)
4159 .constrainAllUses(TII, TRI, RBI)) {
4160 return false;
4161 }
4162
4163 // For UAV/SSBO counters, the scope is Device. The counter variable is not
4164 // used as a flag. So the memory semantics can be None.
4165 auto Scope = buildI32Constant(Val: SPIRV::Scope::Device, I);
4166 if (!Scope.second)
4167 return false;
4168 auto Semantics = buildI32Constant(Val: SPIRV::MemorySemantics::None, I);
4169 if (!Semantics.second)
4170 return false;
4171
4172 int64_t IncrVal = getIConstValSext(ConstReg: IncrReg, MRI);
4173 auto Incr = buildI32Constant(Val: static_cast<uint32_t>(IncrVal), I);
4174 if (!Incr.second)
4175 return false;
4176
4177 Register AtomicRes = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4178 if (!BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAtomicIAdd))
4179 .addDef(RegNo: AtomicRes)
4180 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4181 .addUse(RegNo: PtrToCounter)
4182 .addUse(RegNo: Scope.first)
4183 .addUse(RegNo: Semantics.first)
4184 .addUse(RegNo: Incr.first)
4185 .constrainAllUses(TII, TRI, RBI)) {
4186 return false;
4187 }
4188 if (IncrVal >= 0) {
4189 return BuildCOPY(DestReg: ResVReg, SrcReg: AtomicRes, I);
4190 }
4191
4192 // In HLSL, IncrementCounter returns the value *before* the increment, while
4193 // DecrementCounter returns the value *after* the decrement. Both are lowered
4194 // to the same atomic intrinsic which returns the value *before* the
4195 // operation. So for decrements (negative IncrVal), we must subtract the
4196 // increment value from the result to get the post-decrement value.
4197 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
4198 .addDef(RegNo: ResVReg)
4199 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4200 .addUse(RegNo: AtomicRes)
4201 .addUse(RegNo: Incr.first)
4202 .constrainAllUses(TII, TRI, RBI);
4203}
4204bool SPIRVInstructionSelector::selectReadImageIntrinsic(
4205 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
4206
4207 // If the load of the image is in a different basic block, then
4208 // this will generate invalid code. A proper solution is to move
4209 // the OpLoad from selectHandleFromBinding here. However, to do
4210 // that we will need to change the return type of the intrinsic.
4211 // We will do that when we can, but for now trying to move forward with other
4212 // issues.
4213 Register ImageReg = I.getOperand(i: 2).getReg();
4214 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4215 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4216 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4217 HandleDef&: *ImageDef, Pos&: I)) {
4218 return false;
4219 }
4220
4221 Register IdxReg = I.getOperand(i: 3).getReg();
4222 DebugLoc Loc = I.getDebugLoc();
4223 MachineInstr &Pos = I;
4224
4225 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewImageReg, IdxReg, Loc,
4226 Pos);
4227}
4228
4229bool SPIRVInstructionSelector::selectSampleIntrinsic(Register &ResVReg,
4230 const SPIRVType *ResType,
4231 MachineInstr &I) const {
4232 Register ImageReg = I.getOperand(i: 2).getReg();
4233 Register SamplerReg = I.getOperand(i: 3).getReg();
4234 Register CoordinateReg = I.getOperand(i: 4).getReg();
4235 std::optional<Register> OffsetReg;
4236 std::optional<Register> ClampReg;
4237
4238 if (I.getNumOperands() > 5)
4239 OffsetReg = I.getOperand(i: 5).getReg();
4240 if (I.getNumOperands() > 6)
4241 ClampReg = I.getOperand(i: 6).getReg();
4242
4243 DebugLoc Loc = I.getDebugLoc();
4244
4245 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4246 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4247 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4248 HandleDef&: *ImageDef, Pos&: I)) {
4249 return false;
4250 }
4251
4252 auto *SamplerDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: SamplerReg));
4253 Register NewSamplerReg =
4254 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: SamplerReg));
4255 if (!loadHandleBeforePosition(
4256 HandleReg&: NewSamplerReg, ResType: GR.getSPIRVTypeForVReg(VReg: SamplerReg), HandleDef&: *SamplerDef, Pos&: I)) {
4257 return false;
4258 }
4259
4260 MachineIRBuilder MIRBuilder(I);
4261 SPIRVType *SampledImageType = GR.getOrCreateOpTypeSampledImage(
4262 ImageType: GR.getSPIRVTypeForVReg(VReg: ImageReg), MIRBuilder);
4263
4264 Register SampledImageReg =
4265 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: SampledImageType));
4266 bool Succeed = BuildMI(BB&: *I.getParent(), I, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpSampledImage))
4267 .addDef(RegNo: SampledImageReg)
4268 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SampledImageType))
4269 .addUse(RegNo: NewImageReg)
4270 .addUse(RegNo: NewSamplerReg)
4271 .constrainAllUses(TII, TRI, RBI);
4272 if (!Succeed)
4273 return false;
4274
4275 auto MIB =
4276 BuildMI(BB&: *I.getParent(), I, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpImageSampleImplicitLod))
4277 .addDef(RegNo: ResVReg)
4278 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4279 .addUse(RegNo: SampledImageReg)
4280 .addUse(RegNo: CoordinateReg);
4281
4282 uint32_t ImageOperands = 0;
4283 if (OffsetReg && !isScalarOrVectorIntConstantZero(Reg: *OffsetReg)) {
4284 ImageOperands |= 0x8; // ConstOffset
4285 }
4286
4287 if (ClampReg) {
4288 ImageOperands |= 0x80; // MinLod
4289 }
4290
4291 if (ImageOperands != 0) {
4292 MIB.addImm(Val: ImageOperands);
4293 if (ImageOperands & 0x8)
4294 MIB.addUse(RegNo: *OffsetReg);
4295 if (ImageOperands & 0x80)
4296 MIB.addUse(RegNo: *ClampReg);
4297 }
4298
4299 return MIB.constrainAllUses(TII, TRI, RBI);
4300}
4301
4302bool SPIRVInstructionSelector::generateImageReadOrFetch(
4303 Register &ResVReg, const SPIRVType *ResType, Register ImageReg,
4304 Register IdxReg, DebugLoc Loc, MachineInstr &Pos) const {
4305 SPIRVType *ImageType = GR.getSPIRVTypeForVReg(VReg: ImageReg);
4306 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
4307 "ImageReg is not an image type.");
4308
4309 bool IsSignedInteger =
4310 sampledTypeIsSignedInteger(HandleType: GR.getTypeForSPIRVType(Ty: ImageType));
4311 // Check if the "sampled" operand of the image type is 1.
4312 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
4313 auto SampledOp = ImageType->getOperand(i: 6);
4314 bool IsFetch = (SampledOp.getImm() == 1);
4315
4316 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
4317 if (ResultSize == 4) {
4318 auto BMI =
4319 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
4320 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
4321 .addDef(RegNo: ResVReg)
4322 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4323 .addUse(RegNo: ImageReg)
4324 .addUse(RegNo: IdxReg);
4325
4326 if (IsSignedInteger)
4327 BMI.addImm(Val: 0x1000); // SignExtend
4328 return BMI.constrainAllUses(TII, TRI, RBI);
4329 }
4330
4331 SPIRVType *ReadType = widenTypeToVec4(Type: ResType, I&: Pos);
4332 Register ReadReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ReadType));
4333 auto BMI =
4334 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
4335 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
4336 .addDef(RegNo: ReadReg)
4337 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ReadType))
4338 .addUse(RegNo: ImageReg)
4339 .addUse(RegNo: IdxReg);
4340 if (IsSignedInteger)
4341 BMI.addImm(Val: 0x1000); // SignExtend
4342 bool Succeed = BMI.constrainAllUses(TII, TRI, RBI);
4343 if (!Succeed)
4344 return false;
4345
4346 if (ResultSize == 1) {
4347 return BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
4348 MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
4349 .addDef(RegNo: ResVReg)
4350 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4351 .addUse(RegNo: ReadReg)
4352 .addImm(Val: 0)
4353 .constrainAllUses(TII, TRI, RBI);
4354 }
4355 return extractSubvector(ResVReg, ResType, ReadReg, InsertionPoint&: Pos);
4356}
4357
4358bool SPIRVInstructionSelector::selectResourceGetPointer(
4359 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
4360 Register ResourcePtr = I.getOperand(i: 2).getReg();
4361 SPIRVType *RegType = GR.getSPIRVTypeForVReg(VReg: ResourcePtr, MF: I.getMF());
4362 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
4363 // For texel buffers, the index into the image is part of the OpImageRead or
4364 // OpImageWrite instructions. So we will do nothing in this case. This
4365 // intrinsic will be combined with the load or store when selecting the load
4366 // or store.
4367 return true;
4368 }
4369
4370 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
4371 MachineIRBuilder MIRBuilder(I);
4372
4373 Register IndexReg = I.getOperand(i: 3).getReg();
4374 Register ZeroReg =
4375 buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
4376 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4377 MCID: TII.get(Opcode: SPIRV::OpAccessChain))
4378 .addDef(RegNo: ResVReg)
4379 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4380 .addUse(RegNo: ResourcePtr)
4381 .addUse(RegNo: ZeroReg)
4382 .addUse(RegNo: IndexReg)
4383 .constrainAllUses(TII, TRI, RBI);
4384}
4385
4386bool SPIRVInstructionSelector::selectPushConstantGetPointer(
4387 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
4388 MRI->replaceRegWith(FromReg: ResVReg, ToReg: I.getOperand(i: 2).getReg());
4389 return true;
4390}
4391
4392bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
4393 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
4394 Register ObjReg = I.getOperand(i: 2).getReg();
4395 if (!BuildCOPY(DestReg: ResVReg, SrcReg: ObjReg, I))
4396 return false;
4397
4398 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
4399 // Check for the registers that use the index marked as non-uniform
4400 // and recursively mark them as non-uniform.
4401 // Per the spec, it's necessary that the final argument used for
4402 // load/store/sample/atomic must be decorated, so we need to propagate the
4403 // decoration through access chains and copies.
4404 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
4405 decorateUsesAsNonUniform(NonUniformReg&: ResVReg);
4406 return true;
4407}
4408
4409void SPIRVInstructionSelector::decorateUsesAsNonUniform(
4410 Register &NonUniformReg) const {
4411 llvm::SmallVector<Register> WorkList = {NonUniformReg};
4412 while (WorkList.size() > 0) {
4413 Register CurrentReg = WorkList.back();
4414 WorkList.pop_back();
4415
4416 bool IsDecorated = false;
4417 for (MachineInstr &Use : MRI->use_instructions(Reg: CurrentReg)) {
4418 if (Use.getOpcode() == SPIRV::OpDecorate &&
4419 Use.getOperand(i: 1).getImm() == SPIRV::Decoration::NonUniformEXT) {
4420 IsDecorated = true;
4421 continue;
4422 }
4423 // Check if the instruction has the result register and add it to the
4424 // worklist.
4425 if (Use.getOperand(i: 0).isReg() && Use.getOperand(i: 0).isDef()) {
4426 Register ResultReg = Use.getOperand(i: 0).getReg();
4427 if (ResultReg == CurrentReg)
4428 continue;
4429 WorkList.push_back(Elt: ResultReg);
4430 }
4431 }
4432
4433 if (!IsDecorated) {
4434 buildOpDecorate(Reg: CurrentReg, I&: *MRI->getVRegDef(Reg: CurrentReg), TII,
4435 Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
4436 }
4437 }
4438}
4439
4440bool SPIRVInstructionSelector::extractSubvector(
4441 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg,
4442 MachineInstr &InsertionPoint) const {
4443 SPIRVType *InputType = GR.getResultType(VReg: ReadReg);
4444 [[maybe_unused]] uint64_t InputSize =
4445 GR.getScalarOrVectorComponentCount(Type: InputType);
4446 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
4447 assert(InputSize > 1 && "The input must be a vector.");
4448 assert(ResultSize > 1 && "The result must be a vector.");
4449 assert(ResultSize < InputSize &&
4450 "Cannot extract more element than there are in the input.");
4451 SmallVector<Register> ComponentRegisters;
4452 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(Type: ResType);
4453 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(SpvType: ScalarType);
4454 for (uint64_t I = 0; I < ResultSize; I++) {
4455 Register ComponentReg = MRI->createVirtualRegister(RegClass: ScalarRegClass);
4456 bool Succeed = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
4457 MIMD: InsertionPoint.getDebugLoc(),
4458 MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
4459 .addDef(RegNo: ComponentReg)
4460 .addUse(RegNo: ScalarType->getOperand(i: 0).getReg())
4461 .addUse(RegNo: ReadReg)
4462 .addImm(Val: I)
4463 .constrainAllUses(TII, TRI, RBI);
4464 if (!Succeed)
4465 return false;
4466 ComponentRegisters.emplace_back(Args&: ComponentReg);
4467 }
4468
4469 MachineInstrBuilder MIB = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
4470 MIMD: InsertionPoint.getDebugLoc(),
4471 MCID: TII.get(Opcode: SPIRV::OpCompositeConstruct))
4472 .addDef(RegNo: ResVReg)
4473 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
4474
4475 for (Register ComponentReg : ComponentRegisters)
4476 MIB.addUse(RegNo: ComponentReg);
4477 return MIB.constrainAllUses(TII, TRI, RBI);
4478}
4479
4480bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
4481 MachineInstr &I) const {
4482 // If the load of the image is in a different basic block, then
4483 // this will generate invalid code. A proper solution is to move
4484 // the OpLoad from selectHandleFromBinding here. However, to do
4485 // that we will need to change the return type of the intrinsic.
4486 // We will do that when we can, but for now trying to move forward with other
4487 // issues.
4488 Register ImageReg = I.getOperand(i: 1).getReg();
4489 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4490 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4491 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4492 HandleDef&: *ImageDef, Pos&: I)) {
4493 return false;
4494 }
4495
4496 Register CoordinateReg = I.getOperand(i: 2).getReg();
4497 Register DataReg = I.getOperand(i: 3).getReg();
4498 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
4499 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
4500 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4501 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
4502 .addUse(RegNo: NewImageReg)
4503 .addUse(RegNo: CoordinateReg)
4504 .addUse(RegNo: DataReg)
4505 .constrainAllUses(TII, TRI, RBI);
4506}
4507
4508Register SPIRVInstructionSelector::buildPointerToResource(
4509 const SPIRVType *SpirvResType, SPIRV::StorageClass::StorageClass SC,
4510 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
4511 StringRef Name, MachineIRBuilder MIRBuilder) const {
4512 const Type *ResType = GR.getTypeForSPIRVType(Ty: SpirvResType);
4513 if (ArraySize == 1) {
4514 SPIRVType *PtrType =
4515 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
4516 assert(GR.getPointeeType(PtrType) == SpirvResType &&
4517 "SpirvResType did not have an explicit layout.");
4518 return GR.getOrCreateGlobalVariableWithBinding(VarType: PtrType, Set, Binding, Name,
4519 MIRBuilder);
4520 }
4521
4522 const Type *VarType = ArrayType::get(ElementType: const_cast<Type *>(ResType), NumElements: ArraySize);
4523 SPIRVType *VarPointerType =
4524 GR.getOrCreateSPIRVPointerType(BaseType: VarType, MIRBuilder, SC);
4525 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
4526 VarType: VarPointerType, Set, Binding, Name, MIRBuilder);
4527
4528 SPIRVType *ResPointerType =
4529 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
4530 Register AcReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResPointerType));
4531
4532 MIRBuilder.buildInstr(Opcode: SPIRV::OpAccessChain)
4533 .addDef(RegNo: AcReg)
4534 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResPointerType))
4535 .addUse(RegNo: VarReg)
4536 .addUse(RegNo: IndexReg);
4537
4538 return AcReg;
4539}
4540
4541bool SPIRVInstructionSelector::selectFirstBitSet16(
4542 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
4543 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
4544 Register ExtReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4545 bool Result = selectOpWithSrcs(ResVReg: ExtReg, ResType, I, Srcs: {I.getOperand(i: 2).getReg()},
4546 Opcode: ExtendOpcode);
4547
4548 return Result &&
4549 selectFirstBitSet32(ResVReg, ResType, I, SrcReg: ExtReg, BitSetOpcode);
4550}
4551
4552bool SPIRVInstructionSelector::selectFirstBitSet32(
4553 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
4554 Register SrcReg, unsigned BitSetOpcode) const {
4555 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
4556 .addDef(RegNo: ResVReg)
4557 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4558 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
4559 .addImm(Val: BitSetOpcode)
4560 .addUse(RegNo: SrcReg)
4561 .constrainAllUses(TII, TRI, RBI);
4562}
4563
4564bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
4565 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
4566 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
4567
4568 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
4569 // requires creating a param register and return register with an invalid
4570 // vector size. If that is resolved, then this function can be used for
4571 // vectors of any component size.
4572 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
4573 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
4574
4575 MachineIRBuilder MIRBuilder(I);
4576 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
4577 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 64, MIRBuilder);
4578 SPIRVType *I64x2Type =
4579 GR.getOrCreateSPIRVVectorType(BaseType: I64Type, NumElements: 2, MIRBuilder, EmitIR: false);
4580 SPIRVType *Vec2ResType =
4581 GR.getOrCreateSPIRVVectorType(BaseType, NumElements: 2, MIRBuilder, EmitIR: false);
4582
4583 std::vector<Register> PartialRegs;
4584
4585 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
4586 unsigned CurrentComponent = 0;
4587 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
4588 // This register holds the firstbitX result for each of the i64x2 vectors
4589 // extracted from SrcReg
4590 Register BitSetResult =
4591 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64x2Type));
4592
4593 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4594 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4595 .addDef(RegNo: BitSetResult)
4596 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: I64x2Type))
4597 .addUse(RegNo: SrcReg)
4598 .addUse(RegNo: SrcReg)
4599 .addImm(Val: CurrentComponent)
4600 .addImm(Val: CurrentComponent + 1);
4601
4602 if (!MIB.constrainAllUses(TII, TRI, RBI))
4603 return false;
4604
4605 Register SubVecBitSetReg =
4606 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Vec2ResType));
4607
4608 if (!selectFirstBitSet64(ResVReg: SubVecBitSetReg, ResType: Vec2ResType, I, SrcReg: BitSetResult,
4609 BitSetOpcode, SwapPrimarySide))
4610 return false;
4611
4612 PartialRegs.push_back(x: SubVecBitSetReg);
4613 }
4614
4615 // On odd component counts we need to handle one more component
4616 if (CurrentComponent != ComponentCount) {
4617 bool ZeroAsNull = !STI.isShader();
4618 Register FinalElemReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64Type));
4619 Register ConstIntLastIdx = GR.getOrCreateConstInt(
4620 Val: ComponentCount - 1, I, SpvType: BaseType, TII, ZeroAsNull);
4621
4622 if (!selectOpWithSrcs(ResVReg: FinalElemReg, ResType: I64Type, I, Srcs: {SrcReg, ConstIntLastIdx},
4623 Opcode: SPIRV::OpVectorExtractDynamic))
4624 return false;
4625
4626 Register FinalElemBitSetReg =
4627 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BaseType));
4628
4629 if (!selectFirstBitSet64(ResVReg: FinalElemBitSetReg, ResType: BaseType, I, SrcReg: FinalElemReg,
4630 BitSetOpcode, SwapPrimarySide))
4631 return false;
4632
4633 PartialRegs.push_back(x: FinalElemBitSetReg);
4634 }
4635
4636 // Join all the resulting registers back into the return type in order
4637 // (ie i32x2, i32x2, i32x1 -> i32x5)
4638 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: std::move(PartialRegs),
4639 Opcode: SPIRV::OpCompositeConstruct);
4640}
4641
4642bool SPIRVInstructionSelector::selectFirstBitSet64(
4643 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
4644 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
4645 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
4646 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
4647 bool ZeroAsNull = !STI.isShader();
4648 Register ConstIntZero =
4649 GR.getOrCreateConstInt(Val: 0, I, SpvType: BaseType, TII, ZeroAsNull);
4650 Register ConstIntOne =
4651 GR.getOrCreateConstInt(Val: 1, I, SpvType: BaseType, TII, ZeroAsNull);
4652
4653 // SPIRV doesn't support vectors with more than 4 components. Since the
4654 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
4655 // operate on vectors with 2 or less components. When largers vectors are
4656 // seen. Split them, recurse, then recombine them.
4657 if (ComponentCount > 2) {
4658 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
4659 BitSetOpcode, SwapPrimarySide);
4660 }
4661
4662 // 1. Split int64 into 2 pieces using a bitcast
4663 MachineIRBuilder MIRBuilder(I);
4664 SPIRVType *PostCastType = GR.getOrCreateSPIRVVectorType(
4665 BaseType, NumElements: 2 * ComponentCount, MIRBuilder, EmitIR: false);
4666 Register BitcastReg =
4667 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
4668
4669 if (!selectOpWithSrcs(ResVReg: BitcastReg, ResType: PostCastType, I, Srcs: {SrcReg},
4670 Opcode: SPIRV::OpBitcast))
4671 return false;
4672
4673 // 2. Find the first set bit from the primary side for all the pieces in #1
4674 Register FBSReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
4675 if (!selectFirstBitSet32(ResVReg: FBSReg, ResType: PostCastType, I, SrcReg: BitcastReg, BitSetOpcode))
4676 return false;
4677
4678 // 3. Split result vector into high bits and low bits
4679 Register HighReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4680 Register LowReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4681
4682 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
4683 if (IsScalarRes) {
4684 // if scalar do a vector extract
4685 if (!selectOpWithSrcs(ResVReg: HighReg, ResType, I, Srcs: {FBSReg, ConstIntZero},
4686 Opcode: SPIRV::OpVectorExtractDynamic))
4687 return false;
4688 if (!selectOpWithSrcs(ResVReg: LowReg, ResType, I, Srcs: {FBSReg, ConstIntOne},
4689 Opcode: SPIRV::OpVectorExtractDynamic))
4690 return false;
4691 } else {
4692 // if vector do a shufflevector
4693 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4694 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4695 .addDef(RegNo: HighReg)
4696 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4697 .addUse(RegNo: FBSReg)
4698 // Per the spec, repeat the vector if only one vec is needed
4699 .addUse(RegNo: FBSReg);
4700
4701 // high bits are stored in even indexes. Extract them from FBSReg
4702 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
4703 MIB.addImm(Val: J);
4704 }
4705
4706 if (!MIB.constrainAllUses(TII, TRI, RBI))
4707 return false;
4708
4709 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4710 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4711 .addDef(RegNo: LowReg)
4712 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4713 .addUse(RegNo: FBSReg)
4714 // Per the spec, repeat the vector if only one vec is needed
4715 .addUse(RegNo: FBSReg);
4716
4717 // low bits are stored in odd indexes. Extract them from FBSReg
4718 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
4719 MIB.addImm(Val: J);
4720 }
4721 if (!MIB.constrainAllUses(TII, TRI, RBI))
4722 return false;
4723 }
4724
4725 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
4726 // primary
4727 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
4728 Register NegOneReg;
4729 Register Reg0;
4730 Register Reg32;
4731 unsigned SelectOp;
4732 unsigned AddOp;
4733
4734 if (IsScalarRes) {
4735 NegOneReg =
4736 GR.getOrCreateConstInt(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
4737 Reg0 = GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
4738 Reg32 = GR.getOrCreateConstInt(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
4739 SelectOp = SPIRV::OpSelectSISCond;
4740 AddOp = SPIRV::OpIAddS;
4741 } else {
4742 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: ComponentCount,
4743 MIRBuilder, EmitIR: false);
4744 NegOneReg =
4745 GR.getOrCreateConstVector(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
4746 Reg0 = GR.getOrCreateConstVector(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
4747 Reg32 = GR.getOrCreateConstVector(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
4748 SelectOp = SPIRV::OpSelectVIVCond;
4749 AddOp = SPIRV::OpIAddV;
4750 }
4751
4752 Register PrimaryReg = HighReg;
4753 Register SecondaryReg = LowReg;
4754 Register PrimaryShiftReg = Reg32;
4755 Register SecondaryShiftReg = Reg0;
4756
4757 // By default the emitted opcodes check for the set bit from the MSB side.
4758 // Setting SwapPrimarySide checks the set bit from the LSB side
4759 if (SwapPrimarySide) {
4760 PrimaryReg = LowReg;
4761 SecondaryReg = HighReg;
4762 PrimaryShiftReg = Reg0;
4763 SecondaryShiftReg = Reg32;
4764 }
4765
4766 // Check if the primary bits are == -1
4767 Register BReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BoolType));
4768 if (!selectOpWithSrcs(ResVReg: BReg, ResType: BoolType, I, Srcs: {PrimaryReg, NegOneReg},
4769 Opcode: SPIRV::OpIEqual))
4770 return false;
4771
4772 // Select secondary bits if true in BReg, otherwise primary bits
4773 Register TmpReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4774 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {BReg, SecondaryReg, PrimaryReg},
4775 Opcode: SelectOp))
4776 return false;
4777
4778 // 5. Add 32 when high bits are used, otherwise 0 for low bits
4779 Register ValReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4780 if (!selectOpWithSrcs(ResVReg: ValReg, ResType, I,
4781 Srcs: {BReg, SecondaryShiftReg, PrimaryShiftReg}, Opcode: SelectOp))
4782 return false;
4783
4784 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {ValReg, TmpReg}, Opcode: AddOp);
4785}
4786
4787bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
4788 const SPIRVType *ResType,
4789 MachineInstr &I,
4790 bool IsSigned) const {
4791 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
4792 Register OpReg = I.getOperand(i: 2).getReg();
4793 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
4794 // zero or sign extend
4795 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
4796 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
4797
4798 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
4799 case 16:
4800 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
4801 case 32:
4802 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
4803 case 64:
4804 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
4805 /*SwapPrimarySide=*/false);
4806 default:
4807 report_fatal_error(
4808 reason: "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
4809 }
4810}
4811
4812bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
4813 const SPIRVType *ResType,
4814 MachineInstr &I) const {
4815 // FindILsb intrinsic only supports 32 bit integers
4816 Register OpReg = I.getOperand(i: 2).getReg();
4817 SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
4818 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
4819 // to an unsigned i32. As this leaves all the least significant bits unchanged
4820 // so the first set bit from the LSB side doesn't change.
4821 unsigned ExtendOpcode = SPIRV::OpUConvert;
4822 unsigned BitSetOpcode = GL::FindILsb;
4823
4824 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
4825 case 16:
4826 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
4827 case 32:
4828 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
4829 case 64:
4830 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
4831 /*SwapPrimarySide=*/true);
4832 default:
4833 report_fatal_error(reason: "spv_firstbitlow only supports 16,32,64 bits.");
4834 }
4835}
4836
4837bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
4838 const SPIRVType *ResType,
4839 MachineInstr &I) const {
4840 // there was an allocation size parameter to the allocation instruction
4841 // that is not 1
4842 MachineBasicBlock &BB = *I.getParent();
4843 bool Res = BuildMI(BB, I, MIMD: I.getDebugLoc(),
4844 MCID: TII.get(Opcode: SPIRV::OpVariableLengthArrayINTEL))
4845 .addDef(RegNo: ResVReg)
4846 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4847 .addUse(RegNo: I.getOperand(i: 2).getReg())
4848 .constrainAllUses(TII, TRI, RBI);
4849 if (!STI.isShader()) {
4850 unsigned Alignment = I.getOperand(i: 3).getImm();
4851 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::Alignment, DecArgs: {Alignment});
4852 }
4853 return Res;
4854}
4855
4856bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
4857 const SPIRVType *ResType,
4858 MachineInstr &I) const {
4859 // Change order of instructions if needed: all OpVariable instructions in a
4860 // function must be the first instructions in the first block
4861 auto It = getOpVariableMBBIt(I);
4862 bool Res = BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(),
4863 MCID: TII.get(Opcode: SPIRV::OpVariable))
4864 .addDef(RegNo: ResVReg)
4865 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4866 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
4867 .constrainAllUses(TII, TRI, RBI);
4868 if (!STI.isShader()) {
4869 unsigned Alignment = I.getOperand(i: 2).getImm();
4870 buildOpDecorate(Reg: ResVReg, I&: *It, TII, Dec: SPIRV::Decoration::Alignment,
4871 DecArgs: {Alignment});
4872 }
4873 return Res;
4874}
4875
4876bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
4877 // InstructionSelector walks backwards through the instructions. We can use
4878 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
4879 // first, so can generate an OpBranchConditional here. If there is no
4880 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
4881 const MachineInstr *PrevI = I.getPrevNode();
4882 MachineBasicBlock &MBB = *I.getParent();
4883 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
4884 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
4885 .addUse(RegNo: PrevI->getOperand(i: 0).getReg())
4886 .addMBB(MBB: PrevI->getOperand(i: 1).getMBB())
4887 .addMBB(MBB: I.getOperand(i: 0).getMBB())
4888 .constrainAllUses(TII, TRI, RBI);
4889 }
4890 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranch))
4891 .addMBB(MBB: I.getOperand(i: 0).getMBB())
4892 .constrainAllUses(TII, TRI, RBI);
4893}
4894
4895bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
4896 // InstructionSelector walks backwards through the instructions. For an
4897 // explicit conditional branch with no fallthrough, we use both a G_BR and a
4898 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
4899 // generate the OpBranchConditional in selectBranch above.
4900 //
4901 // If an OpBranchConditional has been generated, we simply return, as the work
4902 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
4903 // implicit fallthrough to the next basic block, so we need to create an
4904 // OpBranchConditional with an explicit "false" argument pointing to the next
4905 // basic block that LLVM would fall through to.
4906 const MachineInstr *NextI = I.getNextNode();
4907 // Check if this has already been successfully selected.
4908 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
4909 return true;
4910 // Must be relying on implicit block fallthrough, so generate an
4911 // OpBranchConditional with the "next" basic block as the "false" target.
4912 MachineBasicBlock &MBB = *I.getParent();
4913 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
4914 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(N: NextMBBNum);
4915 return BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
4916 .addUse(RegNo: I.getOperand(i: 0).getReg())
4917 .addMBB(MBB: I.getOperand(i: 1).getMBB())
4918 .addMBB(MBB: NextMBB)
4919 .constrainAllUses(TII, TRI, RBI);
4920}
4921
4922bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
4923 const SPIRVType *ResType,
4924 MachineInstr &I) const {
4925 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpPhi))
4926 .addDef(RegNo: ResVReg)
4927 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
4928 const unsigned NumOps = I.getNumOperands();
4929 for (unsigned i = 1; i < NumOps; i += 2) {
4930 MIB.addUse(RegNo: I.getOperand(i: i + 0).getReg());
4931 MIB.addMBB(MBB: I.getOperand(i: i + 1).getMBB());
4932 }
4933 bool Res = MIB.constrainAllUses(TII, TRI, RBI);
4934 MIB->setDesc(TII.get(Opcode: TargetOpcode::PHI));
4935 MIB->removeOperand(OpNo: 1);
4936 return Res;
4937}
4938
4939bool SPIRVInstructionSelector::selectGlobalValue(
4940 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
4941 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
4942 MachineIRBuilder MIRBuilder(I);
4943 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
4944 Type *GVType = toTypedPointer(Ty: GR.getDeducedGlobalValueType(Global: GV));
4945
4946 std::string GlobalIdent;
4947 if (!GV->hasName()) {
4948 unsigned &ID = UnnamedGlobalIDs[GV];
4949 if (ID == 0)
4950 ID = UnnamedGlobalIDs.size();
4951 GlobalIdent = "__unnamed_" + Twine(ID).str();
4952 } else {
4953 GlobalIdent = GV->getName();
4954 }
4955
4956 // Behaviour of functions as operands depends on availability of the
4957 // corresponding extension (SPV_INTEL_function_pointers):
4958 // - If there is an extension to operate with functions as operands:
4959 // We create a proper constant operand and evaluate a correct type for a
4960 // function pointer.
4961 // - Without the required extension:
4962 // We have functions as operands in tests with blocks of instruction e.g. in
4963 // transcoding/global_block.ll. These operands are not used and should be
4964 // substituted by zero constants. Their type is expected to be always
4965 // OpTypePointer Function %uchar.
4966 if (isa<Function>(Val: GV)) {
4967 const Constant *ConstVal = GV;
4968 MachineBasicBlock &BB = *I.getParent();
4969 Register NewReg = GR.find(V: ConstVal, MF: GR.CurMF);
4970 if (!NewReg.isValid()) {
4971 Register NewReg = ResVReg;
4972 const Function *GVFun =
4973 STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
4974 ? dyn_cast<Function>(Val: GV)
4975 : nullptr;
4976 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
4977 BaseType: GVType, I,
4978 SC: GVFun ? SPIRV::StorageClass::CodeSectionINTEL
4979 : addressSpaceToStorageClass(AddrSpace: GV->getAddressSpace(), STI));
4980 if (GVFun) {
4981 // References to a function via function pointers generate virtual
4982 // registers without a definition. We will resolve it later, during
4983 // module analysis stage.
4984 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
4985 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4986 Register FuncVReg =
4987 MRI->createGenericVirtualRegister(Ty: GR.getRegType(SpvType: ResType));
4988 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::pIDRegClass);
4989 MachineInstrBuilder MIB1 =
4990 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
4991 .addDef(RegNo: FuncVReg)
4992 .addUse(RegNo: ResTypeReg);
4993 MachineInstrBuilder MIB2 =
4994 BuildMI(BB, I, MIMD: I.getDebugLoc(),
4995 MCID: TII.get(Opcode: SPIRV::OpConstantFunctionPointerINTEL))
4996 .addDef(RegNo: NewReg)
4997 .addUse(RegNo: ResTypeReg)
4998 .addUse(RegNo: FuncVReg);
4999 GR.add(V: ConstVal, MI: MIB2);
5000 // mapping the function pointer to the used Function
5001 GR.recordFunctionPointer(MO: &MIB2.getInstr()->getOperand(i: 2), F: GVFun);
5002 return MIB1.constrainAllUses(TII, TRI, RBI) &&
5003 MIB2.constrainAllUses(TII, TRI, RBI);
5004 }
5005 MachineInstrBuilder MIB3 =
5006 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
5007 .addDef(RegNo: NewReg)
5008 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
5009 GR.add(V: ConstVal, MI: MIB3);
5010 return MIB3.constrainAllUses(TII, TRI, RBI);
5011 }
5012 assert(NewReg != ResVReg);
5013 return BuildCOPY(DestReg: ResVReg, SrcReg: NewReg, I);
5014 }
5015 auto GlobalVar = cast<GlobalVariable>(Val: GV);
5016 assert(GlobalVar->getName() != "llvm.global.annotations");
5017
5018 // Skip empty declaration for GVs with initializers till we get the decl with
5019 // passed initializer.
5020 if (hasInitializer(GV: GlobalVar) && !Init)
5021 return true;
5022
5023 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
5024 getSpirvLinkageTypeFor(ST: STI, GV: *GV);
5025
5026 const unsigned AddrSpace = GV->getAddressSpace();
5027 SPIRV::StorageClass::StorageClass StorageClass =
5028 addressSpaceToStorageClass(AddrSpace, STI);
5029 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(BaseType: GVType, I, SC: StorageClass);
5030 Register Reg = GR.buildGlobalVariable(
5031 Reg: ResVReg, BaseType: ResType, Name: GlobalIdent, GV, Storage: StorageClass, Init,
5032 IsConst: GlobalVar->isConstant(), LinkageType: LnkType, MIRBuilder, IsInstSelector: true);
5033 // TODO: For AMDGCN, we pipe externally_initialized through via
5034 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
5035 // reverse translation. We should remove this once SPIR-V gains the ability to
5036 // express the concept.
5037 if (GlobalVar->isExternallyInitialized() &&
5038 STI.getTargetTriple().getVendor() == Triple::AMD) {
5039 constexpr unsigned ReadWriteINTEL = 3u;
5040 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::HostAccessINTEL,
5041 DecArgs: {ReadWriteINTEL});
5042 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
5043 addStringImm(Str: GV->getName(), MIB);
5044 }
5045 return Reg.isValid();
5046}
5047
5048bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
5049 const SPIRVType *ResType,
5050 MachineInstr &I) const {
5051 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5052 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log10);
5053 }
5054
5055 // There is no log10 instruction in the GLSL Extended Instruction set, so it
5056 // is implemented as:
5057 // log10(x) = log2(x) * (1 / log2(10))
5058 // = log2(x) * 0.30103
5059
5060 MachineIRBuilder MIRBuilder(I);
5061 MachineBasicBlock &BB = *I.getParent();
5062
5063 // Build log2(x).
5064 Register VarReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5065 bool Result =
5066 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5067 .addDef(RegNo: VarReg)
5068 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5069 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
5070 .addImm(Val: GL::Log2)
5071 .add(MO: I.getOperand(i: 1))
5072 .constrainAllUses(TII, TRI, RBI);
5073
5074 // Build 0.30103.
5075 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
5076 ResType->getOpcode() == SPIRV::OpTypeFloat);
5077 // TODO: Add matrix implementation once supported by the HLSL frontend.
5078 const SPIRVType *SpirvScalarType =
5079 ResType->getOpcode() == SPIRV::OpTypeVector
5080 ? GR.getSPIRVTypeForVReg(VReg: ResType->getOperand(i: 1).getReg())
5081 : ResType;
5082 Register ScaleReg =
5083 GR.buildConstantFP(Val: APFloat(0.30103f), MIRBuilder, SpvType: SpirvScalarType);
5084
5085 // Multiply log2(x) by 0.30103 to get log10(x) result.
5086 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
5087 ? SPIRV::OpVectorTimesScalar
5088 : SPIRV::OpFMulS;
5089 return Result && BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
5090 .addDef(RegNo: ResVReg)
5091 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5092 .addUse(RegNo: VarReg)
5093 .addUse(RegNo: ScaleReg)
5094 .constrainAllUses(TII, TRI, RBI);
5095}
5096
5097bool SPIRVInstructionSelector::selectModf(Register ResVReg,
5098 const SPIRVType *ResType,
5099 MachineInstr &I) const {
5100 // llvm.modf has a single arg --the number to be decomposed-- and returns a
5101 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
5102 // number to be decomposed and a pointer--, returns the fractional part and
5103 // the integral part is stored in the pointer argument. Therefore, we can't
5104 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
5105 // scaffolding to make it work. The idea is to create an alloca instruction
5106 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
5107 // from this ptr to place it in the struct. llvm.modf returns the fractional
5108 // part as the first element of the result, and the integral part as the
5109 // second element of the result.
5110
5111 // At this point, the return type is not a struct anymore, but rather two
5112 // independent elements of SPIRVResType. We can get each independent element
5113 // from I.getDefs() or I.getOperands().
5114 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5115 MachineIRBuilder MIRBuilder(I);
5116 // Get pointer type for alloca variable.
5117 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
5118 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Function);
5119 // Create new register for the pointer type of alloca variable.
5120 Register PtrTyReg =
5121 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5122 MIRBuilder.getMRI()->setType(
5123 VReg: PtrTyReg,
5124 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Function),
5125 SizeInBits: GR.getPointerSize()));
5126
5127 // Assign SPIR-V type of the pointer type of the alloca variable to the
5128 // new register.
5129 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: PtrTyReg, MF: MIRBuilder.getMF());
5130 MachineBasicBlock &EntryBB = I.getMF()->front();
5131 MachineBasicBlock::iterator VarPos =
5132 getFirstValidInstructionInsertPoint(BB&: EntryBB);
5133 auto AllocaMIB =
5134 BuildMI(BB&: EntryBB, I: VarPos, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
5135 .addDef(RegNo: PtrTyReg)
5136 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PtrType))
5137 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function));
5138 Register Variable = AllocaMIB->getOperand(i: 0).getReg();
5139
5140 MachineBasicBlock &BB = *I.getParent();
5141 // Create the OpenCLLIB::modf instruction.
5142 auto MIB =
5143 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5144 .addDef(RegNo: ResVReg)
5145 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5146 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
5147 .addImm(Val: CL::modf)
5148 .setMIFlags(I.getFlags())
5149 .add(MO: I.getOperand(i: I.getNumExplicitDefs())) // Floating point value.
5150 .addUse(RegNo: Variable); // Pointer to integral part.
5151 // Assign the integral part stored in the ptr to the second element of the
5152 // result.
5153 Register IntegralPartReg = I.getOperand(i: 1).getReg();
5154 if (IntegralPartReg.isValid()) {
5155 // Load the value from the pointer to integral part.
5156 auto LoadMIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5157 .addDef(RegNo: IntegralPartReg)
5158 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5159 .addUse(RegNo: Variable);
5160 return LoadMIB.constrainAllUses(TII, TRI, RBI);
5161 }
5162
5163 return MIB.constrainAllUses(TII, TRI, RBI);
5164 } else if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
5165 assert(false && "GLSL::Modf is deprecated.");
5166 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
5167 return false;
5168 }
5169 return false;
5170}
5171
5172// Generate the instructions to load 3-element vector builtin input
5173// IDs/Indices.
5174// Like: GlobalInvocationId, LocalInvocationId, etc....
5175
5176bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
5177 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5178 const SPIRVType *ResType, MachineInstr &I) const {
5179 MachineIRBuilder MIRBuilder(I);
5180 const SPIRVType *Vec3Ty =
5181 GR.getOrCreateSPIRVVectorType(BaseType: ResType, NumElements: 3, MIRBuilder, EmitIR: false);
5182 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
5183 BaseType: Vec3Ty, MIRBuilder, SC: SPIRV::StorageClass::Input);
5184
5185 // Create new register for the input ID builtin variable.
5186 Register NewRegister =
5187 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5188 MIRBuilder.getMRI()->setType(VReg: NewRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5189 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5190
5191 // Build global variable with the necessary decorations for the input ID
5192 // builtin variable.
5193 Register Variable = GR.buildGlobalVariable(
5194 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5195 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5196 IsInstSelector: false);
5197
5198 // Create new register for loading value.
5199 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5200 Register LoadedRegister = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5201 MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5202 GR.assignSPIRVTypeToVReg(Type: Vec3Ty, VReg: LoadedRegister, MF: MIRBuilder.getMF());
5203
5204 // Load v3uint value from the global variable.
5205 bool Result =
5206 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5207 .addDef(RegNo: LoadedRegister)
5208 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: Vec3Ty))
5209 .addUse(RegNo: Variable);
5210
5211 // Get the input ID index. Expecting operand is a constant immediate value,
5212 // wrapped in a type assignment.
5213 assert(I.getOperand(2).isReg());
5214 const uint32_t ThreadId = foldImm(MO: I.getOperand(i: 2), MRI);
5215
5216 // Extract the input ID from the loaded vector value.
5217 MachineBasicBlock &BB = *I.getParent();
5218 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
5219 .addDef(RegNo: ResVReg)
5220 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5221 .addUse(RegNo: LoadedRegister)
5222 .addImm(Val: ThreadId);
5223 return Result && MIB.constrainAllUses(TII, TRI, RBI);
5224}
5225
5226// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
5227// Like LocalInvocationIndex
5228bool SPIRVInstructionSelector::loadBuiltinInputID(
5229 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5230 const SPIRVType *ResType, MachineInstr &I) const {
5231 MachineIRBuilder MIRBuilder(I);
5232 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
5233 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Input);
5234
5235 // Create new register for the input ID builtin variable.
5236 Register NewRegister =
5237 MIRBuilder.getMRI()->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PtrType));
5238 MIRBuilder.getMRI()->setType(
5239 VReg: NewRegister,
5240 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Input),
5241 SizeInBits: GR.getPointerSize()));
5242 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5243
5244 // Build global variable with the necessary decorations for the input ID
5245 // builtin variable.
5246 Register Variable = GR.buildGlobalVariable(
5247 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5248 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5249 IsInstSelector: false);
5250
5251 // Load uint value from the global variable.
5252 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5253 .addDef(RegNo: ResVReg)
5254 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5255 .addUse(RegNo: Variable);
5256
5257 return MIB.constrainAllUses(TII, TRI, RBI);
5258}
5259
5260SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type,
5261 MachineInstr &I) const {
5262 MachineIRBuilder MIRBuilder(I);
5263 if (Type->getOpcode() != SPIRV::OpTypeVector)
5264 return GR.getOrCreateSPIRVVectorType(BaseType: Type, NumElements: 4, MIRBuilder, EmitIR: false);
5265
5266 uint64_t VectorSize = Type->getOperand(i: 2).getImm();
5267 if (VectorSize == 4)
5268 return Type;
5269
5270 Register ScalarTypeReg = Type->getOperand(i: 1).getReg();
5271 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(VReg: ScalarTypeReg);
5272 return GR.getOrCreateSPIRVVectorType(BaseType: ScalarType, NumElements: 4, MIRBuilder, EmitIR: false);
5273}
5274
5275bool SPIRVInstructionSelector::loadHandleBeforePosition(
5276 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef,
5277 MachineInstr &Pos) const {
5278
5279 assert(HandleDef.getIntrinsicID() ==
5280 Intrinsic::spv_resource_handlefrombinding);
5281 uint32_t Set = foldImm(MO: HandleDef.getOperand(i: 2), MRI);
5282 uint32_t Binding = foldImm(MO: HandleDef.getOperand(i: 3), MRI);
5283 uint32_t ArraySize = foldImm(MO: HandleDef.getOperand(i: 4), MRI);
5284 Register IndexReg = HandleDef.getOperand(i: 5).getReg();
5285 std::string Name =
5286 getStringValueFromReg(Reg: HandleDef.getOperand(i: 6).getReg(), MRI&: *MRI);
5287
5288 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
5289 MachineIRBuilder MIRBuilder(HandleDef);
5290 SPIRVType *VarType = ResType;
5291 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
5292
5293 if (IsStructuredBuffer) {
5294 VarType = GR.getPointeeType(PtrType: ResType);
5295 SC = GR.getPointerStorageClass(Type: ResType);
5296 }
5297
5298 Register VarReg = buildPointerToResource(SpirvResType: VarType, SC, Set, Binding, ArraySize,
5299 IndexReg, Name, MIRBuilder);
5300
5301 // The handle for the buffer is the pointer to the resource. For an image, the
5302 // handle is the image object. So images get an extra load.
5303 uint32_t LoadOpcode =
5304 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
5305 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: HandleReg, MF: *Pos.getMF());
5306 return BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: HandleDef.getDebugLoc(),
5307 MCID: TII.get(Opcode: LoadOpcode))
5308 .addDef(RegNo: HandleReg)
5309 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5310 .addUse(RegNo: VarReg)
5311 .constrainAllUses(TII, TRI, RBI);
5312}
5313
5314void SPIRVInstructionSelector::errorIfInstrOutsideShader(
5315 MachineInstr &I) const {
5316 if (!STI.isShader()) {
5317 std::string DiagMsg;
5318 raw_string_ostream OS(DiagMsg);
5319 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
5320 DiagMsg += " is only supported in shaders.\n";
5321 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
5322 }
5323}
5324
5325namespace llvm {
5326InstructionSelector *
5327createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
5328 const SPIRVSubtarget &Subtarget,
5329 const RegisterBankInfo &RBI) {
5330 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
5331}
5332} // namespace llvm
5333