1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVUtils.h"
23#include "llvm/ADT/APFloat.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/Register.h"
31#include "llvm/CodeGen/TargetOpcodes.h"
32#include "llvm/IR/IntrinsicsSPIRV.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35
36#define DEBUG_TYPE "spirv-isel"
37
38using namespace llvm;
39namespace CL = SPIRV::OpenCLExtInst;
40namespace GL = SPIRV::GLSLExtInst;
41
42using ExtInstList =
43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
44
45namespace {
46
47struct ImageOperands {
48 std::optional<Register> Bias;
49 std::optional<Register> Offset;
50 std::optional<Register> MinLod;
51 std::optional<Register> GradX;
52 std::optional<Register> GradY;
53 std::optional<Register> Lod;
54 std::optional<Register> Compare;
55};
56
57llvm::SPIRV::SelectionControl::SelectionControl
58getSelectionOperandForImm(int Imm) {
59 if (Imm == 2)
60 return SPIRV::SelectionControl::Flatten;
61 if (Imm == 1)
62 return SPIRV::SelectionControl::DontFlatten;
63 if (Imm == 0)
64 return SPIRV::SelectionControl::None;
65 llvm_unreachable("Invalid immediate");
66}
67
68#define GET_GLOBALISEL_PREDICATE_BITSET
69#include "SPIRVGenGlobalISel.inc"
70#undef GET_GLOBALISEL_PREDICATE_BITSET
71
72class SPIRVInstructionSelector : public InstructionSelector {
73 const SPIRVSubtarget &STI;
74 const SPIRVInstrInfo &TII;
75 const SPIRVRegisterInfo &TRI;
76 const RegisterBankInfo &RBI;
77 SPIRVGlobalRegistry &GR;
78 MachineRegisterInfo *MRI;
79 MachineFunction *HasVRegsReset = nullptr;
80
81 /// We need to keep track of the number we give to anonymous global values to
82 /// generate the same name every time when this is needed.
83 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
84 SmallPtrSet<MachineInstr *, 8> DeadMIs;
85
86public:
87 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
88 const SPIRVSubtarget &ST,
89 const RegisterBankInfo &RBI);
90 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
91 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
92 BlockFrequencyInfo *BFI) override;
93 // Common selection code. Instruction-specific selection occurs in spvSelect.
94 bool select(MachineInstr &I) override;
95 static const char *getName() { return DEBUG_TYPE; }
96
97#define GET_GLOBALISEL_PREDICATES_DECL
98#include "SPIRVGenGlobalISel.inc"
99#undef GET_GLOBALISEL_PREDICATES_DECL
100
101#define GET_GLOBALISEL_TEMPORARIES_DECL
102#include "SPIRVGenGlobalISel.inc"
103#undef GET_GLOBALISEL_TEMPORARIES_DECL
104
105private:
106 void resetVRegsType(MachineFunction &MF);
107 void removeDeadInstruction(MachineInstr &MI) const;
108 void removeOpNamesForDeadMI(MachineInstr &MI) const;
109
110 // tblgen-erated 'select' implementation, used as the initial selector for
111 // the patterns that don't require complex C++.
112 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
113
114 // All instruction-specific selection that didn't happen in "select()".
115 // Is basically a large Switch/Case delegating to all other select method.
116 bool spvSelect(Register ResVReg, SPIRVTypeInst ResType,
117 MachineInstr &I) const;
118
119 bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType,
120 MachineInstr &I, bool IsSigned) const;
121
122 bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType,
123 MachineInstr &I) const;
124
125 bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType,
126 MachineInstr &I, unsigned ExtendOpcode,
127 unsigned BitSetOpcode) const;
128
129 bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType,
130 MachineInstr &I, Register SrcReg,
131 unsigned BitSetOpcode) const;
132
133 bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType,
134 MachineInstr &I, Register SrcReg,
135 unsigned BitSetOpcode, bool SwapPrimarySide) const;
136
137 bool selectFirstBitSet64Overflow(Register ResVReg, SPIRVTypeInst ResType,
138 MachineInstr &I, Register SrcReg,
139 unsigned BitSetOpcode,
140 bool SwapPrimarySide) const;
141
142 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
143 const MachineInstr *Init = nullptr) const;
144
145 bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType,
146 MachineInstr &I, std::vector<Register> SrcRegs,
147 unsigned Opcode) const;
148
149 bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
150 unsigned Opcode) const;
151
152 bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType,
153 MachineInstr &I) const;
154
155 bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
156 MachineInstr &I) const;
157 bool selectStore(MachineInstr &I) const;
158
159 bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
160 MachineInstr &I) const;
161 bool selectStackRestore(MachineInstr &I) const;
162
163 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
164 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
165 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
166 bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
167
168 bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
169 unsigned NewOpcode, unsigned NegateOpcode = 0) const;
170
171 bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType,
172 MachineInstr &I) const;
173
174 bool selectFence(MachineInstr &I) const;
175
176 bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType,
177 MachineInstr &I) const;
178
179 bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
180 unsigned OpType) const;
181
182 bool selectAll(Register ResVReg, SPIRVTypeInst ResType,
183 MachineInstr &I) const;
184
185 bool selectAny(Register ResVReg, SPIRVTypeInst ResType,
186 MachineInstr &I) const;
187
188 bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType,
189 MachineInstr &I) const;
190
191 bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType,
192 MachineInstr &I) const;
193 bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType,
194 MachineInstr &I) const;
195
196 bool selectCmp(Register ResVReg, SPIRVTypeInst ResType,
197 unsigned comparisonOpcode, MachineInstr &I) const;
198 bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType,
199 MachineInstr &I) const;
200
201 bool selectICmp(Register ResVReg, SPIRVTypeInst ResType,
202 MachineInstr &I) const;
203 bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType,
204 MachineInstr &I) const;
205
206 bool selectSign(Register ResVReg, SPIRVTypeInst ResType,
207 MachineInstr &I) const;
208
209 bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType,
210 MachineInstr &I) const;
211
212 bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType,
213 MachineInstr &I, unsigned Opcode) const;
214 bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType,
215 MachineInstr &I) const;
216
217 bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType,
218 MachineInstr &I, bool Signed) const;
219
220 bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType,
221 MachineInstr &I) const;
222
223 bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType,
224 MachineInstr &I) const;
225
226 bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType,
227 MachineInstr &I) const;
228
229 template <bool Signed>
230 bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType,
231 MachineInstr &I) const;
232 template <bool Signed>
233 bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType,
234 MachineInstr &I) const;
235
236 bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType,
237 MachineInstr &I) const;
238
239 template <typename PickOpcodeFn>
240 bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType,
241 MachineInstr &I, bool IsUnsigned,
242 PickOpcodeFn &&PickOpcode) const;
243
244 bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType,
245 MachineInstr &I, bool IsUnsigned) const;
246
247 bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType,
248 MachineInstr &I, bool IsUnsigned) const;
249
250 bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType,
251 MachineInstr &I) const;
252
253 template <typename PickOpcodeFn>
254 bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType,
255 MachineInstr &I, bool IsUnsigned,
256 PickOpcodeFn &&PickOpcode) const;
257
258 bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType,
259 MachineInstr &I) const;
260
261 bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
262 MachineInstr &I) const;
263
264 bool selectConst(Register ResVReg, SPIRVTypeInst ResType,
265 MachineInstr &I) const;
266
267 bool selectSelect(Register ResVReg, SPIRVTypeInst ResType,
268 MachineInstr &I) const;
269 bool selectSelectDefaultArgs(Register ResVReg, SPIRVTypeInst ResType,
270 MachineInstr &I, bool IsSigned) const;
271 bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
272 bool IsSigned, unsigned Opcode) const;
273 bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
274 bool IsSigned) const;
275
276 bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType,
277 MachineInstr &I) const;
278
279 bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
280 bool IsSigned) const;
281
282 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
283 SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const;
284
285 bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType,
286 MachineInstr &I) const;
287 bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType,
288 MachineInstr &I) const;
289 bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType,
290 MachineInstr &I) const;
291 bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType,
292 MachineInstr &I) const;
293 bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType,
294 MachineInstr &I) const;
295 bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType,
296 MachineInstr &I) const;
297 bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType,
298 MachineInstr &I) const;
299 bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
300 MachineInstr &I) const;
301
302 bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
303 MachineInstr &I) const;
304 bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
305 MachineInstr &I) const;
306
307 bool selectBranch(MachineInstr &I) const;
308 bool selectBranchCond(MachineInstr &I) const;
309
310 bool selectPhi(Register ResVReg, SPIRVTypeInst ResType,
311 MachineInstr &I) const;
312
313 bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I,
314 GL::GLSLExtInst GLInst) const;
315 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
316 CL::OpenCLExtInst CLInst) const;
317 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
318 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst) const;
319 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
320 const ExtInstList &ExtInsts) const;
321 bool selectExtInstForLRound(Register ResVReg, SPIRVTypeInst ResType,
322 MachineInstr &I, CL::OpenCLExtInst CLInst,
323 GL::GLSLExtInst GLInst) const;
324 bool selectExtInstForLRound(Register ResVReg, SPIRVTypeInst ResType,
325 MachineInstr &I,
326 const ExtInstList &ExtInsts) const;
327
328 bool selectLog10(Register ResVReg, SPIRVTypeInst ResType,
329 MachineInstr &I) const;
330
331 bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType,
332 MachineInstr &I) const;
333
334 bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType,
335 MachineInstr &I, unsigned Opcode) const;
336
337 bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType,
338 MachineInstr &I) const;
339
340 bool selectUnmergeValues(MachineInstr &I) const;
341
342 bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
343 MachineInstr &I) const;
344
345 bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
346 MachineInstr &I) const;
347
348 bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
349 MachineInstr &I) const;
350 bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
351 MachineInstr &I) const;
352 bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
353 MachineInstr &I) const;
354 bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
355 MachineInstr &I) const;
356 bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
357 MachineInstr &I) const;
358 bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
359 MachineInstr &I) const;
360 bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg,
361 SPIRVTypeInst ResType,
362 MachineInstr &I) const;
363 bool selectImageWriteIntrinsic(MachineInstr &I) const;
364 bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
365 MachineInstr &I) const;
366 bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
367 MachineInstr &I) const;
368 bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType,
369 MachineInstr &I) const;
370 bool selectModf(Register ResVReg, SPIRVTypeInst ResType,
371 MachineInstr &I) const;
372 bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType,
373 MachineInstr &I) const;
374 bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType,
375 MachineInstr &I) const;
376 bool selectSincos(Register ResVReg, SPIRVTypeInst ResType,
377 MachineInstr &I) const;
378 bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType,
379 MachineInstr &I, const unsigned DPdOpCode) const;
380 // Utilities
381 Register buildI32Constant(uint32_t Val, MachineInstr &I,
382 SPIRVTypeInst ResType = nullptr) const;
383
384 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const;
385 bool isScalarOrVectorIntConstantZero(Register Reg) const;
386 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const;
387 Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
388 MachineInstr &I) const;
389 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
390
391 bool wrapIntoSpecConstantOp(MachineInstr &I,
392 SmallVector<Register> &CompositeArgs) const;
393
394 Register getUcharPtrTypeReg(MachineInstr &I,
395 SPIRV::StorageClass::StorageClass SC) const;
396 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
397 Register Src, Register DestType,
398 uint32_t Opcode) const;
399 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
400 SPIRVTypeInst SrcPtrTy) const;
401 Register buildPointerToResource(SPIRVTypeInst ResType,
402 SPIRV::StorageClass::StorageClass SC,
403 uint32_t Set, uint32_t Binding,
404 uint32_t ArraySize, Register IndexReg,
405 StringRef Name,
406 MachineIRBuilder MIRBuilder) const;
407 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const;
408 bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType,
409 Register &ReadReg, MachineInstr &InsertionPoint) const;
410 bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType,
411 Register ImageReg, Register IdxReg,
412 DebugLoc Loc, MachineInstr &Pos) const;
413 bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType,
414 Register ImageReg, Register SamplerReg,
415 Register CoordinateReg, const ImageOperands &ImOps,
416 DebugLoc Loc, MachineInstr &I) const;
417 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
418 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
419 Register ResVReg, SPIRVTypeInst ResType,
420 MachineInstr &I) const;
421 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
422 Register ResVReg, SPIRVTypeInst ResType,
423 MachineInstr &I) const;
424 bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType,
425 GIntrinsic &HandleDef, MachineInstr &Pos) const;
426 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
427 void errorIfInstrOutsideShader(MachineInstr &I) const;
428};
429
430bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
431 const TargetExtType *TET = cast<TargetExtType>(Val: HandleType);
432 if (TET->getTargetExtName() == "spirv.Image") {
433 return false;
434 }
435 assert(TET->getTargetExtName() == "spirv.SignedImage");
436 return TET->getTypeParameter(i: 0)->isIntegerTy();
437}
438} // end anonymous namespace
439
440#define GET_GLOBALISEL_IMPL
441#include "SPIRVGenGlobalISel.inc"
442#undef GET_GLOBALISEL_IMPL
443
444SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
445 const SPIRVSubtarget &ST,
446 const RegisterBankInfo &RBI)
447 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
448 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
449 MRI(nullptr),
450#define GET_GLOBALISEL_PREDICATES_INIT
451#include "SPIRVGenGlobalISel.inc"
452#undef GET_GLOBALISEL_PREDICATES_INIT
453#define GET_GLOBALISEL_TEMPORARIES_INIT
454#include "SPIRVGenGlobalISel.inc"
455#undef GET_GLOBALISEL_TEMPORARIES_INIT
456{
457}
458
459void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
460 GISelValueTracking *VT,
461 CodeGenCoverage *CoverageInfo,
462 ProfileSummaryInfo *PSI,
463 BlockFrequencyInfo *BFI) {
464 MRI = &MF.getRegInfo();
465 GR.setCurrentFunc(MF);
466 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
467}
468
469// Ensure that register classes correspond to pattern matching rules.
470void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
471 if (HasVRegsReset == &MF)
472 return;
473 HasVRegsReset = &MF;
474
475 MachineRegisterInfo &MRI = MF.getRegInfo();
476 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
477 Register Reg = Register::index2VirtReg(Index: I);
478 LLT RegType = MRI.getType(Reg);
479 if (RegType.isScalar())
480 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: 64));
481 else if (RegType.isPointer())
482 MRI.setType(VReg: Reg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
483 else if (RegType.isVector())
484 MRI.setType(VReg: Reg, Ty: LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::scalar(SizeInBits: 64)));
485 }
486 for (const auto &MBB : MF) {
487 for (const auto &MI : MBB) {
488 if (isPreISelGenericOpcode(Opcode: MI.getOpcode()))
489 GR.erase(MI: &MI);
490 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
491 continue;
492
493 Register DstReg = MI.getOperand(i: 0).getReg();
494 LLT DstType = MRI.getType(Reg: DstReg);
495 Register SrcReg = MI.getOperand(i: 1).getReg();
496 LLT SrcType = MRI.getType(Reg: SrcReg);
497 if (DstType != SrcType)
498 MRI.setType(VReg: DstReg, Ty: MRI.getType(Reg: SrcReg));
499
500 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(Reg: DstReg);
501 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(Reg: SrcReg);
502 if (DstRC != SrcRC && SrcRC)
503 MRI.setRegClass(Reg: DstReg, RC: SrcRC);
504 }
505 }
506}
507
508// Return true if the MachineInstr represents a constant register
509static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef) {
510
511 SmallVector<MachineInstr *> Stack = {OpDef};
512 SmallPtrSet<MachineInstr *, 4> Visited;
513
514 while (!Stack.empty()) {
515 MachineInstr *MI = Stack.pop_back_val();
516 MI = passCopy(Def: MI, MRI);
517 if (!Visited.insert(Ptr: MI).second)
518 continue;
519 switch (MI->getOpcode()) {
520 case TargetOpcode::G_INTRINSIC:
521 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
522 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
523 if (cast<GIntrinsic>(Val&: *OpDef).getIntrinsicID() !=
524 Intrinsic::spv_const_composite)
525 return false;
526 continue;
527 case TargetOpcode::G_BUILD_VECTOR:
528 case TargetOpcode::G_SPLAT_VECTOR:
529 for (unsigned i = OpDef->getNumExplicitDefs();
530 i < OpDef->getNumOperands(); i++) {
531 if (!OpDef->getOperand(i).isReg())
532 continue;
533 MachineInstr *OpNestedDef =
534 MRI->getVRegDef(Reg: OpDef->getOperand(i).getReg());
535 Stack.push_back(Elt: OpNestedDef);
536 }
537 continue;
538 case TargetOpcode::G_CONSTANT:
539 case TargetOpcode::G_FCONSTANT:
540 case TargetOpcode::G_IMPLICIT_DEF:
541 case SPIRV::OpConstantTrue:
542 case SPIRV::OpConstantFalse:
543 case SPIRV::OpConstantI:
544 case SPIRV::OpConstantF:
545 case SPIRV::OpConstantComposite:
546 case SPIRV::OpConstantCompositeContinuedINTEL:
547 case SPIRV::OpConstantSampler:
548 case SPIRV::OpConstantNull:
549 case SPIRV::OpUndef:
550 case SPIRV::OpConstantFunctionPointerINTEL:
551 continue;
552 default:
553 return false;
554 }
555 }
556 return true;
557}
558
559// Return true if the virtual register represents a constant
560static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
561 if (MachineInstr *OpDef = MRI->getVRegDef(Reg: OpReg))
562 return isConstReg(MRI, OpDef);
563 return false;
564}
565
566// TODO(168736): We should make this either a flag in tabelgen
567// or reduce our dependence on the global registry, so we can remove this
568// function. It can easily be missed when new intrinsics are added.
569
570// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
571// definition because they are referenced in the global registry. This is a list
572// of intrinsics that have no side effects other than their references in the
573// global registry.
574static bool intrinsicHasSideEffects(Intrinsic::ID ID) {
575 switch (ID) {
576 // This is not an exhaustive list and may need to be updated.
577 case Intrinsic::spv_all:
578 case Intrinsic::spv_alloca:
579 case Intrinsic::spv_any:
580 case Intrinsic::spv_bitcast:
581 case Intrinsic::spv_const_composite:
582 case Intrinsic::spv_cross:
583 case Intrinsic::spv_degrees:
584 case Intrinsic::spv_distance:
585 case Intrinsic::spv_extractelt:
586 case Intrinsic::spv_extractv:
587 case Intrinsic::spv_faceforward:
588 case Intrinsic::spv_fdot:
589 case Intrinsic::spv_firstbitlow:
590 case Intrinsic::spv_firstbitshigh:
591 case Intrinsic::spv_firstbituhigh:
592 case Intrinsic::spv_frac:
593 case Intrinsic::spv_gep:
594 case Intrinsic::spv_global_offset:
595 case Intrinsic::spv_global_size:
596 case Intrinsic::spv_group_id:
597 case Intrinsic::spv_insertelt:
598 case Intrinsic::spv_insertv:
599 case Intrinsic::spv_isinf:
600 case Intrinsic::spv_isnan:
601 case Intrinsic::spv_lerp:
602 case Intrinsic::spv_length:
603 case Intrinsic::spv_normalize:
604 case Intrinsic::spv_num_subgroups:
605 case Intrinsic::spv_num_workgroups:
606 case Intrinsic::spv_ptrcast:
607 case Intrinsic::spv_radians:
608 case Intrinsic::spv_reflect:
609 case Intrinsic::spv_refract:
610 case Intrinsic::spv_resource_getpointer:
611 case Intrinsic::spv_resource_handlefrombinding:
612 case Intrinsic::spv_resource_handlefromimplicitbinding:
613 case Intrinsic::spv_resource_nonuniformindex:
614 case Intrinsic::spv_resource_sample:
615 case Intrinsic::spv_rsqrt:
616 case Intrinsic::spv_saturate:
617 case Intrinsic::spv_sdot:
618 case Intrinsic::spv_sign:
619 case Intrinsic::spv_smoothstep:
620 case Intrinsic::spv_step:
621 case Intrinsic::spv_subgroup_id:
622 case Intrinsic::spv_subgroup_local_invocation_id:
623 case Intrinsic::spv_subgroup_max_size:
624 case Intrinsic::spv_subgroup_size:
625 case Intrinsic::spv_thread_id:
626 case Intrinsic::spv_thread_id_in_group:
627 case Intrinsic::spv_udot:
628 case Intrinsic::spv_undef:
629 case Intrinsic::spv_value_md:
630 case Intrinsic::spv_workgroup_size:
631 return false;
632 default:
633 return true;
634 }
635}
636
637// TODO(168736): We should make this either a flag in tabelgen
638// or reduce our dependence on the global registry, so we can remove this
639// function. It can easily be missed when new intrinsics are added.
640static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
641 switch (Opcode) {
642 case SPIRV::OpTypeVoid:
643 case SPIRV::OpTypeBool:
644 case SPIRV::OpTypeInt:
645 case SPIRV::OpTypeFloat:
646 case SPIRV::OpTypeVector:
647 case SPIRV::OpTypeMatrix:
648 case SPIRV::OpTypeImage:
649 case SPIRV::OpTypeSampler:
650 case SPIRV::OpTypeSampledImage:
651 case SPIRV::OpTypeArray:
652 case SPIRV::OpTypeRuntimeArray:
653 case SPIRV::OpTypeStruct:
654 case SPIRV::OpTypeOpaque:
655 case SPIRV::OpTypePointer:
656 case SPIRV::OpTypeFunction:
657 case SPIRV::OpTypeEvent:
658 case SPIRV::OpTypeDeviceEvent:
659 case SPIRV::OpTypeReserveId:
660 case SPIRV::OpTypeQueue:
661 case SPIRV::OpTypePipe:
662 case SPIRV::OpTypeForwardPointer:
663 case SPIRV::OpTypePipeStorage:
664 case SPIRV::OpTypeNamedBarrier:
665 case SPIRV::OpTypeAccelerationStructureNV:
666 case SPIRV::OpTypeCooperativeMatrixNV:
667 case SPIRV::OpTypeCooperativeMatrixKHR:
668 return true;
669 default:
670 return false;
671 }
672}
673
674bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
675 // If there are no definitions, then assume there is some other
676 // side-effect that makes this instruction live.
677 if (MI.getNumDefs() == 0)
678 return false;
679
680 for (const auto &MO : MI.all_defs()) {
681 Register Reg = MO.getReg();
682 if (Reg.isPhysical()) {
683 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
684 return false;
685 }
686 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
687 if (UseMI.getOpcode() != SPIRV::OpName) {
688 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
689 return false;
690 }
691 }
692 }
693
694 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
695 MI.isLifetimeMarker()) {
696 LLVM_DEBUG(
697 dbgs()
698 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
699 return false;
700 }
701 if (MI.isPHI()) {
702 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
703 return true;
704 }
705
706 // It is possible that the only side effect is that the instruction is
707 // referenced in the global registry. If that is the only side effect, the
708 // intrinsic is dead.
709 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
710 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
711 const auto &Intr = cast<GIntrinsic>(Val: MI);
712 if (!intrinsicHasSideEffects(ID: Intr.getIntrinsicID())) {
713 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
714 return true;
715 }
716 }
717
718 if (MI.mayStore() || MI.isCall() ||
719 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
720 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
721 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
722 return false;
723 }
724
725 if (isPreISelGenericOpcode(Opcode: MI.getOpcode())) {
726 // TODO: Is there a generic way to check if the opcode has side effects?
727 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
728 return true;
729 }
730
731 if (isOpcodeWithNoSideEffects(Opcode: MI.getOpcode())) {
732 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
733 return true;
734 }
735
736 return false;
737}
738
739void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
740 // Delete the OpName that uses the result if there is one.
741 for (const auto &MO : MI.all_defs()) {
742 Register Reg = MO.getReg();
743 if (Reg.isPhysical())
744 continue;
745 SmallVector<MachineInstr *, 4> UselessOpNames;
746 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
747 assert(UseMI.getOpcode() == SPIRV::OpName &&
748 "There is still a use of the dead function.");
749 UselessOpNames.push_back(Elt: &UseMI);
750 }
751 for (MachineInstr *OpNameMI : UselessOpNames) {
752 GR.invalidateMachineInstr(MI: OpNameMI);
753 OpNameMI->eraseFromParent();
754 }
755 }
756}
757
758void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
759 salvageDebugInfo(MRI: *MRI, MI);
760 GR.invalidateMachineInstr(MI: &MI);
761 removeOpNamesForDeadMI(MI);
762 MI.eraseFromParent();
763}
764
765bool SPIRVInstructionSelector::select(MachineInstr &I) {
766 resetVRegsType(MF&: *I.getParent()->getParent());
767
768 assert(I.getParent() && "Instruction should be in a basic block!");
769 assert(I.getParent()->getParent() && "Instruction should be in a function!");
770
771 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
772 if (isDead(MI: I, MRI: *MRI)) {
773 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
774 removeDeadInstruction(MI&: I);
775 return true;
776 }
777
778 Register Opcode = I.getOpcode();
779 // If it's not a GMIR instruction, we've selected it already.
780 if (!isPreISelGenericOpcode(Opcode)) {
781 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
782 Register DstReg = I.getOperand(i: 0).getReg();
783 Register SrcReg = I.getOperand(i: 1).getReg();
784 auto *Def = MRI->getVRegDef(Reg: SrcReg);
785 if (isTypeFoldingSupported(Opcode: Def->getOpcode()) &&
786 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
787 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
788 bool Res = false;
789 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
790 Register SelectDstReg = Def->getOperand(i: 0).getReg();
791 Res = selectSelect(ResVReg: SelectDstReg, ResType: GR.getSPIRVTypeForVReg(VReg: SelectDstReg),
792 I&: *Def);
793 GR.invalidateMachineInstr(MI: Def);
794 Def->removeFromParent();
795 MRI->replaceRegWith(FromReg: DstReg, ToReg: SelectDstReg);
796 GR.invalidateMachineInstr(MI: &I);
797 I.removeFromParent();
798 } else
799 Res = selectImpl(I, CoverageInfo&: *CoverageInfo);
800 LLVM_DEBUG({
801 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
802 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
803 I.print(dbgs());
804 }
805 });
806 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
807 if (Res) {
808 if (!isTriviallyDead(MI: *Def, MRI: *MRI) && isDead(MI: *Def, MRI: *MRI))
809 DeadMIs.insert(Ptr: Def);
810 return Res;
811 }
812 }
813 MRI->setRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg));
814 MRI->replaceRegWith(FromReg: SrcReg, ToReg: DstReg);
815 GR.invalidateMachineInstr(MI: &I);
816 I.removeFromParent();
817 return true;
818 } else if (I.getNumDefs() == 1) {
819 // Make all vregs 64 bits (for SPIR-V IDs).
820 MRI->setType(VReg: I.getOperand(i: 0).getReg(), Ty: LLT::scalar(SizeInBits: 64));
821 }
822 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
823 return true;
824 }
825
826 if (DeadMIs.contains(Ptr: &I)) {
827 // if the instruction has been already made dead by folding it away
828 // erase it
829 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
830 removeDeadInstruction(MI&: I);
831 return true;
832 }
833
834 if (I.getNumOperands() != I.getNumExplicitOperands()) {
835 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
836 return false;
837 }
838
839 // Common code for getting return reg+type, and removing selected instr
840 // from parent occurs here. Instr-specific selection happens in spvSelect().
841 bool HasDefs = I.getNumDefs() > 0;
842 Register ResVReg = HasDefs ? I.getOperand(i: 0).getReg() : Register(0);
843 SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(VReg: ResVReg) : nullptr;
844 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
845 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
846 if (spvSelect(ResVReg, ResType, I)) {
847 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
848 for (unsigned i = 0; i < I.getNumDefs(); ++i)
849 MRI->setType(VReg: I.getOperand(i).getReg(), Ty: LLT::scalar(SizeInBits: 64));
850 GR.invalidateMachineInstr(MI: &I);
851 I.eraseFromParent();
852 return true;
853 }
854 return false;
855}
856
857static bool mayApplyGenericSelection(unsigned Opcode) {
858 switch (Opcode) {
859 case TargetOpcode::G_CONSTANT:
860 case TargetOpcode::G_FCONSTANT:
861 return false;
862 case TargetOpcode::G_SADDO:
863 case TargetOpcode::G_SSUBO:
864 return true;
865 }
866 return isTypeFoldingSupported(Opcode);
867}
868
869bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
870 MachineInstr &I) const {
871 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: DestReg);
872 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(Reg: SrcReg);
873 if (DstRC != SrcRC && SrcRC)
874 MRI->setRegClass(Reg: DestReg, RC: SrcRC);
875 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY))
876 .addDef(RegNo: DestReg)
877 .addUse(RegNo: SrcReg)
878 .constrainAllUses(TII, TRI, RBI);
879 return true;
880}
881
882bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
883 SPIRVTypeInst ResType,
884 MachineInstr &I) const {
885 const unsigned Opcode = I.getOpcode();
886 if (mayApplyGenericSelection(Opcode))
887 return selectImpl(I, CoverageInfo&: *CoverageInfo);
888 switch (Opcode) {
889 case TargetOpcode::G_CONSTANT:
890 case TargetOpcode::G_FCONSTANT:
891 return selectConst(ResVReg, ResType, I);
892 case TargetOpcode::G_GLOBAL_VALUE:
893 return selectGlobalValue(ResVReg, I);
894 case TargetOpcode::G_IMPLICIT_DEF:
895 return selectOpUndef(ResVReg, ResType, I);
896 case TargetOpcode::G_FREEZE:
897 return selectFreeze(ResVReg, ResType, I);
898
899 case TargetOpcode::G_INTRINSIC:
900 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
901 case TargetOpcode::G_INTRINSIC_CONVERGENT:
902 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
903 return selectIntrinsic(ResVReg, ResType, I);
904 case TargetOpcode::G_BITREVERSE:
905 return selectBitreverse(ResVReg, ResType, I);
906
907 case TargetOpcode::G_BUILD_VECTOR:
908 return selectBuildVector(ResVReg, ResType, I);
909 case TargetOpcode::G_SPLAT_VECTOR:
910 return selectSplatVector(ResVReg, ResType, I);
911
912 case TargetOpcode::G_SHUFFLE_VECTOR: {
913 MachineBasicBlock &BB = *I.getParent();
914 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
915 .addDef(RegNo: ResVReg)
916 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
917 .addUse(RegNo: I.getOperand(i: 1).getReg())
918 .addUse(RegNo: I.getOperand(i: 2).getReg());
919 for (auto V : I.getOperand(i: 3).getShuffleMask())
920 MIB.addImm(Val: V);
921 MIB.constrainAllUses(TII, TRI, RBI);
922 return true;
923 }
924 case TargetOpcode::G_MEMMOVE:
925 case TargetOpcode::G_MEMCPY:
926 case TargetOpcode::G_MEMSET:
927 return selectMemOperation(ResVReg, I);
928
929 case TargetOpcode::G_ICMP:
930 return selectICmp(ResVReg, ResType, I);
931 case TargetOpcode::G_FCMP:
932 return selectFCmp(ResVReg, ResType, I);
933
934 case TargetOpcode::G_FRAME_INDEX:
935 return selectFrameIndex(ResVReg, ResType, I);
936
937 case TargetOpcode::G_LOAD:
938 return selectLoad(ResVReg, ResType, I);
939 case TargetOpcode::G_STORE:
940 return selectStore(I);
941
942 case TargetOpcode::G_BR:
943 return selectBranch(I);
944 case TargetOpcode::G_BRCOND:
945 return selectBranchCond(I);
946
947 case TargetOpcode::G_PHI:
948 return selectPhi(ResVReg, ResType, I);
949
950 case TargetOpcode::G_FPTOSI:
951 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
952 case TargetOpcode::G_FPTOUI:
953 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
954
955 case TargetOpcode::G_FPTOSI_SAT:
956 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
957 case TargetOpcode::G_FPTOUI_SAT:
958 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
959
960 case TargetOpcode::G_SITOFP:
961 return selectIToF(ResVReg, ResType, I, IsSigned: true, Opcode: SPIRV::OpConvertSToF);
962 case TargetOpcode::G_UITOFP:
963 return selectIToF(ResVReg, ResType, I, IsSigned: false, Opcode: SPIRV::OpConvertUToF);
964
965 case TargetOpcode::G_CTPOP:
966 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitCount);
967 case TargetOpcode::G_SMIN:
968 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_min, GLInst: GL::SMin);
969 case TargetOpcode::G_UMIN:
970 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_min, GLInst: GL::UMin);
971
972 case TargetOpcode::G_SMAX:
973 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_max, GLInst: GL::SMax);
974 case TargetOpcode::G_UMAX:
975 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_max, GLInst: GL::UMax);
976
977 case TargetOpcode::G_SCMP:
978 return selectSUCmp(ResVReg, ResType, I, IsSigned: true);
979 case TargetOpcode::G_UCMP:
980 return selectSUCmp(ResVReg, ResType, I, IsSigned: false);
981 case TargetOpcode::G_LROUND:
982 case TargetOpcode::G_LLROUND: {
983 Register regForLround =
984 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ResVReg), Name: "lround");
985 MRI->setRegClass(Reg: regForLround, RC: &SPIRV::iIDRegClass);
986 GR.assignSPIRVTypeToVReg(Type: GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg()),
987 VReg: regForLround, MF: *(I.getParent()->getParent()));
988 selectExtInstForLRound(ResVReg: regForLround, ResType: GR.getSPIRVTypeForVReg(VReg: regForLround),
989 I, CLInst: CL::round, GLInst: GL::Round);
990 MachineBasicBlock &BB = *I.getParent();
991 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConvertFToS))
992 .addDef(RegNo: ResVReg)
993 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
994 .addUse(RegNo: regForLround);
995 MIB.constrainAllUses(TII, TRI, RBI);
996 return true;
997 }
998 case TargetOpcode::G_STRICT_FMA:
999 case TargetOpcode::G_FMA: {
1000 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
1001 MachineBasicBlock &BB = *I.getParent();
1002 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFmaKHR))
1003 .addDef(RegNo: ResVReg)
1004 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1005 .addUse(RegNo: I.getOperand(i: 1).getReg())
1006 .addUse(RegNo: I.getOperand(i: 2).getReg())
1007 .addUse(RegNo: I.getOperand(i: 3).getReg())
1008 .setMIFlags(I.getFlags());
1009 MIB.constrainAllUses(TII, TRI, RBI);
1010 return true;
1011 }
1012 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fma, GLInst: GL::Fma);
1013 }
1014
1015 case TargetOpcode::G_STRICT_FLDEXP:
1016 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ldexp);
1017
1018 case TargetOpcode::G_FPOW:
1019 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pow, GLInst: GL::Pow);
1020 case TargetOpcode::G_FPOWI:
1021 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pown);
1022
1023 case TargetOpcode::G_FEXP:
1024 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp, GLInst: GL::Exp);
1025 case TargetOpcode::G_FEXP2:
1026 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp2, GLInst: GL::Exp2);
1027 case TargetOpcode::G_FMODF:
1028 return selectModf(ResVReg, ResType, I);
1029 case TargetOpcode::G_FSINCOS:
1030 return selectSincos(ResVReg, ResType, I);
1031
1032 case TargetOpcode::G_FLOG:
1033 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log, GLInst: GL::Log);
1034 case TargetOpcode::G_FLOG2:
1035 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log2, GLInst: GL::Log2);
1036 case TargetOpcode::G_FLOG10:
1037 return selectLog10(ResVReg, ResType, I);
1038
1039 case TargetOpcode::G_FABS:
1040 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fabs, GLInst: GL::FAbs);
1041 case TargetOpcode::G_ABS:
1042 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_abs, GLInst: GL::SAbs);
1043
1044 case TargetOpcode::G_FMINNUM:
1045 case TargetOpcode::G_FMINIMUM:
1046 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmin, GLInst: GL::NMin);
1047 case TargetOpcode::G_FMAXNUM:
1048 case TargetOpcode::G_FMAXIMUM:
1049 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmax, GLInst: GL::NMax);
1050
1051 case TargetOpcode::G_FCOPYSIGN:
1052 return selectExtInst(ResVReg, ResType, I, CLInst: CL::copysign);
1053
1054 case TargetOpcode::G_FCEIL:
1055 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ceil, GLInst: GL::Ceil);
1056 case TargetOpcode::G_FFLOOR:
1057 return selectExtInst(ResVReg, ResType, I, CLInst: CL::floor, GLInst: GL::Floor);
1058
1059 case TargetOpcode::G_FCOS:
1060 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cos, GLInst: GL::Cos);
1061 case TargetOpcode::G_FSIN:
1062 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sin, GLInst: GL::Sin);
1063 case TargetOpcode::G_FTAN:
1064 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tan, GLInst: GL::Tan);
1065 case TargetOpcode::G_FACOS:
1066 return selectExtInst(ResVReg, ResType, I, CLInst: CL::acos, GLInst: GL::Acos);
1067 case TargetOpcode::G_FASIN:
1068 return selectExtInst(ResVReg, ResType, I, CLInst: CL::asin, GLInst: GL::Asin);
1069 case TargetOpcode::G_FATAN:
1070 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan, GLInst: GL::Atan);
1071 case TargetOpcode::G_FATAN2:
1072 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan2, GLInst: GL::Atan2);
1073 case TargetOpcode::G_FCOSH:
1074 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cosh, GLInst: GL::Cosh);
1075 case TargetOpcode::G_FSINH:
1076 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sinh, GLInst: GL::Sinh);
1077 case TargetOpcode::G_FTANH:
1078 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tanh, GLInst: GL::Tanh);
1079
1080 case TargetOpcode::G_STRICT_FSQRT:
1081 case TargetOpcode::G_FSQRT:
1082 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sqrt, GLInst: GL::Sqrt);
1083
1084 case TargetOpcode::G_CTTZ:
1085 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1086 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ctz);
1087 case TargetOpcode::G_CTLZ:
1088 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1089 return selectExtInst(ResVReg, ResType, I, CLInst: CL::clz);
1090
1091 case TargetOpcode::G_INTRINSIC_ROUND:
1092 return selectExtInst(ResVReg, ResType, I, CLInst: CL::round, GLInst: GL::Round);
1093 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1094 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1095 case TargetOpcode::G_INTRINSIC_TRUNC:
1096 return selectExtInst(ResVReg, ResType, I, CLInst: CL::trunc, GLInst: GL::Trunc);
1097 case TargetOpcode::G_FRINT:
1098 case TargetOpcode::G_FNEARBYINT:
1099 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1100
1101 case TargetOpcode::G_SMULH:
1102 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_mul_hi);
1103 case TargetOpcode::G_UMULH:
1104 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_mul_hi);
1105
1106 case TargetOpcode::G_SADDSAT:
1107 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_add_sat);
1108 case TargetOpcode::G_UADDSAT:
1109 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_add_sat);
1110 case TargetOpcode::G_SSUBSAT:
1111 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_sub_sat);
1112 case TargetOpcode::G_USUBSAT:
1113 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_sub_sat);
1114
1115 case TargetOpcode::G_FFREXP:
1116 return selectFrexp(ResVReg, ResType, I);
1117
1118 case TargetOpcode::G_UADDO:
1119 return selectOverflowArith(ResVReg, ResType, I,
1120 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1121 ? SPIRV::OpIAddCarryV
1122 : SPIRV::OpIAddCarryS);
1123 case TargetOpcode::G_USUBO:
1124 return selectOverflowArith(ResVReg, ResType, I,
1125 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1126 ? SPIRV::OpISubBorrowV
1127 : SPIRV::OpISubBorrowS);
1128 case TargetOpcode::G_UMULO:
1129 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpUMulExtended);
1130 case TargetOpcode::G_SMULO:
1131 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpSMulExtended);
1132
1133 case TargetOpcode::G_SEXT:
1134 return selectExt(ResVReg, ResType, I, IsSigned: true);
1135 case TargetOpcode::G_ANYEXT:
1136 case TargetOpcode::G_ZEXT:
1137 return selectExt(ResVReg, ResType, I, IsSigned: false);
1138 case TargetOpcode::G_TRUNC:
1139 return selectTrunc(ResVReg, ResType, I);
1140 case TargetOpcode::G_FPTRUNC:
1141 case TargetOpcode::G_FPEXT:
1142 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpFConvert);
1143
1144 case TargetOpcode::G_PTRTOINT:
1145 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertPtrToU);
1146 case TargetOpcode::G_INTTOPTR:
1147 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertUToPtr);
1148 case TargetOpcode::G_BITCAST:
1149 return selectBitcast(ResVReg, ResType, I);
1150 case TargetOpcode::G_ADDRSPACE_CAST:
1151 return selectAddrSpaceCast(ResVReg, ResType, I);
1152 case TargetOpcode::G_PTR_ADD: {
1153 // Currently, we get G_PTR_ADD only applied to global variables.
1154 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1155 Register GV = I.getOperand(i: 1).getReg();
1156 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(RegNo: GV);
1157 (void)II;
1158 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1159 (*II).getOpcode() == TargetOpcode::COPY ||
1160 (*II).getOpcode() == SPIRV::OpVariable) &&
1161 getImm(I.getOperand(2), MRI));
1162 // It may be the initialization of a global variable.
1163 bool IsGVInit = false;
1164 for (MachineRegisterInfo::use_instr_iterator
1165 UseIt = MRI->use_instr_begin(RegNo: I.getOperand(i: 0).getReg()),
1166 UseEnd = MRI->use_instr_end();
1167 UseIt != UseEnd; UseIt = std::next(x: UseIt)) {
1168 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1169 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1170 (*UseIt).getOpcode() == SPIRV::OpVariable) {
1171 IsGVInit = true;
1172 break;
1173 }
1174 }
1175 MachineBasicBlock &BB = *I.getParent();
1176 if (!IsGVInit) {
1177 SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(VReg: GV);
1178 SPIRVTypeInst GVPointeeType = GR.getPointeeType(PtrType: GVType);
1179 SPIRVTypeInst ResPointeeType = GR.getPointeeType(PtrType: ResType);
1180 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1181 // Build a new virtual register that is associated with the required
1182 // data type.
1183 Register NewVReg = MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: GV));
1184 MRI->setRegClass(Reg: NewVReg, RC: MRI->getRegClass(Reg: GV));
1185 // Having a correctly typed base we are ready to build the actually
1186 // required GEP. It may not be a constant though, because all Operands
1187 // of OpSpecConstantOp is to originate from other const instructions,
1188 // and only the AccessChain named opcodes accept a global OpVariable
1189 // instruction. We can't use an AccessChain opcode because of the type
1190 // mismatch between result and base types.
1191 if (!GR.isBitcastCompatible(Type1: ResType, Type2: GVType))
1192 report_fatal_error(
1193 reason: "incompatible result and operand types in a bitcast");
1194 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
1195 MachineInstrBuilder MIB =
1196 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitcast))
1197 .addDef(RegNo: NewVReg)
1198 .addUse(RegNo: ResTypeReg)
1199 .addUse(RegNo: GV);
1200 MIB.constrainAllUses(TII, TRI, RBI);
1201 BuildMI(BB, I, MIMD: I.getDebugLoc(),
1202 MCID: TII.get(Opcode: STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain
1203 : SPIRV::OpInBoundsPtrAccessChain))
1204 .addDef(RegNo: ResVReg)
1205 .addUse(RegNo: ResTypeReg)
1206 .addUse(RegNo: NewVReg)
1207 .addUse(RegNo: I.getOperand(i: 2).getReg())
1208 .constrainAllUses(TII, TRI, RBI);
1209 } else {
1210 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1211 .addDef(RegNo: ResVReg)
1212 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1213 .addImm(
1214 Val: static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1215 .addUse(RegNo: GV)
1216 .addUse(RegNo: I.getOperand(i: 2).getReg())
1217 .constrainAllUses(TII, TRI, RBI);
1218 }
1219 return true;
1220 }
1221 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1222 // initialize a global variable with a constant expression (e.g., the test
1223 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1224 Register Idx = buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
1225 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1226 .addDef(RegNo: ResVReg)
1227 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1228 .addImm(Val: static_cast<uint32_t>(
1229 SPIRV::Opcode::InBoundsPtrAccessChain))
1230 .addUse(RegNo: GV)
1231 .addUse(RegNo: Idx)
1232 .addUse(RegNo: I.getOperand(i: 2).getReg());
1233 MIB.constrainAllUses(TII, TRI, RBI);
1234 return true;
1235 }
1236
1237 case TargetOpcode::G_ATOMICRMW_OR:
1238 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicOr);
1239 case TargetOpcode::G_ATOMICRMW_ADD:
1240 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicIAdd);
1241 case TargetOpcode::G_ATOMICRMW_AND:
1242 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicAnd);
1243 case TargetOpcode::G_ATOMICRMW_MAX:
1244 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMax);
1245 case TargetOpcode::G_ATOMICRMW_MIN:
1246 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMin);
1247 case TargetOpcode::G_ATOMICRMW_SUB:
1248 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicISub);
1249 case TargetOpcode::G_ATOMICRMW_XOR:
1250 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicXor);
1251 case TargetOpcode::G_ATOMICRMW_UMAX:
1252 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMax);
1253 case TargetOpcode::G_ATOMICRMW_UMIN:
1254 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMin);
1255 case TargetOpcode::G_ATOMICRMW_XCHG:
1256 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicExchange);
1257 case TargetOpcode::G_ATOMIC_CMPXCHG:
1258 return selectAtomicCmpXchg(ResVReg, ResType, I);
1259
1260 case TargetOpcode::G_ATOMICRMW_FADD:
1261 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT);
1262 case TargetOpcode::G_ATOMICRMW_FSUB:
1263 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1264 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT,
1265 NegateOpcode: ResType->getOpcode() == SPIRV::OpTypeVector
1266 ? SPIRV::OpFNegateV
1267 : SPIRV::OpFNegate);
1268 case TargetOpcode::G_ATOMICRMW_FMIN:
1269 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMinEXT);
1270 case TargetOpcode::G_ATOMICRMW_FMAX:
1271 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMaxEXT);
1272
1273 case TargetOpcode::G_FENCE:
1274 return selectFence(I);
1275
1276 case TargetOpcode::G_STACKSAVE:
1277 return selectStackSave(ResVReg, ResType, I);
1278 case TargetOpcode::G_STACKRESTORE:
1279 return selectStackRestore(I);
1280
1281 case TargetOpcode::G_UNMERGE_VALUES:
1282 return selectUnmergeValues(I);
1283
1284 // Discard gen opcodes for intrinsics which we do not expect to actually
1285 // represent code after lowering or intrinsics which are not implemented but
1286 // should not crash when found in a customer's LLVM IR input.
1287 case TargetOpcode::G_TRAP:
1288 case TargetOpcode::G_UBSANTRAP:
1289 case TargetOpcode::DBG_LABEL:
1290 return true;
1291 case TargetOpcode::G_DEBUGTRAP:
1292 return selectDebugTrap(ResVReg, ResType, I);
1293
1294 default:
1295 return false;
1296 }
1297}
1298
1299bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1300 SPIRVTypeInst ResType,
1301 MachineInstr &I) const {
1302 unsigned Opcode = SPIRV::OpNop;
1303 MachineBasicBlock &BB = *I.getParent();
1304 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1305 .constrainAllUses(TII, TRI, RBI);
1306 return true;
1307}
1308
1309bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1310 SPIRVTypeInst ResType,
1311 MachineInstr &I,
1312 GL::GLSLExtInst GLInst) const {
1313 if (!STI.canUseExtInstSet(
1314 E: SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) {
1315 std::string DiagMsg;
1316 raw_string_ostream OS(DiagMsg);
1317 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
1318 DiagMsg += " is only supported with the GLSL extended instruction set.\n";
1319 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1320 }
1321 return selectExtInst(ResVReg, ResType, I,
1322 ExtInsts: {{SPIRV::InstructionSet::GLSL_std_450, GLInst}});
1323}
1324
1325bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1326 SPIRVTypeInst ResType,
1327 MachineInstr &I,
1328 CL::OpenCLExtInst CLInst) const {
1329 return selectExtInst(ResVReg, ResType, I,
1330 ExtInsts: {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
1331}
1332
1333bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1334 SPIRVTypeInst ResType,
1335 MachineInstr &I,
1336 CL::OpenCLExtInst CLInst,
1337 GL::GLSLExtInst GLInst) const {
1338 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1339 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1340 return selectExtInst(ResVReg, ResType, I, ExtInsts);
1341}
1342
1343bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1344 SPIRVTypeInst ResType,
1345 MachineInstr &I,
1346 const ExtInstList &Insts) const {
1347
1348 for (const auto &Ex : Insts) {
1349 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1350 uint32_t Opcode = Ex.second;
1351 if (STI.canUseExtInstSet(E: Set)) {
1352 MachineBasicBlock &BB = *I.getParent();
1353 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1354 .addDef(RegNo: ResVReg)
1355 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1356 .addImm(Val: static_cast<uint32_t>(Set))
1357 .addImm(Val: Opcode)
1358 .setMIFlags(I.getFlags());
1359 const unsigned NumOps = I.getNumOperands();
1360 unsigned Index = 1;
1361 if (Index < NumOps &&
1362 I.getOperand(i: Index).getType() ==
1363 MachineOperand::MachineOperandType::MO_IntrinsicID)
1364 Index = 2;
1365 for (; Index < NumOps; ++Index)
1366 MIB.add(MO: I.getOperand(i: Index));
1367 MIB.constrainAllUses(TII, TRI, RBI);
1368 return true;
1369 }
1370 }
1371 return false;
1372}
1373bool SPIRVInstructionSelector::selectExtInstForLRound(
1374 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1375 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst) const {
1376 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1377 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1378 return selectExtInstForLRound(ResVReg, ResType, I, ExtInsts);
1379}
1380
1381bool SPIRVInstructionSelector::selectExtInstForLRound(
1382 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1383 const ExtInstList &Insts) const {
1384 for (const auto &Ex : Insts) {
1385 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1386 uint32_t Opcode = Ex.second;
1387 if (STI.canUseExtInstSet(E: Set)) {
1388 MachineBasicBlock &BB = *I.getParent();
1389 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1390 .addDef(RegNo: ResVReg)
1391 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1392 .addImm(Val: static_cast<uint32_t>(Set))
1393 .addImm(Val: Opcode);
1394 const unsigned NumOps = I.getNumOperands();
1395 unsigned Index = 1;
1396 if (Index < NumOps &&
1397 I.getOperand(i: Index).getType() ==
1398 MachineOperand::MachineOperandType::MO_IntrinsicID)
1399 Index = 2;
1400 for (; Index < NumOps; ++Index)
1401 MIB.add(MO: I.getOperand(i: Index));
1402 MIB.constrainAllUses(TII, TRI, RBI);
1403 return true;
1404 }
1405 }
1406 return false;
1407}
1408
1409bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1410 SPIRVTypeInst ResType,
1411 MachineInstr &I) const {
1412 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1413 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1414 for (const auto &Ex : ExtInsts) {
1415 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1416 uint32_t Opcode = Ex.second;
1417 if (!STI.canUseExtInstSet(E: Set))
1418 continue;
1419
1420 MachineIRBuilder MIRBuilder(I);
1421 SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg());
1422 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1423 BaseType: PointeeTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
1424 Register PointerVReg =
1425 createVirtualRegister(SpvType: PointerType, GR: &GR, MRI, MF: MRI->getMF());
1426
1427 auto It = getOpVariableMBBIt(I);
1428 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1429 .addDef(RegNo: PointerVReg)
1430 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointerType))
1431 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
1432 .constrainAllUses(TII, TRI, RBI);
1433
1434 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1435 .addDef(RegNo: ResVReg)
1436 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1437 .addImm(Val: static_cast<uint32_t>(Ex.first))
1438 .addImm(Val: Opcode)
1439 .add(MO: I.getOperand(i: 2))
1440 .addUse(RegNo: PointerVReg)
1441 .constrainAllUses(TII, TRI, RBI);
1442
1443 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1444 .addDef(RegNo: I.getOperand(i: 1).getReg())
1445 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointeeTy))
1446 .addUse(RegNo: PointerVReg)
1447 .constrainAllUses(TII, TRI, RBI);
1448 return true;
1449 }
1450 return false;
1451}
1452
1453bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
1454 SPIRVTypeInst ResType,
1455 MachineInstr &I) const {
1456 Register CosResVReg = I.getOperand(i: 1).getReg();
1457 unsigned SrcIdx = I.getNumExplicitDefs();
1458 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
1459
1460 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
1461 // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr.
1462 MachineIRBuilder MIRBuilder(I);
1463 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1464 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Function);
1465 Register PointerVReg =
1466 createVirtualRegister(SpvType: PointerType, GR: &GR, MRI, MF: MRI->getMF());
1467
1468 auto It = getOpVariableMBBIt(I);
1469 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1470 .addDef(RegNo: PointerVReg)
1471 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointerType))
1472 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
1473 .constrainAllUses(TII, TRI, RBI);
1474 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1475 .addDef(RegNo: ResVReg)
1476 .addUse(RegNo: ResTypeReg)
1477 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1478 .addImm(Val: CL::sincos)
1479 .add(MO: I.getOperand(i: SrcIdx))
1480 .addUse(RegNo: PointerVReg)
1481 .constrainAllUses(TII, TRI, RBI);
1482 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1483 .addDef(RegNo: CosResVReg)
1484 .addUse(RegNo: ResTypeReg)
1485 .addUse(RegNo: PointerVReg)
1486 .constrainAllUses(TII, TRI, RBI);
1487 return true;
1488 } else if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
1489 // GLSL.std.450 has no combined sincos; emit separate Sin and Cos.
1490 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1491 .addDef(RegNo: ResVReg)
1492 .addUse(RegNo: ResTypeReg)
1493 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1494 .addImm(Val: GL::Sin)
1495 .add(MO: I.getOperand(i: SrcIdx))
1496 .constrainAllUses(TII, TRI, RBI);
1497 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1498 .addDef(RegNo: CosResVReg)
1499 .addUse(RegNo: ResTypeReg)
1500 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1501 .addImm(Val: GL::Cos)
1502 .add(MO: I.getOperand(i: SrcIdx))
1503 .constrainAllUses(TII, TRI, RBI);
1504 return true;
1505 }
1506 return false;
1507}
1508
1509bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1510 SPIRVTypeInst ResType,
1511 MachineInstr &I,
1512 std::vector<Register> Srcs,
1513 unsigned Opcode) const {
1514 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1515 .addDef(RegNo: ResVReg)
1516 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
1517 for (Register SReg : Srcs) {
1518 MIB.addUse(RegNo: SReg);
1519 }
1520 MIB.constrainAllUses(TII, TRI, RBI);
1521 return true;
1522}
1523
1524bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1525 SPIRVTypeInst ResType,
1526 MachineInstr &I,
1527 unsigned Opcode) const {
1528 if (STI.isPhysicalSPIRV() && I.getOperand(i: 1).isReg()) {
1529 Register SrcReg = I.getOperand(i: 1).getReg();
1530 bool IsGV = false;
1531 for (MachineRegisterInfo::def_instr_iterator DefIt =
1532 MRI->def_instr_begin(RegNo: SrcReg);
1533 DefIt != MRI->def_instr_end(); DefIt = std::next(x: DefIt)) {
1534 unsigned DefOpCode = DefIt->getOpcode();
1535 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1536 // We need special handling to look through the type assignment or the
1537 // COPY pseudo-op and see if this is a constant or a global.
1538 if (auto *VRD = getVRegDef(MRI&: *MRI, Reg: DefIt->getOperand(i: 1).getReg()))
1539 DefOpCode = VRD->getOpcode();
1540 }
1541 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1542 DefOpCode == TargetOpcode::G_CONSTANT ||
1543 DefOpCode == SPIRV::OpVariable || DefOpCode == SPIRV::OpConstantI) {
1544 IsGV = true;
1545 break;
1546 }
1547 }
1548 if (IsGV) {
1549 uint32_t SpecOpcode = 0;
1550 switch (Opcode) {
1551 case SPIRV::OpConvertPtrToU:
1552 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1553 break;
1554 case SPIRV::OpConvertUToPtr:
1555 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1556 break;
1557 }
1558 if (SpecOpcode) {
1559 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1560 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1561 .addDef(RegNo: ResVReg)
1562 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1563 .addImm(Val: SpecOpcode)
1564 .addUse(RegNo: SrcReg)
1565 .constrainAllUses(TII, TRI, RBI);
1566 return true;
1567 }
1568 }
1569 }
1570 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {I.getOperand(i: 1).getReg()},
1571 Opcode);
1572}
1573
1574bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1575 SPIRVTypeInst ResType,
1576 MachineInstr &I) const {
1577 Register OpReg = I.getOperand(i: 1).getReg();
1578 SPIRVTypeInst OpType =
1579 OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
1580 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
1581 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
1582 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitcast);
1583}
1584
1585static void addMemoryOperands(MachineMemOperand *MemOp,
1586 MachineInstrBuilder &MIB,
1587 MachineIRBuilder &MIRBuilder,
1588 SPIRVGlobalRegistry &GR) {
1589 const SPIRVSubtarget *ST =
1590 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1591 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1592 if (MemOp->isVolatile())
1593 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1594 if (MemOp->isNonTemporal())
1595 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1596 // Aligned memory operand requires the Kernel capability.
1597 if (!ST->isShader() && MemOp->getAlign().value())
1598 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1599
1600 [[maybe_unused]] MachineInstr *AliasList = nullptr;
1601 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
1602 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
1603 if (auto *MD = MemOp->getAAInfo().Scope) {
1604 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1605 if (AliasList)
1606 SpvMemOp |=
1607 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
1608 }
1609 if (auto *MD = MemOp->getAAInfo().NoAlias) {
1610 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1611 if (NoAliasList)
1612 SpvMemOp |=
1613 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
1614 }
1615 }
1616
1617 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1618 MIB.addImm(Val: SpvMemOp);
1619 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1620 MIB.addImm(Val: MemOp->getAlign().value());
1621 if (AliasList)
1622 MIB.addUse(RegNo: AliasList->getOperand(i: 0).getReg());
1623 if (NoAliasList)
1624 MIB.addUse(RegNo: NoAliasList->getOperand(i: 0).getReg());
1625 }
1626}
1627
1628static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
1629 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1630 if (Flags & MachineMemOperand::Flags::MOVolatile)
1631 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1632 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1633 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1634
1635 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1636 MIB.addImm(Val: SpvMemOp);
1637}
1638
1639bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1640 SPIRVTypeInst ResType,
1641 MachineInstr &I) const {
1642 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1643 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1644
1645 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1646 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1647 if (IntPtrDef &&
1648 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1649 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1650 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1651 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1652 Register NewHandleReg =
1653 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1654 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1655 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1656 return false;
1657 }
1658
1659 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1660 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewHandleReg, IdxReg,
1661 Loc: I.getDebugLoc(), Pos&: I);
1662 }
1663 }
1664
1665 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1666 .addDef(RegNo: ResVReg)
1667 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1668 .addUse(RegNo: Ptr);
1669 if (!I.getNumMemOperands()) {
1670 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1671 I.getOpcode() ==
1672 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1673 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1674 } else {
1675 MachineIRBuilder MIRBuilder(I);
1676 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1677 }
1678 MIB.constrainAllUses(TII, TRI, RBI);
1679 return true;
1680}
1681
1682bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1683 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1684 Register StoreVal = I.getOperand(i: 0 + OpOffset).getReg();
1685 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1686
1687 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1688 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1689 if (IntPtrDef &&
1690 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1691 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1692 Register NewHandleReg =
1693 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1694 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1695 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1696 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1697 return false;
1698 }
1699
1700 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1701 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1702 auto BMI = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1703 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
1704 .addUse(RegNo: NewHandleReg)
1705 .addUse(RegNo: IdxReg)
1706 .addUse(RegNo: StoreVal);
1707
1708 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(Ty: HandleType);
1709 if (sampledTypeIsSignedInteger(HandleType: LLVMHandleType))
1710 BMI.addImm(Val: 0x1000); // SignExtend
1711
1712 BMI.constrainAllUses(TII, TRI, RBI);
1713 return true;
1714 }
1715 }
1716
1717 MachineBasicBlock &BB = *I.getParent();
1718 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpStore))
1719 .addUse(RegNo: Ptr)
1720 .addUse(RegNo: StoreVal);
1721 if (!I.getNumMemOperands()) {
1722 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1723 I.getOpcode() ==
1724 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1725 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1726 } else {
1727 MachineIRBuilder MIRBuilder(I);
1728 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1729 }
1730 MIB.constrainAllUses(TII, TRI, RBI);
1731 return true;
1732}
1733
1734bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
1735 SPIRVTypeInst ResType,
1736 MachineInstr &I) const {
1737 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1738 report_fatal_error(
1739 reason: "llvm.stacksave intrinsic: this instruction requires the following "
1740 "SPIR-V extension: SPV_INTEL_variable_length_array",
1741 gen_crash_diag: false);
1742 MachineBasicBlock &BB = *I.getParent();
1743 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSaveMemoryINTEL))
1744 .addDef(RegNo: ResVReg)
1745 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1746 .constrainAllUses(TII, TRI, RBI);
1747 return true;
1748}
1749
1750bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
1751 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1752 report_fatal_error(
1753 reason: "llvm.stackrestore intrinsic: this instruction requires the following "
1754 "SPIR-V extension: SPV_INTEL_variable_length_array",
1755 gen_crash_diag: false);
1756 if (!I.getOperand(i: 0).isReg())
1757 return false;
1758 MachineBasicBlock &BB = *I.getParent();
1759 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpRestoreMemoryINTEL))
1760 .addUse(RegNo: I.getOperand(i: 0).getReg())
1761 .constrainAllUses(TII, TRI, RBI);
1762 return true;
1763}
1764
1765Register
1766SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
1767 MachineIRBuilder MIRBuilder(I);
1768 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1769
1770 // TODO: check if we have such GV, add init, use buildGlobalVariable.
1771 unsigned Num = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1772 Function &CurFunction = GR.CurMF->getFunction();
1773 Type *LLVMArrTy =
1774 ArrayType::get(ElementType: IntegerType::get(C&: CurFunction.getContext(), NumBits: 8), NumElements: Num);
1775 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
1776 true, GlobalValue::InternalLinkage,
1777 Constant::getNullValue(Ty: LLVMArrTy));
1778
1779 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1780 Type *ArrTy = ArrayType::get(ElementType: ValTy, NumElements: Num);
1781 SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType(
1782 BaseType: ArrTy, MIRBuilder, SC: SPIRV::StorageClass::UniformConstant);
1783
1784 SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType(
1785 Type: ArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::None, EmitIR: false);
1786
1787 unsigned Val = getIConstVal(ConstReg: I.getOperand(i: 1).getReg(), MRI);
1788 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvType: SpvArrTy, TII);
1789
1790 Register VarReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1791 auto MIBVar =
1792 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1793 .addDef(RegNo: VarReg)
1794 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VarTy))
1795 .addImm(Val: SPIRV::StorageClass::UniformConstant)
1796 .addUse(RegNo: Const);
1797 MIBVar.constrainAllUses(TII, TRI, RBI);
1798
1799 GR.add(V: GV, MI: MIBVar);
1800 GR.addGlobalObject(V: GV, MF: GR.CurMF, R: VarReg);
1801
1802 buildOpDecorate(Reg: VarReg, I, TII, Dec: SPIRV::Decoration::Constant, DecArgs: {});
1803 return VarReg;
1804}
1805
1806bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
1807 Register SrcReg) const {
1808 MachineBasicBlock &BB = *I.getParent();
1809 Register DstReg = I.getOperand(i: 0).getReg();
1810 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(VReg: DstReg);
1811 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(VReg: SrcReg);
1812 if (GR.getPointeeType(PtrType: DstTy) != GR.getPointeeType(PtrType: SrcTy))
1813 report_fatal_error(reason: "OpCopyMemory requires operands to have the same type");
1814 uint64_t CopySize = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1815 SPIRVTypeInst PointeeTy = GR.getPointeeType(PtrType: DstTy);
1816 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(Ty: PointeeTy);
1817 if (!LLVMPointeeTy)
1818 report_fatal_error(
1819 reason: "Unable to determine pointee type size for OpCopyMemory");
1820 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
1821 if (CopySize != DL.getTypeStoreSize(Ty: const_cast<Type *>(LLVMPointeeTy)))
1822 report_fatal_error(
1823 reason: "OpCopyMemory requires the size to match the pointee type size");
1824 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemory))
1825 .addUse(RegNo: DstReg)
1826 .addUse(RegNo: SrcReg);
1827 if (I.getNumMemOperands()) {
1828 MachineIRBuilder MIRBuilder(I);
1829 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1830 }
1831 MIB.constrainAllUses(TII, TRI, RBI);
1832 return true;
1833}
1834
1835bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
1836 Register SrcReg) const {
1837 MachineBasicBlock &BB = *I.getParent();
1838 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemorySized))
1839 .addUse(RegNo: I.getOperand(i: 0).getReg())
1840 .addUse(RegNo: SrcReg)
1841 .addUse(RegNo: I.getOperand(i: 2).getReg());
1842 if (I.getNumMemOperands()) {
1843 MachineIRBuilder MIRBuilder(I);
1844 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1845 }
1846 MIB.constrainAllUses(TII, TRI, RBI);
1847 return true;
1848}
1849
1850bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
1851 MachineInstr &I) const {
1852 Register SrcReg = I.getOperand(i: 1).getReg();
1853 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
1854 Register VarReg = getOrCreateMemSetGlobal(I);
1855 if (!VarReg.isValid())
1856 return false;
1857 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1858 SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType(
1859 BaseType: ValTy, I, SC: SPIRV::StorageClass::UniformConstant);
1860 SrcReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1861 if (!selectOpWithSrcs(ResVReg: SrcReg, ResType: SourceTy, I, Srcs: {VarReg}, Opcode: SPIRV::OpBitcast))
1862 return false;
1863 }
1864 if (STI.isLogicalSPIRV()) {
1865 if (!selectCopyMemory(I, SrcReg))
1866 return false;
1867 } else {
1868 if (!selectCopyMemorySized(I, SrcReg))
1869 return false;
1870 }
1871 if (ResVReg.isValid() && ResVReg != I.getOperand(i: 0).getReg())
1872 if (!BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 0).getReg(), I))
1873 return false;
1874 return true;
1875}
1876
1877bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
1878 SPIRVTypeInst ResType,
1879 MachineInstr &I,
1880 unsigned NewOpcode,
1881 unsigned NegateOpcode) const {
1882 assert(I.hasOneMemOperand());
1883 const MachineMemOperand *MemOp = *I.memoperands_begin();
1884 uint32_t Scope = static_cast<uint32_t>(getMemScope(
1885 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1886 Register ScopeReg = buildI32Constant(Val: Scope, I);
1887
1888 Register Ptr = I.getOperand(i: 1).getReg();
1889 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
1890 // auto ScSem =
1891 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
1892 AtomicOrdering AO = MemOp->getSuccessOrdering();
1893 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1894 Register MemSemReg = buildI32Constant(Val: MemSem /*| ScSem*/, I);
1895
1896 Register ValueReg = I.getOperand(i: 2).getReg();
1897 if (NegateOpcode != 0) {
1898 // Translation with negative value operand is requested
1899 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: MRI->getMF());
1900 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {ValueReg}, Opcode: NegateOpcode))
1901 return false;
1902 ValueReg = TmpReg;
1903 }
1904
1905 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: NewOpcode))
1906 .addDef(RegNo: ResVReg)
1907 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1908 .addUse(RegNo: Ptr)
1909 .addUse(RegNo: ScopeReg)
1910 .addUse(RegNo: MemSemReg)
1911 .addUse(RegNo: ValueReg)
1912 .constrainAllUses(TII, TRI, RBI);
1913 return true;
1914}
1915
1916bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
1917 unsigned ArgI = I.getNumOperands() - 1;
1918 Register SrcReg =
1919 I.getOperand(i: ArgI).isReg() ? I.getOperand(i: ArgI).getReg() : Register(0);
1920 SPIRVTypeInst SrcType =
1921 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: SrcReg) : nullptr;
1922 if (!SrcType || SrcType->getOpcode() != SPIRV::OpTypeVector)
1923 report_fatal_error(
1924 reason: "cannot select G_UNMERGE_VALUES with a non-vector argument");
1925
1926 SPIRVTypeInst ScalarType =
1927 GR.getSPIRVTypeForVReg(VReg: SrcType->getOperand(i: 1).getReg());
1928 MachineBasicBlock &BB = *I.getParent();
1929 unsigned CurrentIndex = 0;
1930 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1931 Register ResVReg = I.getOperand(i).getReg();
1932 SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(VReg: ResVReg);
1933 if (!ResType) {
1934 LLT ResLLT = MRI->getType(Reg: ResVReg);
1935 assert(ResLLT.isValid());
1936 if (ResLLT.isVector()) {
1937 ResType = GR.getOrCreateSPIRVVectorType(
1938 BaseType: ScalarType, NumElements: ResLLT.getNumElements(), I, TII);
1939 } else {
1940 ResType = ScalarType;
1941 }
1942 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
1943 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *GR.CurMF);
1944 }
1945
1946 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1947 Register UndefReg = GR.getOrCreateUndef(I, SpvType: SrcType, TII);
1948 auto MIB =
1949 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
1950 .addDef(RegNo: ResVReg)
1951 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1952 .addUse(RegNo: SrcReg)
1953 .addUse(RegNo: UndefReg);
1954 unsigned NumElements = GR.getScalarOrVectorComponentCount(Type: ResType);
1955 for (unsigned j = 0; j < NumElements; ++j) {
1956 MIB.addImm(Val: CurrentIndex + j);
1957 }
1958 CurrentIndex += NumElements;
1959 MIB.constrainAllUses(TII, TRI, RBI);
1960 } else {
1961 auto MIB =
1962 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
1963 .addDef(RegNo: ResVReg)
1964 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1965 .addUse(RegNo: SrcReg)
1966 .addImm(Val: CurrentIndex);
1967 CurrentIndex++;
1968 MIB.constrainAllUses(TII, TRI, RBI);
1969 }
1970 }
1971 return true;
1972}
1973
1974bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
1975 AtomicOrdering AO = AtomicOrdering(I.getOperand(i: 0).getImm());
1976 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1977 Register MemSemReg = buildI32Constant(Val: MemSem, I);
1978 SyncScope::ID Ord = SyncScope::ID(I.getOperand(i: 1).getImm());
1979 uint32_t Scope = static_cast<uint32_t>(
1980 getMemScope(Ctx&: GR.CurMF->getFunction().getContext(), Id: Ord));
1981 Register ScopeReg = buildI32Constant(Val: Scope, I);
1982 MachineBasicBlock &BB = *I.getParent();
1983 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMemoryBarrier))
1984 .addUse(RegNo: ScopeReg)
1985 .addUse(RegNo: MemSemReg)
1986 .constrainAllUses(TII, TRI, RBI);
1987 return true;
1988}
1989
1990bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
1991 SPIRVTypeInst ResType,
1992 MachineInstr &I,
1993 unsigned Opcode) const {
1994 Type *ResTy = nullptr;
1995 StringRef ResName;
1996 if (!GR.findValueAttrs(Key: &I, Ty&: ResTy, Name&: ResName))
1997 report_fatal_error(
1998 reason: "Not enough info to select the arithmetic with overflow instruction");
1999 if (!ResTy || !ResTy->isStructTy())
2000 report_fatal_error(reason: "Expect struct type result for the arithmetic "
2001 "with overflow instruction");
2002 // "Result Type must be from OpTypeStruct. The struct must have two members,
2003 // and the two members must be the same type."
2004 Type *ResElemTy = cast<StructType>(Val: ResTy)->getElementType(N: 0);
2005 ResTy = StructType::get(elt1: ResElemTy, elts: ResElemTy);
2006 // Build SPIR-V types and constant(s) if needed.
2007 MachineIRBuilder MIRBuilder(I);
2008 SPIRVTypeInst StructType = GR.getOrCreateSPIRVType(
2009 Type: ResTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
2010 assert(I.getNumDefs() > 1 && "Not enought operands");
2011 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2012 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
2013 if (N > 1)
2014 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
2015 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
2016 Register ZeroReg = buildZerosVal(ResType, I);
2017 // A new virtual register to store the result struct.
2018 Register StructVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2019 MRI->setRegClass(Reg: StructVReg, RC: &SPIRV::IDRegClass);
2020 // Build the result name if needed.
2021 if (ResName.size() > 0)
2022 buildOpName(Target: StructVReg, Name: ResName, MIRBuilder);
2023 // Build the arithmetic with overflow instruction.
2024 MachineBasicBlock &BB = *I.getParent();
2025 auto MIB =
2026 BuildMI(BB, I: MIRBuilder.getInsertPt(), MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2027 .addDef(RegNo: StructVReg)
2028 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: StructType));
2029 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
2030 MIB.addUse(RegNo: I.getOperand(i).getReg());
2031 MIB.constrainAllUses(TII, TRI, RBI);
2032 // Build instructions to extract fields of the instruction's result.
2033 // A new virtual register to store the higher part of the result struct.
2034 Register HigherVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2035 MRI->setRegClass(Reg: HigherVReg, RC: &SPIRV::iIDRegClass);
2036 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2037 auto MIB =
2038 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2039 .addDef(RegNo: i == 1 ? HigherVReg : I.getOperand(i).getReg())
2040 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2041 .addUse(RegNo: StructVReg)
2042 .addImm(Val: i);
2043 MIB.constrainAllUses(TII, TRI, RBI);
2044 }
2045 // Build boolean value from the higher part.
2046 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
2047 .addDef(RegNo: I.getOperand(i: 1).getReg())
2048 .addUse(RegNo: BoolTypeReg)
2049 .addUse(RegNo: HigherVReg)
2050 .addUse(RegNo: ZeroReg)
2051 .constrainAllUses(TII, TRI, RBI);
2052 return true;
2053}
2054
2055bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
2056 SPIRVTypeInst ResType,
2057 MachineInstr &I) const {
2058 Register ScopeReg;
2059 Register MemSemEqReg;
2060 Register MemSemNeqReg;
2061 Register Ptr = I.getOperand(i: 2).getReg();
2062 if (!isa<GIntrinsic>(Val: I)) {
2063 assert(I.hasOneMemOperand());
2064 const MachineMemOperand *MemOp = *I.memoperands_begin();
2065 unsigned Scope = static_cast<uint32_t>(getMemScope(
2066 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
2067 ScopeReg = buildI32Constant(Val: Scope, I);
2068
2069 unsigned ScSem = static_cast<uint32_t>(
2070 getMemSemanticsForStorageClass(SC: GR.getPointerStorageClass(VReg: Ptr)));
2071 AtomicOrdering AO = MemOp->getSuccessOrdering();
2072 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(Ord: AO)) | ScSem;
2073 Register MemSemEqReg = buildI32Constant(Val: MemSemEq, I);
2074 AtomicOrdering FO = MemOp->getFailureOrdering();
2075 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(Ord: FO)) | ScSem;
2076 if (MemSemEq == MemSemNeq)
2077 MemSemNeqReg = MemSemEqReg;
2078 else {
2079 MemSemNeqReg = buildI32Constant(Val: MemSemEq, I);
2080 }
2081 } else {
2082 ScopeReg = I.getOperand(i: 5).getReg();
2083 MemSemEqReg = I.getOperand(i: 6).getReg();
2084 MemSemNeqReg = I.getOperand(i: 7).getReg();
2085 }
2086
2087 Register Cmp = I.getOperand(i: 3).getReg();
2088 Register Val = I.getOperand(i: 4).getReg();
2089 SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(VReg: Val);
2090 Register ACmpRes = createVirtualRegister(SpvType: SpvValTy, GR: &GR, MRI, MF: *I.getMF());
2091 const DebugLoc &DL = I.getDebugLoc();
2092 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpAtomicCompareExchange))
2093 .addDef(RegNo: ACmpRes)
2094 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvValTy))
2095 .addUse(RegNo: Ptr)
2096 .addUse(RegNo: ScopeReg)
2097 .addUse(RegNo: MemSemEqReg)
2098 .addUse(RegNo: MemSemNeqReg)
2099 .addUse(RegNo: Val)
2100 .addUse(RegNo: Cmp)
2101 .constrainAllUses(TII, TRI, RBI);
2102 SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2103 Register CmpSuccReg = createVirtualRegister(SpvType: BoolTy, GR: &GR, MRI, MF: *I.getMF());
2104 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpIEqual))
2105 .addDef(RegNo: CmpSuccReg)
2106 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
2107 .addUse(RegNo: ACmpRes)
2108 .addUse(RegNo: Cmp)
2109 .constrainAllUses(TII, TRI, RBI);
2110 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: *I.getMF());
2111 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2112 .addDef(RegNo: TmpReg)
2113 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2114 .addUse(RegNo: ACmpRes)
2115 .addUse(RegNo: GR.getOrCreateUndef(I, SpvType: ResType, TII))
2116 .addImm(Val: 0)
2117 .constrainAllUses(TII, TRI, RBI);
2118 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2119 .addDef(RegNo: ResVReg)
2120 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2121 .addUse(RegNo: CmpSuccReg)
2122 .addUse(RegNo: TmpReg)
2123 .addImm(Val: 1)
2124 .constrainAllUses(TII, TRI, RBI);
2125 return true;
2126}
2127
2128static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2129 switch (SC) {
2130 case SPIRV::StorageClass::DeviceOnlyINTEL:
2131 case SPIRV::StorageClass::HostOnlyINTEL:
2132 return true;
2133 default:
2134 return false;
2135 }
2136}
2137
2138// Returns true ResVReg is referred only from global vars and OpName's.
2139static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
2140 bool IsGRef = false;
2141 bool IsAllowedRefs =
2142 llvm::all_of(Range: MRI->use_instructions(Reg: ResVReg), P: [&IsGRef](auto const &It) {
2143 unsigned Opcode = It.getOpcode();
2144 if (Opcode == SPIRV::OpConstantComposite ||
2145 Opcode == SPIRV::OpVariable ||
2146 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2147 return IsGRef = true;
2148 return Opcode == SPIRV::OpName;
2149 });
2150 return IsAllowedRefs && IsGRef;
2151}
2152
2153Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2154 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2155 return GR.getSPIRVTypeID(SpirvType: GR.getOrCreateSPIRVPointerType(
2156 BaseType: Type::getInt8Ty(C&: I.getMF()->getFunction().getContext()), I, SC));
2157}
2158
2159MachineInstrBuilder
2160SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2161 Register Src, Register DestType,
2162 uint32_t Opcode) const {
2163 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2164 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
2165 .addDef(RegNo: Dest)
2166 .addUse(RegNo: DestType)
2167 .addImm(Val: Opcode)
2168 .addUse(RegNo: Src);
2169}
2170
2171MachineInstrBuilder
2172SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2173 SPIRVTypeInst SrcPtrTy) const {
2174 SPIRVTypeInst GenericPtrTy =
2175 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2176 Register Tmp = MRI->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
2177 MRI->setType(VReg: Tmp, Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(
2178 SC: SPIRV::StorageClass::Generic),
2179 SizeInBits: GR.getPointerSize()));
2180 MachineFunction *MF = I.getParent()->getParent();
2181 GR.assignSPIRVTypeToVReg(Type: GenericPtrTy, VReg: Tmp, MF: *MF);
2182 MachineInstrBuilder MIB = buildSpecConstantOp(
2183 I, Dest: Tmp, Src: SrcPtr, DestType: GR.getSPIRVTypeID(SpirvType: GenericPtrTy),
2184 Opcode: static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
2185 GR.add(Obj: MIB.getInstr(), MI: MIB);
2186 return MIB;
2187}
2188
2189// In SPIR-V address space casting can only happen to and from the Generic
2190// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
2191// pointers to and from Generic pointers. As such, we can convert e.g. from
2192// Workgroup to Function by going via a Generic pointer as an intermediary. All
2193// other combinations can only be done by a bitcast, and are probably not safe.
2194bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
2195 SPIRVTypeInst ResType,
2196 MachineInstr &I) const {
2197 MachineBasicBlock &BB = *I.getParent();
2198 const DebugLoc &DL = I.getDebugLoc();
2199
2200 Register SrcPtr = I.getOperand(i: 1).getReg();
2201 SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(VReg: SrcPtr);
2202
2203 // don't generate a cast for a null that may be represented by OpTypeInt
2204 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
2205 ResType->getOpcode() != SPIRV::OpTypePointer)
2206 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2207
2208 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(Type: SrcPtrTy);
2209 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(Type: ResType);
2210
2211 if (isASCastInGVar(MRI, ResVReg)) {
2212 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
2213 // are expressed by OpSpecConstantOp with an Opcode.
2214 // TODO: maybe insert a check whether the Kernel capability was declared and
2215 // so PtrCastToGeneric/GenericCastToPtr are available.
2216 unsigned SpecOpcode =
2217 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC)
2218 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
2219 : (SrcSC == SPIRV::StorageClass::Generic &&
2220 isGenericCastablePtr(SC: DstSC)
2221 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
2222 : 0);
2223 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
2224 // correct value of ResType and use general i8* instead. Maybe this should
2225 // be addressed in the emit-intrinsic step to infer a correct
2226 // OpConstantComposite type.
2227 if (SpecOpcode) {
2228 buildSpecConstantOp(I, Dest: ResVReg, Src: SrcPtr, DestType: getUcharPtrTypeReg(I, SC: DstSC),
2229 Opcode: SpecOpcode)
2230 .constrainAllUses(TII, TRI, RBI);
2231 } else if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2232 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
2233 MIB.constrainAllUses(TII, TRI, RBI);
2234 buildSpecConstantOp(
2235 I, Dest: ResVReg, Src: MIB->getOperand(i: 0).getReg(), DestType: getUcharPtrTypeReg(I, SC: DstSC),
2236 Opcode: static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
2237 .constrainAllUses(TII, TRI, RBI);
2238 }
2239 return true;
2240 }
2241
2242 // don't generate a cast between identical storage classes
2243 if (SrcSC == DstSC)
2244 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2245
2246 if ((SrcSC == SPIRV::StorageClass::Function &&
2247 DstSC == SPIRV::StorageClass::Private) ||
2248 (DstSC == SPIRV::StorageClass::Function &&
2249 SrcSC == SPIRV::StorageClass::Private))
2250 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2251
2252 // Casting from an eligible pointer to Generic.
2253 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC))
2254 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2255 // Casting from Generic to an eligible pointer.
2256 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: DstSC))
2257 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2258 // Casting between 2 eligible pointers using Generic as an intermediary.
2259 if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2260 SPIRVTypeInst GenericPtrTy =
2261 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2262 Register Tmp = createVirtualRegister(SpvType: GenericPtrTy, GR: &GR, MRI, MF: MRI->getMF());
2263 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpPtrCastToGeneric))
2264 .addDef(RegNo: Tmp)
2265 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: GenericPtrTy))
2266 .addUse(RegNo: SrcPtr)
2267 .constrainAllUses(TII, TRI, RBI);
2268 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtr))
2269 .addDef(RegNo: ResVReg)
2270 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2271 .addUse(RegNo: Tmp)
2272 .constrainAllUses(TII, TRI, RBI);
2273 return true;
2274 }
2275
2276 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
2277 // be applied
2278 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
2279 return selectUnOp(ResVReg, ResType, I,
2280 Opcode: SPIRV::OpPtrCastToCrossWorkgroupINTEL);
2281 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(SC: DstSC))
2282 return selectUnOp(ResVReg, ResType, I,
2283 Opcode: SPIRV::OpCrossWorkgroupCastToPtrINTEL);
2284 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::Generic)
2285 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2286 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(SC: DstSC))
2287 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2288
2289 // Bitcast for pointers requires that the address spaces must match
2290 return false;
2291}
2292
2293static unsigned getFCmpOpcode(unsigned PredNum) {
2294 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2295 switch (Pred) {
2296 case CmpInst::FCMP_OEQ:
2297 return SPIRV::OpFOrdEqual;
2298 case CmpInst::FCMP_OGE:
2299 return SPIRV::OpFOrdGreaterThanEqual;
2300 case CmpInst::FCMP_OGT:
2301 return SPIRV::OpFOrdGreaterThan;
2302 case CmpInst::FCMP_OLE:
2303 return SPIRV::OpFOrdLessThanEqual;
2304 case CmpInst::FCMP_OLT:
2305 return SPIRV::OpFOrdLessThan;
2306 case CmpInst::FCMP_ONE:
2307 return SPIRV::OpFOrdNotEqual;
2308 case CmpInst::FCMP_ORD:
2309 return SPIRV::OpOrdered;
2310 case CmpInst::FCMP_UEQ:
2311 return SPIRV::OpFUnordEqual;
2312 case CmpInst::FCMP_UGE:
2313 return SPIRV::OpFUnordGreaterThanEqual;
2314 case CmpInst::FCMP_UGT:
2315 return SPIRV::OpFUnordGreaterThan;
2316 case CmpInst::FCMP_ULE:
2317 return SPIRV::OpFUnordLessThanEqual;
2318 case CmpInst::FCMP_ULT:
2319 return SPIRV::OpFUnordLessThan;
2320 case CmpInst::FCMP_UNE:
2321 return SPIRV::OpFUnordNotEqual;
2322 case CmpInst::FCMP_UNO:
2323 return SPIRV::OpUnordered;
2324 default:
2325 llvm_unreachable("Unknown predicate type for FCmp");
2326 }
2327}
2328
2329static unsigned getICmpOpcode(unsigned PredNum) {
2330 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2331 switch (Pred) {
2332 case CmpInst::ICMP_EQ:
2333 return SPIRV::OpIEqual;
2334 case CmpInst::ICMP_NE:
2335 return SPIRV::OpINotEqual;
2336 case CmpInst::ICMP_SGE:
2337 return SPIRV::OpSGreaterThanEqual;
2338 case CmpInst::ICMP_SGT:
2339 return SPIRV::OpSGreaterThan;
2340 case CmpInst::ICMP_SLE:
2341 return SPIRV::OpSLessThanEqual;
2342 case CmpInst::ICMP_SLT:
2343 return SPIRV::OpSLessThan;
2344 case CmpInst::ICMP_UGE:
2345 return SPIRV::OpUGreaterThanEqual;
2346 case CmpInst::ICMP_UGT:
2347 return SPIRV::OpUGreaterThan;
2348 case CmpInst::ICMP_ULE:
2349 return SPIRV::OpULessThanEqual;
2350 case CmpInst::ICMP_ULT:
2351 return SPIRV::OpULessThan;
2352 default:
2353 llvm_unreachable("Unknown predicate type for ICmp");
2354 }
2355}
2356
2357static unsigned getPtrCmpOpcode(unsigned Pred) {
2358 switch (static_cast<CmpInst::Predicate>(Pred)) {
2359 case CmpInst::ICMP_EQ:
2360 return SPIRV::OpPtrEqual;
2361 case CmpInst::ICMP_NE:
2362 return SPIRV::OpPtrNotEqual;
2363 default:
2364 llvm_unreachable("Unknown predicate type for pointer comparison");
2365 }
2366}
2367
2368// Return the logical operation, or abort if none exists.
2369static unsigned getBoolCmpOpcode(unsigned PredNum) {
2370 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2371 switch (Pred) {
2372 case CmpInst::ICMP_EQ:
2373 return SPIRV::OpLogicalEqual;
2374 case CmpInst::ICMP_NE:
2375 return SPIRV::OpLogicalNotEqual;
2376 default:
2377 llvm_unreachable("Unknown predicate type for Bool comparison");
2378 }
2379}
2380
2381static APFloat getZeroFP(const Type *LLVMFloatTy) {
2382 if (!LLVMFloatTy)
2383 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2384 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2385 case Type::HalfTyID:
2386 return APFloat::getZero(Sem: APFloat::IEEEhalf());
2387 default:
2388 case Type::FloatTyID:
2389 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2390 case Type::DoubleTyID:
2391 return APFloat::getZero(Sem: APFloat::IEEEdouble());
2392 }
2393}
2394
2395static APFloat getOneFP(const Type *LLVMFloatTy) {
2396 if (!LLVMFloatTy)
2397 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2398 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2399 case Type::HalfTyID:
2400 return APFloat::getOne(Sem: APFloat::IEEEhalf());
2401 default:
2402 case Type::FloatTyID:
2403 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2404 case Type::DoubleTyID:
2405 return APFloat::getOne(Sem: APFloat::IEEEdouble());
2406 }
2407}
2408
2409bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
2410 SPIRVTypeInst ResType,
2411 MachineInstr &I,
2412 unsigned OpAnyOrAll) const {
2413 assert(I.getNumOperands() == 3);
2414 assert(I.getOperand(2).isReg());
2415 MachineBasicBlock &BB = *I.getParent();
2416 Register InputRegister = I.getOperand(i: 2).getReg();
2417 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2418
2419 if (!InputType)
2420 report_fatal_error(reason: "Input Type could not be determined.");
2421
2422 bool IsBoolTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeBool);
2423 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
2424 if (IsBoolTy && !IsVectorTy) {
2425 assert(ResVReg == I.getOperand(0).getReg());
2426 return BuildCOPY(DestReg: ResVReg, SrcReg: InputRegister, I);
2427 }
2428
2429 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2430 unsigned SpirvNotEqualId =
2431 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
2432 SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
2433 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
2434 Register NotEqualReg = ResVReg;
2435
2436 if (IsVectorTy) {
2437 NotEqualReg =
2438 IsBoolTy ? InputRegister
2439 : createVirtualRegister(SpvType: SpvBoolTy, GR: &GR, MRI, MF: MRI->getMF());
2440 const unsigned NumElts = InputType->getOperand(i: 2).getImm();
2441 SpvBoolTy = GR.getOrCreateSPIRVVectorType(BaseType: SpvBoolTy, NumElements: NumElts, I, TII);
2442 }
2443
2444 if (!IsBoolTy) {
2445 Register ConstZeroReg =
2446 IsFloatTy ? buildZerosValF(ResType: InputType, I) : buildZerosVal(ResType: InputType, I);
2447
2448 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SpirvNotEqualId))
2449 .addDef(RegNo: NotEqualReg)
2450 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolTy))
2451 .addUse(RegNo: InputRegister)
2452 .addUse(RegNo: ConstZeroReg)
2453 .constrainAllUses(TII, TRI, RBI);
2454 }
2455
2456 if (IsVectorTy)
2457 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpAnyOrAll))
2458 .addDef(RegNo: ResVReg)
2459 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolScalarTy))
2460 .addUse(RegNo: NotEqualReg)
2461 .constrainAllUses(TII, TRI, RBI);
2462 return true;
2463}
2464
2465bool SPIRVInstructionSelector::selectAll(Register ResVReg,
2466 SPIRVTypeInst ResType,
2467 MachineInstr &I) const {
2468 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAll);
2469}
2470
2471bool SPIRVInstructionSelector::selectAny(Register ResVReg,
2472 SPIRVTypeInst ResType,
2473 MachineInstr &I) const {
2474 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAny);
2475}
2476
2477// Select the OpDot instruction for the given float dot
2478bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
2479 SPIRVTypeInst ResType,
2480 MachineInstr &I) const {
2481 assert(I.getNumOperands() == 4);
2482 assert(I.getOperand(2).isReg());
2483 assert(I.getOperand(3).isReg());
2484
2485 [[maybe_unused]] SPIRVTypeInst VecType =
2486 GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 2).getReg());
2487
2488 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2489 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2490 "dot product requires a vector of at least 2 components");
2491
2492 [[maybe_unused]] SPIRVTypeInst EltType =
2493 GR.getSPIRVTypeForVReg(VReg: VecType->getOperand(i: 1).getReg());
2494
2495 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
2496
2497 MachineBasicBlock &BB = *I.getParent();
2498 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDot))
2499 .addDef(RegNo: ResVReg)
2500 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2501 .addUse(RegNo: I.getOperand(i: 2).getReg())
2502 .addUse(RegNo: I.getOperand(i: 3).getReg())
2503 .constrainAllUses(TII, TRI, RBI);
2504 return true;
2505}
2506
2507bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
2508 SPIRVTypeInst ResType,
2509 MachineInstr &I,
2510 bool Signed) const {
2511 assert(I.getNumOperands() == 4);
2512 assert(I.getOperand(2).isReg());
2513 assert(I.getOperand(3).isReg());
2514 MachineBasicBlock &BB = *I.getParent();
2515
2516 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2517 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2518 .addDef(RegNo: ResVReg)
2519 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2520 .addUse(RegNo: I.getOperand(i: 2).getReg())
2521 .addUse(RegNo: I.getOperand(i: 3).getReg())
2522 .constrainAllUses(TII, TRI, RBI);
2523 return true;
2524}
2525
2526// Since pre-1.6 SPIRV has no integer dot implementation,
2527// expand by piecewise multiplying and adding the results
2528bool SPIRVInstructionSelector::selectIntegerDotExpansion(
2529 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2530 assert(I.getNumOperands() == 4);
2531 assert(I.getOperand(2).isReg());
2532 assert(I.getOperand(3).isReg());
2533 MachineBasicBlock &BB = *I.getParent();
2534
2535 // Multiply the vectors, then sum the results
2536 Register Vec0 = I.getOperand(i: 2).getReg();
2537 Register Vec1 = I.getOperand(i: 3).getReg();
2538 Register TmpVec = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2539 SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(VReg: Vec0);
2540
2541 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulV))
2542 .addDef(RegNo: TmpVec)
2543 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VecType))
2544 .addUse(RegNo: Vec0)
2545 .addUse(RegNo: Vec1)
2546 .constrainAllUses(TII, TRI, RBI);
2547
2548 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2549 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2550 "dot product requires a vector of at least 2 components");
2551
2552 Register Res = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2553 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2554 .addDef(RegNo: Res)
2555 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2556 .addUse(RegNo: TmpVec)
2557 .addImm(Val: 0)
2558 .constrainAllUses(TII, TRI, RBI);
2559
2560 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(Type: VecType); i++) {
2561 Register Elt = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2562
2563 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2564 .addDef(RegNo: Elt)
2565 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2566 .addUse(RegNo: TmpVec)
2567 .addImm(Val: i)
2568 .constrainAllUses(TII, TRI, RBI);
2569
2570 Register Sum = i < GR.getScalarOrVectorComponentCount(Type: VecType) - 1
2571 ? MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType))
2572 : ResVReg;
2573
2574 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2575 .addDef(RegNo: Sum)
2576 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2577 .addUse(RegNo: Res)
2578 .addUse(RegNo: Elt)
2579 .constrainAllUses(TII, TRI, RBI);
2580 Res = Sum;
2581 }
2582
2583 return true;
2584}
2585
2586bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
2587 SPIRVTypeInst ResType,
2588 MachineInstr &I) const {
2589 MachineBasicBlock &BB = *I.getParent();
2590 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsInf))
2591 .addDef(RegNo: ResVReg)
2592 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2593 .addUse(RegNo: I.getOperand(i: 2).getReg())
2594 .constrainAllUses(TII, TRI, RBI);
2595 return true;
2596}
2597
2598bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
2599 SPIRVTypeInst ResType,
2600 MachineInstr &I) const {
2601 MachineBasicBlock &BB = *I.getParent();
2602 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsNan))
2603 .addDef(RegNo: ResVReg)
2604 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2605 .addUse(RegNo: I.getOperand(i: 2).getReg())
2606 .constrainAllUses(TII, TRI, RBI);
2607 return true;
2608}
2609
2610template <bool Signed>
2611bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
2612 SPIRVTypeInst ResType,
2613 MachineInstr &I) const {
2614 assert(I.getNumOperands() == 5);
2615 assert(I.getOperand(2).isReg());
2616 assert(I.getOperand(3).isReg());
2617 assert(I.getOperand(4).isReg());
2618 MachineBasicBlock &BB = *I.getParent();
2619
2620 Register Acc = I.getOperand(i: 2).getReg();
2621 Register X = I.getOperand(i: 3).getReg();
2622 Register Y = I.getOperand(i: 4).getReg();
2623
2624 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2625 Register Dot = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2626 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2627 .addDef(RegNo: Dot)
2628 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2629 .addUse(RegNo: X)
2630 .addUse(RegNo: Y);
2631 MIB.addImm(Val: SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
2632 MIB.constrainAllUses(TII, TRI, RBI);
2633
2634 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2635 .addDef(RegNo: ResVReg)
2636 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2637 .addUse(RegNo: Dot)
2638 .addUse(RegNo: Acc)
2639 .constrainAllUses(TII, TRI, RBI);
2640 return true;
2641}
2642
2643// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
2644// extract the elements of the packed inputs, multiply them and add the result
2645// to the accumulator.
2646template <bool Signed>
2647bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
2648 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2649 assert(I.getNumOperands() == 5);
2650 assert(I.getOperand(2).isReg());
2651 assert(I.getOperand(3).isReg());
2652 assert(I.getOperand(4).isReg());
2653 MachineBasicBlock &BB = *I.getParent();
2654
2655 Register Acc = I.getOperand(i: 2).getReg();
2656 Register X = I.getOperand(i: 3).getReg();
2657 Register Y = I.getOperand(i: 4).getReg();
2658
2659 SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII);
2660 auto ExtractOp =
2661 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
2662
2663 bool ZeroAsNull = !STI.isShader();
2664 // Extract the i8 element, multiply and add it to the accumulator
2665 for (unsigned i = 0; i < 4; i++) {
2666 // A[i]
2667 Register AElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2668 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2669 .addDef(RegNo: AElt)
2670 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2671 .addUse(RegNo: X)
2672 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2673 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2674 .constrainAllUses(TII, TRI, RBI);
2675
2676 // B[i]
2677 Register BElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2678 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2679 .addDef(RegNo: BElt)
2680 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2681 .addUse(RegNo: Y)
2682 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2683 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2684 .constrainAllUses(TII, TRI, RBI);
2685
2686 // A[i] * B[i]
2687 Register Mul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2688 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulS))
2689 .addDef(RegNo: Mul)
2690 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2691 .addUse(RegNo: AElt)
2692 .addUse(RegNo: BElt)
2693 .constrainAllUses(TII, TRI, RBI);
2694
2695 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
2696 Register MaskMul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2697 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2698 .addDef(RegNo: MaskMul)
2699 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2700 .addUse(RegNo: Mul)
2701 .addUse(RegNo: GR.getOrCreateConstInt(Val: 0, I, SpvType: EltType, TII, ZeroAsNull))
2702 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2703 .constrainAllUses(TII, TRI, RBI);
2704
2705 // Acc = Acc + A[i] * B[i]
2706 Register Sum =
2707 i < 3 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass) : ResVReg;
2708 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2709 .addDef(RegNo: Sum)
2710 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2711 .addUse(RegNo: Acc)
2712 .addUse(RegNo: MaskMul)
2713 .constrainAllUses(TII, TRI, RBI);
2714
2715 Acc = Sum;
2716 }
2717
2718 return true;
2719}
2720
2721/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
2722/// does not have a saturate builtin.
2723bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
2724 SPIRVTypeInst ResType,
2725 MachineInstr &I) const {
2726 assert(I.getNumOperands() == 3);
2727 assert(I.getOperand(2).isReg());
2728 MachineBasicBlock &BB = *I.getParent();
2729 Register VZero = buildZerosValF(ResType, I);
2730 Register VOne = buildOnesValF(ResType, I);
2731
2732 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
2733 .addDef(RegNo: ResVReg)
2734 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2735 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2736 .addImm(Val: GL::FClamp)
2737 .addUse(RegNo: I.getOperand(i: 2).getReg())
2738 .addUse(RegNo: VZero)
2739 .addUse(RegNo: VOne)
2740 .constrainAllUses(TII, TRI, RBI);
2741 return true;
2742}
2743
2744bool SPIRVInstructionSelector::selectSign(Register ResVReg,
2745 SPIRVTypeInst ResType,
2746 MachineInstr &I) const {
2747 assert(I.getNumOperands() == 3);
2748 assert(I.getOperand(2).isReg());
2749 MachineBasicBlock &BB = *I.getParent();
2750 Register InputRegister = I.getOperand(i: 2).getReg();
2751 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2752 auto &DL = I.getDebugLoc();
2753
2754 if (!InputType)
2755 report_fatal_error(reason: "Input Type could not be determined.");
2756
2757 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2758
2759 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(Type: InputType);
2760 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2761
2762 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
2763
2764 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
2765 Register SignReg = NeedsConversion
2766 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass)
2767 : ResVReg;
2768
2769 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpExtInst))
2770 .addDef(RegNo: SignReg)
2771 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: InputType))
2772 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2773 .addImm(Val: SignOpcode)
2774 .addUse(RegNo: InputRegister)
2775 .constrainAllUses(TII, TRI, RBI);
2776
2777 if (NeedsConversion) {
2778 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
2779 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: ConvertOpcode))
2780 .addDef(RegNo: ResVReg)
2781 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2782 .addUse(RegNo: SignReg)
2783 .constrainAllUses(TII, TRI, RBI);
2784 }
2785
2786 return true;
2787}
2788
2789bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
2790 SPIRVTypeInst ResType,
2791 MachineInstr &I,
2792 unsigned Opcode) const {
2793 MachineBasicBlock &BB = *I.getParent();
2794 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2795
2796 auto BMI = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2797 .addDef(RegNo: ResVReg)
2798 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2799 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I,
2800 SpvType: IntTy, TII, ZeroAsNull: !STI.isShader()));
2801
2802 for (unsigned J = 2; J < I.getNumOperands(); J++) {
2803 BMI.addUse(RegNo: I.getOperand(i: J).getReg());
2804 }
2805
2806 BMI.constrainAllUses(TII, TRI, RBI);
2807 return true;
2808}
2809
2810bool SPIRVInstructionSelector::selectWaveActiveCountBits(
2811 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2812
2813 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2814 SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(BaseType: IntTy, NumElements: 4, I, TII);
2815 Register BallotReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BallotType));
2816 if (!selectWaveOpInst(ResVReg: BallotReg, ResType: BallotType, I,
2817 Opcode: SPIRV::OpGroupNonUniformBallot))
2818 return false;
2819
2820 MachineBasicBlock &BB = *I.getParent();
2821 BuildMI(BB, I, MIMD: I.getDebugLoc(),
2822 MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2823 .addDef(RegNo: ResVReg)
2824 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2825 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2826 ZeroAsNull: !STI.isShader()))
2827 .addImm(Val: SPIRV::GroupOperation::Reduce)
2828 .addUse(RegNo: BallotReg)
2829 .constrainAllUses(TII, TRI, RBI);
2830
2831 return true;
2832}
2833
2834bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg,
2835 SPIRVTypeInst ResType,
2836 MachineInstr &I) const {
2837
2838 assert(I.getNumOperands() == 3);
2839
2840 auto Op = I.getOperand(i: 2);
2841 assert(Op.isReg());
2842
2843 MachineBasicBlock &BB = *I.getParent();
2844 DebugLoc DL = I.getDebugLoc();
2845
2846 Register InputRegister = Op.getReg();
2847 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2848
2849 if (!InputType)
2850 report_fatal_error(reason: "Input Type could not be determined.");
2851
2852 if (InputType->getOpcode() != SPIRV::OpTypeBool)
2853 report_fatal_error(reason: "WavePrefixBitCount requires boolean input");
2854
2855 // Types
2856 SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2857
2858 // Ballot result type: vector<uint32>
2859 // Match DXC: %v4uint for Subgroup size
2860 SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(BaseType: Int32Ty, NumElements: 4, I, TII);
2861
2862 // Create a vreg for the ballot result
2863 Register BallotVReg = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2864
2865 // 1. OpGroupNonUniformBallot
2866 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallot))
2867 .addDef(RegNo: BallotVReg)
2868 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BallotTy))
2869 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
2870 .addUse(RegNo: InputRegister)
2871 .constrainAllUses(TII, TRI, RBI);
2872
2873 // 2. OpGroupNonUniformBallotBitCount
2874 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2875 .addDef(RegNo: ResVReg)
2876 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2877 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
2878 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
2879 .addUse(RegNo: BallotVReg)
2880 .constrainAllUses(TII, TRI, RBI);
2881
2882 return true;
2883}
2884
2885bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
2886 SPIRVTypeInst ResType,
2887 MachineInstr &I,
2888 bool IsUnsigned) const {
2889 return selectWaveReduce(
2890 ResVReg, ResType, I, IsUnsigned,
2891 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2892 const bool IsFloatTy =
2893 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2894 const unsigned IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
2895 : SPIRV::OpGroupNonUniformSMax;
2896 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
2897 });
2898}
2899
2900bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
2901 SPIRVTypeInst ResType,
2902 MachineInstr &I,
2903 bool IsUnsigned) const {
2904 return selectWaveReduce(
2905 ResVReg, ResType, I, IsUnsigned,
2906 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2907 const bool IsFloatTy =
2908 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2909 const unsigned IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
2910 : SPIRV::OpGroupNonUniformSMin;
2911 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
2912 });
2913}
2914
2915bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
2916 SPIRVTypeInst ResType,
2917 MachineInstr &I) const {
2918 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
2919 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2920 bool IsFloatTy = GR.isScalarOrVectorOfType(
2921 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2922 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
2923 : SPIRV::OpGroupNonUniformIAdd;
2924 });
2925}
2926
2927template <typename PickOpcodeFn>
2928bool SPIRVInstructionSelector::selectWaveReduce(
2929 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
2930 PickOpcodeFn &&PickOpcode) const {
2931 assert(I.getNumOperands() == 3);
2932 assert(I.getOperand(2).isReg());
2933 MachineBasicBlock &BB = *I.getParent();
2934 Register InputRegister = I.getOperand(i: 2).getReg();
2935 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2936
2937 if (!InputType)
2938 report_fatal_error(reason: "Input Type could not be determined.");
2939
2940 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2941 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
2942 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2943 .addDef(RegNo: ResVReg)
2944 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2945 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2946 ZeroAsNull: !STI.isShader()))
2947 .addImm(Val: SPIRV::GroupOperation::Reduce)
2948 .addUse(RegNo: I.getOperand(i: 2).getReg())
2949 .constrainAllUses(TII, TRI, RBI);
2950 return true;
2951}
2952
2953bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
2954 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2955 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
2956 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2957 bool IsFloatTy = GR.isScalarOrVectorOfType(
2958 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2959 return IsFloatTy
2960 ? SPIRV::OpGroupNonUniformFAdd
2961 : SPIRV::OpGroupNonUniformIAdd;
2962 });
2963}
2964
2965bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
2966 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2967 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
2968 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
2969 bool IsFloatTy = GR.isScalarOrVectorOfType(
2970 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2971 return IsFloatTy
2972 ? SPIRV::OpGroupNonUniformFMul
2973 : SPIRV::OpGroupNonUniformIMul;
2974 });
2975}
2976
2977template <typename PickOpcodeFn>
2978bool SPIRVInstructionSelector::selectWaveExclusiveScan(
2979 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
2980 PickOpcodeFn &&PickOpcode) const {
2981 assert(I.getNumOperands() == 3);
2982 assert(I.getOperand(2).isReg());
2983 MachineBasicBlock &BB = *I.getParent();
2984 Register InputRegister = I.getOperand(i: 2).getReg();
2985 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2986
2987 if (!InputType)
2988 report_fatal_error(reason: "Input Type could not be determined.");
2989
2990 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2991 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
2992 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2993 .addDef(RegNo: ResVReg)
2994 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2995 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2996 ZeroAsNull: !STI.isShader()))
2997 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
2998 .addUse(RegNo: I.getOperand(i: 2).getReg())
2999 .constrainAllUses(TII, TRI, RBI);
3000 return true;
3001}
3002
3003bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
3004 SPIRVTypeInst ResType,
3005 MachineInstr &I) const {
3006 MachineBasicBlock &BB = *I.getParent();
3007 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitReverse))
3008 .addDef(RegNo: ResVReg)
3009 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3010 .addUse(RegNo: I.getOperand(i: 1).getReg())
3011 .constrainAllUses(TII, TRI, RBI);
3012 return true;
3013}
3014
3015bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
3016 SPIRVTypeInst ResType,
3017 MachineInstr &I) const {
3018 // There is no way to implement `freeze` correctly without support on SPIR-V
3019 // standard side, but we may at least address a simple (static) case when
3020 // undef/poison value presence is obvious. The main benefit of even
3021 // incomplete `freeze` support is preventing of translation from crashing due
3022 // to lack of support on legalization and instruction selection steps.
3023 if (!I.getOperand(i: 0).isReg() || !I.getOperand(i: 1).isReg())
3024 return false;
3025 Register OpReg = I.getOperand(i: 1).getReg();
3026 if (MachineInstr *Def = MRI->getVRegDef(Reg: OpReg)) {
3027 if (Def->getOpcode() == TargetOpcode::COPY)
3028 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
3029 Register Reg;
3030 switch (Def->getOpcode()) {
3031 case SPIRV::ASSIGN_TYPE:
3032 if (MachineInstr *AssignToDef =
3033 MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())) {
3034 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
3035 Reg = Def->getOperand(i: 2).getReg();
3036 }
3037 break;
3038 case SPIRV::OpUndef:
3039 Reg = Def->getOperand(i: 1).getReg();
3040 break;
3041 }
3042 unsigned DestOpCode;
3043 if (Reg.isValid()) {
3044 DestOpCode = SPIRV::OpConstantNull;
3045 } else {
3046 DestOpCode = TargetOpcode::COPY;
3047 Reg = OpReg;
3048 }
3049 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DestOpCode))
3050 .addDef(RegNo: I.getOperand(i: 0).getReg())
3051 .addUse(RegNo: Reg)
3052 .constrainAllUses(TII, TRI, RBI);
3053 return true;
3054 }
3055 return false;
3056}
3057
3058bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
3059 SPIRVTypeInst ResType,
3060 MachineInstr &I) const {
3061 unsigned N = 0;
3062 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3063 N = GR.getScalarOrVectorComponentCount(Type: ResType);
3064 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3065 N = getArrayComponentCount(MRI, ResType);
3066 else
3067 report_fatal_error(reason: "Cannot select G_BUILD_VECTOR with a non-vector result");
3068 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
3069 report_fatal_error(reason: "G_BUILD_VECTOR and the result type are inconsistent");
3070
3071 // check if we may construct a constant vector
3072 bool IsConst = true;
3073 for (unsigned i = I.getNumExplicitDefs();
3074 i < I.getNumExplicitOperands() && IsConst; ++i)
3075 if (!isConstReg(MRI, OpReg: I.getOperand(i).getReg()))
3076 IsConst = false;
3077
3078 if (!IsConst && N < 2)
3079 report_fatal_error(
3080 reason: "There must be at least two constituent operands in a vector");
3081
3082 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3083 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3084 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
3085 : SPIRV::OpCompositeConstruct))
3086 .addDef(RegNo: ResVReg)
3087 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3088 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
3089 MIB.addUse(RegNo: I.getOperand(i).getReg());
3090 MIB.constrainAllUses(TII, TRI, RBI);
3091 return true;
3092}
3093
3094bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
3095 SPIRVTypeInst ResType,
3096 MachineInstr &I) const {
3097 unsigned N = 0;
3098 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3099 N = GR.getScalarOrVectorComponentCount(Type: ResType);
3100 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3101 N = getArrayComponentCount(MRI, ResType);
3102 else
3103 report_fatal_error(reason: "Cannot select G_SPLAT_VECTOR with a non-vector result");
3104
3105 unsigned OpIdx = I.getNumExplicitDefs();
3106 if (!I.getOperand(i: OpIdx).isReg())
3107 report_fatal_error(reason: "Unexpected argument in G_SPLAT_VECTOR");
3108
3109 // check if we may construct a constant vector
3110 Register OpReg = I.getOperand(i: OpIdx).getReg();
3111 bool IsConst = isConstReg(MRI, OpReg);
3112
3113 if (!IsConst && N < 2)
3114 report_fatal_error(
3115 reason: "There must be at least two constituent operands in a vector");
3116
3117 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3118 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3119 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
3120 : SPIRV::OpCompositeConstruct))
3121 .addDef(RegNo: ResVReg)
3122 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3123 for (unsigned i = 0; i < N; ++i)
3124 MIB.addUse(RegNo: OpReg);
3125 MIB.constrainAllUses(TII, TRI, RBI);
3126 return true;
3127}
3128
3129bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
3130 SPIRVTypeInst ResType,
3131 MachineInstr &I) const {
3132
3133 unsigned Opcode;
3134
3135 if (STI.canUseExtension(
3136 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
3137 STI.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6))) {
3138 Opcode = SPIRV::OpDemoteToHelperInvocation;
3139 } else {
3140 Opcode = SPIRV::OpKill;
3141 // OpKill must be the last operation of any basic block.
3142 if (MachineInstr *NextI = I.getNextNode()) {
3143 GR.invalidateMachineInstr(MI: NextI);
3144 NextI->eraseFromParent();
3145 }
3146 }
3147
3148 MachineBasicBlock &BB = *I.getParent();
3149 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3150 .constrainAllUses(TII, TRI, RBI);
3151 return true;
3152}
3153
3154bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
3155 SPIRVTypeInst ResType, unsigned CmpOpc,
3156 MachineInstr &I) const {
3157 Register Cmp0 = I.getOperand(i: 2).getReg();
3158 Register Cmp1 = I.getOperand(i: 3).getReg();
3159 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
3160 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
3161 "CMP operands should have the same type");
3162 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CmpOpc))
3163 .addDef(RegNo: ResVReg)
3164 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3165 .addUse(RegNo: Cmp0)
3166 .addUse(RegNo: Cmp1)
3167 .setMIFlags(I.getFlags())
3168 .constrainAllUses(TII, TRI, RBI);
3169 return true;
3170}
3171
3172bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
3173 SPIRVTypeInst ResType,
3174 MachineInstr &I) const {
3175 auto Pred = I.getOperand(i: 1).getPredicate();
3176 unsigned CmpOpc;
3177
3178 Register CmpOperand = I.getOperand(i: 2).getReg();
3179 if (GR.isScalarOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypePointer))
3180 CmpOpc = getPtrCmpOpcode(Pred);
3181 else if (GR.isScalarOrVectorOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypeBool))
3182 CmpOpc = getBoolCmpOpcode(PredNum: Pred);
3183 else
3184 CmpOpc = getICmpOpcode(PredNum: Pred);
3185 return selectCmp(ResVReg, ResType, CmpOpc, I);
3186}
3187
3188Register
3189SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
3190 SPIRVTypeInst ResType) const {
3191 Type *LLVMTy = IntegerType::get(C&: GR.CurMF->getFunction().getContext(), NumBits: 32);
3192 SPIRVTypeInst SpvI32Ty =
3193 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3194 // Find a constant in DT or build a new one.
3195 auto ConstInt = ConstantInt::get(Ty: LLVMTy, V: Val);
3196 Register NewReg = GR.find(V: ConstInt, MF: GR.CurMF);
3197 if (!NewReg.isValid()) {
3198 NewReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
3199 MachineBasicBlock &BB = *I.getParent();
3200 MachineInstr *MI =
3201 Val == 0
3202 ? BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3203 .addDef(RegNo: NewReg)
3204 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3205 : BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantI))
3206 .addDef(RegNo: NewReg)
3207 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3208 .addImm(Val: APInt(32, Val).getZExtValue());
3209 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
3210 GR.add(V: ConstInt, MI);
3211 }
3212 return NewReg;
3213}
3214
3215bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
3216 SPIRVTypeInst ResType,
3217 MachineInstr &I) const {
3218 unsigned CmpOp = getFCmpOpcode(PredNum: I.getOperand(i: 1).getPredicate());
3219 return selectCmp(ResVReg, ResType, CmpOpc: CmpOp, I);
3220}
3221
3222Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
3223 MachineInstr &I) const {
3224 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3225 bool ZeroAsNull = !STI.isShader();
3226 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3227 return GR.getOrCreateConstVector(Val: 0UL, I, SpvType: ResType, TII, ZeroAsNull);
3228 return GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
3229}
3230
3231bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
3232 Register Reg) const {
3233 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(VReg: Reg);
3234 if (!Type)
3235 return false;
3236 SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type);
3237 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
3238 return false;
3239
3240 auto IsZero = [this](Register Reg) {
3241 MachineInstr *Def = getDefInstrMaybeConstant(ConstReg&: Reg, MRI);
3242 if (!Def)
3243 return false;
3244
3245 if (Def->getOpcode() == SPIRV::OpConstantNull)
3246 return true;
3247
3248 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
3249 Def->getOpcode() == SPIRV::OpConstantI)
3250 return getIConstVal(ConstReg: Reg, MRI) == 0;
3251
3252 return false;
3253 };
3254
3255 if (IsZero(Reg))
3256 return true;
3257
3258 MachineInstr *Def = MRI->getVRegDef(Reg);
3259 if (!Def)
3260 return false;
3261
3262 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
3263 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
3264 cast<GIntrinsic>(Val: Def)->getIntrinsicID() ==
3265 Intrinsic::spv_const_composite)) {
3266 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
3267 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
3268 if (!IsZero(Def->getOperand(i).getReg()))
3269 return false;
3270 }
3271 return true;
3272 }
3273
3274 return false;
3275}
3276
3277Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
3278 MachineInstr &I) const {
3279 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3280 bool ZeroAsNull = !STI.isShader();
3281 APFloat VZero = getZeroFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3282 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3283 return GR.getOrCreateConstVector(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3284 return GR.getOrCreateConstFP(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3285}
3286
3287Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
3288 MachineInstr &I) const {
3289 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3290 bool ZeroAsNull = !STI.isShader();
3291 APFloat VOne = getOneFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3292 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3293 return GR.getOrCreateConstVector(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3294 return GR.getOrCreateConstFP(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3295}
3296
3297Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
3298 SPIRVTypeInst ResType,
3299 MachineInstr &I) const {
3300 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3301 APInt One =
3302 AllOnes ? APInt::getAllOnes(numBits: BitWidth) : APInt::getOneBitSet(numBits: BitWidth, BitNo: 0);
3303 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3304 return GR.getOrCreateConstVector(Val: One, I, SpvType: ResType, TII);
3305 return GR.getOrCreateConstInt(Val: One, I, SpvType: ResType, TII);
3306}
3307
3308bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
3309 SPIRVTypeInst ResType,
3310 MachineInstr &I) const {
3311 Register SelectFirstArg = I.getOperand(i: 2).getReg();
3312 Register SelectSecondArg = I.getOperand(i: 3).getReg();
3313 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
3314 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
3315
3316 bool IsFloatTy =
3317 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypeFloat);
3318 bool IsPtrTy =
3319 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypePointer);
3320 bool IsVectorTy = GR.getSPIRVTypeForVReg(VReg: SelectFirstArg)->getOpcode() ==
3321 SPIRV::OpTypeVector;
3322
3323 bool IsScalarBool =
3324 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
3325 unsigned Opcode;
3326 if (IsVectorTy) {
3327 if (IsFloatTy) {
3328 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
3329 } else if (IsPtrTy) {
3330 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
3331 } else {
3332 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
3333 }
3334 } else {
3335 if (IsFloatTy) {
3336 Opcode = IsScalarBool ? SPIRV::OpSelectSFSCond : SPIRV::OpSelectVFVCond;
3337 } else if (IsPtrTy) {
3338 Opcode = IsScalarBool ? SPIRV::OpSelectSPSCond : SPIRV::OpSelectVPVCond;
3339 } else {
3340 Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3341 }
3342 }
3343 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3344 .addDef(RegNo: ResVReg)
3345 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3346 .addUse(RegNo: I.getOperand(i: 1).getReg())
3347 .addUse(RegNo: SelectFirstArg)
3348 .addUse(RegNo: SelectSecondArg)
3349 .constrainAllUses(TII, TRI, RBI);
3350 return true;
3351}
3352
3353bool SPIRVInstructionSelector::selectSelectDefaultArgs(Register ResVReg,
3354 SPIRVTypeInst ResType,
3355 MachineInstr &I,
3356 bool IsSigned) const {
3357 // To extend a bool, we need to use OpSelect between constants.
3358 Register ZeroReg = buildZerosVal(ResType, I);
3359 Register OneReg = buildOnesVal(AllOnes: IsSigned, ResType, I);
3360 bool IsScalarBool =
3361 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
3362 unsigned Opcode =
3363 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3364 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3365 .addDef(RegNo: ResVReg)
3366 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3367 .addUse(RegNo: I.getOperand(i: 1).getReg())
3368 .addUse(RegNo: OneReg)
3369 .addUse(RegNo: ZeroReg)
3370 .constrainAllUses(TII, TRI, RBI);
3371 return true;
3372}
3373
3374bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
3375 SPIRVTypeInst ResType,
3376 MachineInstr &I, bool IsSigned,
3377 unsigned Opcode) const {
3378 Register SrcReg = I.getOperand(i: 1).getReg();
3379 // We can convert bool value directly to float type without OpConvert*ToF,
3380 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
3381 if (GR.isScalarOrVectorOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool)) {
3382 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3383 SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
3384 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
3385 const unsigned NumElts = ResType->getOperand(i: 2).getImm();
3386 TmpType = GR.getOrCreateSPIRVVectorType(BaseType: TmpType, NumElements: NumElts, I, TII);
3387 }
3388 SrcReg = createVirtualRegister(SpvType: TmpType, GR: &GR, MRI, MF: MRI->getMF());
3389 selectSelectDefaultArgs(ResVReg: SrcReg, ResType: TmpType, I, IsSigned: false);
3390 }
3391 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {SrcReg}, Opcode);
3392}
3393
3394bool SPIRVInstructionSelector::selectExt(Register ResVReg,
3395 SPIRVTypeInst ResType, MachineInstr &I,
3396 bool IsSigned) const {
3397 Register SrcReg = I.getOperand(i: 1).getReg();
3398 if (GR.isScalarOrVectorOfType(VReg: SrcReg, TypeOpcode: SPIRV::OpTypeBool))
3399 return selectSelectDefaultArgs(ResVReg, ResType, I, IsSigned);
3400
3401 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
3402 if (ResType == SrcType)
3403 return BuildCOPY(DestReg: ResVReg, SrcReg, I);
3404
3405 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3406 return selectUnOp(ResVReg, ResType, I, Opcode);
3407}
3408
3409bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
3410 SPIRVTypeInst ResType,
3411 MachineInstr &I,
3412 bool IsSigned) const {
3413 MachineIRBuilder MIRBuilder(I);
3414 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3415 MachineBasicBlock &BB = *I.getParent();
3416 // Ensure we have bool.
3417 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
3418 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
3419 if (N > 1)
3420 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
3421 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
3422 // Build less-than-equal and less-than.
3423 // TODO: replace with one-liner createVirtualRegister() from
3424 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
3425 Register IsLessEqReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3426 MRI->setType(VReg: IsLessEqReg, Ty: LLT::scalar(SizeInBits: 64));
3427 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessEqReg, MF: MIRBuilder.getMF());
3428 BuildMI(BB, I, MIMD: I.getDebugLoc(),
3429 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
3430 .addDef(RegNo: IsLessEqReg)
3431 .addUse(RegNo: BoolTypeReg)
3432 .addUse(RegNo: I.getOperand(i: 1).getReg())
3433 .addUse(RegNo: I.getOperand(i: 2).getReg())
3434 .constrainAllUses(TII, TRI, RBI);
3435 Register IsLessReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3436 MRI->setType(VReg: IsLessReg, Ty: LLT::scalar(SizeInBits: 64));
3437 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessReg, MF: MIRBuilder.getMF());
3438 BuildMI(BB, I, MIMD: I.getDebugLoc(),
3439 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
3440 .addDef(RegNo: IsLessReg)
3441 .addUse(RegNo: BoolTypeReg)
3442 .addUse(RegNo: I.getOperand(i: 1).getReg())
3443 .addUse(RegNo: I.getOperand(i: 2).getReg())
3444 .constrainAllUses(TII, TRI, RBI);
3445 // Build selects.
3446 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
3447 Register NegOneOrZeroReg =
3448 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3449 MRI->setType(VReg: NegOneOrZeroReg, Ty: LLT::scalar(SizeInBits: 64));
3450 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: NegOneOrZeroReg, MF: MIRBuilder.getMF());
3451 unsigned SelectOpcode =
3452 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
3453 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3454 .addDef(RegNo: NegOneOrZeroReg)
3455 .addUse(RegNo: ResTypeReg)
3456 .addUse(RegNo: IsLessReg)
3457 .addUse(RegNo: buildOnesVal(AllOnes: true, ResType, I)) // -1
3458 .addUse(RegNo: buildZerosVal(ResType, I))
3459 .constrainAllUses(TII, TRI, RBI);
3460 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3461 .addDef(RegNo: ResVReg)
3462 .addUse(RegNo: ResTypeReg)
3463 .addUse(RegNo: IsLessEqReg)
3464 .addUse(RegNo: NegOneOrZeroReg) // -1 or 0
3465 .addUse(RegNo: buildOnesVal(AllOnes: false, ResType, I))
3466 .constrainAllUses(TII, TRI, RBI);
3467 return true;
3468}
3469
3470bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
3471 Register ResVReg,
3472 MachineInstr &I,
3473 SPIRVTypeInst IntTy,
3474 SPIRVTypeInst BoolTy) const {
3475 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
3476 Register BitIntReg = createVirtualRegister(SpvType: IntTy, GR: &GR, MRI, MF: MRI->getMF());
3477 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
3478 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
3479 Register Zero = buildZerosVal(ResType: IntTy, I);
3480 Register One = buildOnesVal(AllOnes: false, ResType: IntTy, I);
3481 MachineBasicBlock &BB = *I.getParent();
3482 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3483 .addDef(RegNo: BitIntReg)
3484 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntTy))
3485 .addUse(RegNo: IntReg)
3486 .addUse(RegNo: One)
3487 .constrainAllUses(TII, TRI, RBI);
3488 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
3489 .addDef(RegNo: ResVReg)
3490 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
3491 .addUse(RegNo: BitIntReg)
3492 .addUse(RegNo: Zero)
3493 .constrainAllUses(TII, TRI, RBI);
3494 return true;
3495}
3496
3497bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
3498 SPIRVTypeInst ResType,
3499 MachineInstr &I) const {
3500 Register IntReg = I.getOperand(i: 1).getReg();
3501 const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(VReg: IntReg);
3502 if (GR.isScalarOrVectorOfType(VReg: ResVReg, TypeOpcode: SPIRV::OpTypeBool))
3503 return selectIntToBool(IntReg, ResVReg, I, IntTy: ArgType, BoolTy: ResType);
3504 if (ArgType == ResType)
3505 return BuildCOPY(DestReg: ResVReg, SrcReg: IntReg, I);
3506 bool IsSigned = GR.isScalarOrVectorSigned(Type: ResType);
3507 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3508 return selectUnOp(ResVReg, ResType, I, Opcode);
3509}
3510
3511bool SPIRVInstructionSelector::selectConst(Register ResVReg,
3512 SPIRVTypeInst ResType,
3513 MachineInstr &I) const {
3514 unsigned Opcode = I.getOpcode();
3515 unsigned TpOpcode = ResType->getOpcode();
3516 Register Reg;
3517 if (TpOpcode == SPIRV::OpTypePointer || TpOpcode == SPIRV::OpTypeEvent) {
3518 assert(Opcode == TargetOpcode::G_CONSTANT &&
3519 I.getOperand(1).getCImm()->isZero());
3520 MachineBasicBlock &DepMBB = I.getMF()->front();
3521 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
3522 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, SpvType: ResType);
3523 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
3524 Reg = GR.getOrCreateConstFP(Val: I.getOperand(i: 1).getFPImm()->getValue(), I,
3525 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3526 } else {
3527 Reg = GR.getOrCreateConstInt(Val: I.getOperand(i: 1).getCImm()->getValue(), I,
3528 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3529 }
3530 return Reg == ResVReg ? true : BuildCOPY(DestReg: ResVReg, SrcReg: Reg, I);
3531}
3532
3533bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
3534 SPIRVTypeInst ResType,
3535 MachineInstr &I) const {
3536 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3537 .addDef(RegNo: ResVReg)
3538 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3539 .constrainAllUses(TII, TRI, RBI);
3540 return true;
3541}
3542
3543bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
3544 SPIRVTypeInst ResType,
3545 MachineInstr &I) const {
3546 MachineBasicBlock &BB = *I.getParent();
3547 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
3548 .addDef(RegNo: ResVReg)
3549 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3550 // object to insert
3551 .addUse(RegNo: I.getOperand(i: 3).getReg())
3552 // composite to insert into
3553 .addUse(RegNo: I.getOperand(i: 2).getReg());
3554 for (unsigned i = 4; i < I.getNumOperands(); i++)
3555 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3556 MIB.constrainAllUses(TII, TRI, RBI);
3557 return true;
3558}
3559
3560bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
3561 SPIRVTypeInst ResType,
3562 MachineInstr &I) const {
3563 Type *MaybeResTy = nullptr;
3564 StringRef ResName;
3565 if (GR.findValueAttrs(Key: &I, Ty&: MaybeResTy, Name&: ResName) &&
3566 MaybeResTy != GR.getTypeForSPIRVType(Ty: ResType)) {
3567 assert((!MaybeResTy || MaybeResTy->isAggregateType()) &&
3568 "Expected aggregate type for extractv instruction");
3569 ResType = GR.getOrCreateSPIRVType(Type: MaybeResTy, I,
3570 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
3571 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *I.getMF());
3572 }
3573 MachineBasicBlock &BB = *I.getParent();
3574 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
3575 .addDef(RegNo: ResVReg)
3576 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3577 .addUse(RegNo: I.getOperand(i: 2).getReg());
3578 for (unsigned i = 3; i < I.getNumOperands(); i++)
3579 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3580 MIB.constrainAllUses(TII, TRI, RBI);
3581 return true;
3582}
3583
3584bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
3585 SPIRVTypeInst ResType,
3586 MachineInstr &I) const {
3587 if (getImm(MO: I.getOperand(i: 4), MRI))
3588 return selectInsertVal(ResVReg, ResType, I);
3589 MachineBasicBlock &BB = *I.getParent();
3590 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorInsertDynamic))
3591 .addDef(RegNo: ResVReg)
3592 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3593 .addUse(RegNo: I.getOperand(i: 2).getReg())
3594 .addUse(RegNo: I.getOperand(i: 3).getReg())
3595 .addUse(RegNo: I.getOperand(i: 4).getReg())
3596 .constrainAllUses(TII, TRI, RBI);
3597 return true;
3598}
3599
3600bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
3601 SPIRVTypeInst ResType,
3602 MachineInstr &I) const {
3603 if (getImm(MO: I.getOperand(i: 3), MRI))
3604 return selectExtractVal(ResVReg, ResType, I);
3605 MachineBasicBlock &BB = *I.getParent();
3606 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorExtractDynamic))
3607 .addDef(RegNo: ResVReg)
3608 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3609 .addUse(RegNo: I.getOperand(i: 2).getReg())
3610 .addUse(RegNo: I.getOperand(i: 3).getReg())
3611 .constrainAllUses(TII, TRI, RBI);
3612 return true;
3613}
3614
3615bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
3616 SPIRVTypeInst ResType,
3617 MachineInstr &I) const {
3618 const bool IsGEPInBounds = I.getOperand(i: 2).getImm();
3619
3620 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
3621 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
3622 // we have to use Op[InBounds]AccessChain.
3623 const unsigned Opcode = STI.isLogicalSPIRV()
3624 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
3625 : SPIRV::OpAccessChain)
3626 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
3627 : SPIRV::OpPtrAccessChain);
3628
3629 auto Res = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3630 .addDef(RegNo: ResVReg)
3631 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3632 // Object to get a pointer to.
3633 .addUse(RegNo: I.getOperand(i: 3).getReg());
3634 assert(
3635 (Opcode == SPIRV::OpPtrAccessChain ||
3636 Opcode == SPIRV::OpInBoundsPtrAccessChain ||
3637 (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0)) &&
3638 "Cannot translate GEP to OpAccessChain. First index must be 0.");
3639
3640 // Adding indices.
3641 const unsigned StartingIndex =
3642 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
3643 ? 5
3644 : 4;
3645 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
3646 Res.addUse(RegNo: I.getOperand(i).getReg());
3647 Res.constrainAllUses(TII, TRI, RBI);
3648 return true;
3649}
3650
3651// Maybe wrap a value into OpSpecConstantOp
3652bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
3653 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
3654 unsigned Lim = I.getNumExplicitOperands();
3655 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
3656 Register OpReg = I.getOperand(i).getReg();
3657 MachineInstr *OpDefine = MRI->getVRegDef(Reg: OpReg);
3658 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3659 if (!OpDefine || !OpType || isConstReg(MRI, OpDef: OpDefine) ||
3660 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
3661 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
3662 GR.isAggregateType(Type: OpType)) {
3663 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
3664 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
3665 CompositeArgs.push_back(Elt: OpReg);
3666 continue;
3667 }
3668 MachineFunction *MF = I.getMF();
3669 Register WrapReg = GR.find(MI: OpDefine, MF);
3670 if (WrapReg.isValid()) {
3671 CompositeArgs.push_back(Elt: WrapReg);
3672 continue;
3673 }
3674 // Create a new register for the wrapper
3675 WrapReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: OpType));
3676 CompositeArgs.push_back(Elt: WrapReg);
3677 // Decorate the wrapper register and generate a new instruction
3678 MRI->setType(VReg: WrapReg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3679 GR.assignSPIRVTypeToVReg(Type: OpType, VReg: WrapReg, MF: *MF);
3680 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3681 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
3682 .addDef(RegNo: WrapReg)
3683 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: OpType))
3684 .addImm(Val: static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
3685 .addUse(RegNo: OpReg);
3686 GR.add(Obj: OpDefine, MI: MIB);
3687 MIB.constrainAllUses(TII, TRI, RBI);
3688 }
3689 return true;
3690}
3691
3692bool SPIRVInstructionSelector::selectDerivativeInst(
3693 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
3694 const unsigned DPdOpCode) const {
3695 // TODO: This should check specifically for Fragment Execution Model, but STI
3696 // doesn't provide that information yet. See #167562
3697 errorIfInstrOutsideShader(I);
3698
3699 // If the arg/result types are half then we need to wrap the instr in
3700 // conversions to float
3701 // This case occurs because a half arg/result is legal in HLSL but not spirv.
3702 Register SrcReg = I.getOperand(i: 2).getReg();
3703 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
3704 unsigned BitWidth = std::min(a: GR.getScalarOrVectorBitWidth(Type: SrcType),
3705 b: GR.getScalarOrVectorBitWidth(Type: ResType));
3706 if (BitWidth == 32)
3707 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
3708 .addDef(RegNo: ResVReg)
3709 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3710 .addUse(RegNo: I.getOperand(i: 2).getReg());
3711
3712 MachineIRBuilder MIRBuilder(I);
3713 unsigned componentCount = GR.getScalarOrVectorComponentCount(Type: SrcType);
3714 SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(BitWidth: 32, I, TII);
3715 if (componentCount != 1)
3716 F32ConvertTy = GR.getOrCreateSPIRVVectorType(BaseType: F32ConvertTy, NumElements: componentCount,
3717 MIRBuilder, EmitIR: false);
3718
3719 const TargetRegisterClass *RegClass = GR.getRegClass(SpvType: SrcType);
3720 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
3721 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
3722
3723 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
3724 .addDef(RegNo: ConvertToVReg)
3725 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
3726 .addUse(RegNo: SrcReg)
3727 .constrainAllUses(TII, TRI, RBI);
3728 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
3729 .addDef(RegNo: DpdOpVReg)
3730 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
3731 .addUse(RegNo: ConvertToVReg)
3732 .constrainAllUses(TII, TRI, RBI);
3733 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
3734 .addDef(RegNo: ResVReg)
3735 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3736 .addUse(RegNo: DpdOpVReg)
3737 .constrainAllUses(TII, TRI, RBI);
3738 return true;
3739}
3740
3741bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
3742 SPIRVTypeInst ResType,
3743 MachineInstr &I) const {
3744 MachineBasicBlock &BB = *I.getParent();
3745 Intrinsic::ID IID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
3746 switch (IID) {
3747 case Intrinsic::spv_load:
3748 return selectLoad(ResVReg, ResType, I);
3749 case Intrinsic::spv_store:
3750 return selectStore(I);
3751 case Intrinsic::spv_extractv:
3752 return selectExtractVal(ResVReg, ResType, I);
3753 case Intrinsic::spv_insertv:
3754 return selectInsertVal(ResVReg, ResType, I);
3755 case Intrinsic::spv_extractelt:
3756 return selectExtractElt(ResVReg, ResType, I);
3757 case Intrinsic::spv_insertelt:
3758 return selectInsertElt(ResVReg, ResType, I);
3759 case Intrinsic::spv_gep:
3760 return selectGEP(ResVReg, ResType, I);
3761 case Intrinsic::spv_bitcast: {
3762 Register OpReg = I.getOperand(i: 2).getReg();
3763 SPIRVTypeInst OpType =
3764 OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
3765 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
3766 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
3767 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {OpReg}, Opcode: SPIRV::OpBitcast);
3768 }
3769 case Intrinsic::spv_unref_global:
3770 case Intrinsic::spv_init_global: {
3771 MachineInstr *MI = MRI->getVRegDef(Reg: I.getOperand(i: 1).getReg());
3772 MachineInstr *Init = I.getNumExplicitOperands() > 2
3773 ? MRI->getVRegDef(Reg: I.getOperand(i: 2).getReg())
3774 : nullptr;
3775 assert(MI);
3776 Register GVarVReg = MI->getOperand(i: 0).getReg();
3777 if (!selectGlobalValue(ResVReg: GVarVReg, I&: *MI, Init))
3778 return false;
3779 // We violate SSA form by inserting OpVariable and still having a gMIR
3780 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
3781 // the duplicated definition.
3782 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
3783 GR.invalidateMachineInstr(MI);
3784 MI->eraseFromParent();
3785 }
3786 return true;
3787 }
3788 case Intrinsic::spv_undef: {
3789 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3790 .addDef(RegNo: ResVReg)
3791 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3792 MIB.constrainAllUses(TII, TRI, RBI);
3793 return true;
3794 }
3795 case Intrinsic::spv_const_composite: {
3796 // If no values are attached, the composite is null constant.
3797 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
3798 SmallVector<Register> CompositeArgs;
3799 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3800
3801 // skip type MD node we already used when generated assign.type for this
3802 if (!IsNull) {
3803 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
3804 return false;
3805 MachineIRBuilder MIR(I);
3806 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
3807 MIRBuilder&: MIR, Opcode: SPIRV::OpConstantComposite, MinWC: 3,
3808 ContinuedOpcode: SPIRV::OpConstantCompositeContinuedINTEL, Args: CompositeArgs, ReturnRegister: ResVReg,
3809 TypeID: GR.getSPIRVTypeID(SpirvType: ResType));
3810 for (auto *Instr : Instructions) {
3811 Instr->setDebugLoc(I.getDebugLoc());
3812 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
3813 }
3814 return true;
3815 } else {
3816 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3817 .addDef(RegNo: ResVReg)
3818 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3819 MIB.constrainAllUses(TII, TRI, RBI);
3820 return true;
3821 }
3822 }
3823 case Intrinsic::spv_assign_name: {
3824 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName));
3825 MIB.addUse(RegNo: I.getOperand(i: I.getNumExplicitDefs() + 1).getReg());
3826 for (unsigned i = I.getNumExplicitDefs() + 2;
3827 i < I.getNumExplicitOperands(); ++i) {
3828 MIB.addImm(Val: I.getOperand(i).getImm());
3829 }
3830 MIB.constrainAllUses(TII, TRI, RBI);
3831 return true;
3832 }
3833 case Intrinsic::spv_switch: {
3834 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSwitch));
3835 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
3836 if (I.getOperand(i).isReg())
3837 MIB.addReg(RegNo: I.getOperand(i).getReg());
3838 else if (I.getOperand(i).isCImm())
3839 addNumImm(Imm: I.getOperand(i).getCImm()->getValue(), MIB);
3840 else if (I.getOperand(i).isMBB())
3841 MIB.addMBB(MBB: I.getOperand(i).getMBB());
3842 else
3843 llvm_unreachable("Unexpected OpSwitch operand");
3844 }
3845 MIB.constrainAllUses(TII, TRI, RBI);
3846 return true;
3847 }
3848 case Intrinsic::spv_loop_merge: {
3849 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopMerge));
3850 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
3851 if (I.getOperand(i).isMBB())
3852 MIB.addMBB(MBB: I.getOperand(i).getMBB());
3853 else
3854 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3855 }
3856 MIB.constrainAllUses(TII, TRI, RBI);
3857 return true;
3858 }
3859 case Intrinsic::spv_loop_control_intel: {
3860 auto MIB =
3861 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopControlINTEL));
3862 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
3863 MIB.addImm(Val: foldImm(MO: I.getOperand(i: J), MRI));
3864 MIB.constrainAllUses(TII, TRI, RBI);
3865 return true;
3866 }
3867 case Intrinsic::spv_selection_merge: {
3868 auto MIB =
3869 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSelectionMerge));
3870 assert(I.getOperand(1).isMBB() &&
3871 "operand 1 to spv_selection_merge must be a basic block");
3872 MIB.addMBB(MBB: I.getOperand(i: 1).getMBB());
3873 MIB.addImm(Val: getSelectionOperandForImm(Imm: I.getOperand(i: 2).getImm()));
3874 MIB.constrainAllUses(TII, TRI, RBI);
3875 return true;
3876 }
3877 case Intrinsic::spv_cmpxchg:
3878 return selectAtomicCmpXchg(ResVReg, ResType, I);
3879 case Intrinsic::spv_unreachable:
3880 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUnreachable))
3881 .constrainAllUses(TII, TRI, RBI);
3882 return true;
3883 case Intrinsic::spv_alloca:
3884 return selectFrameIndex(ResVReg, ResType, I);
3885 case Intrinsic::spv_alloca_array:
3886 return selectAllocaArray(ResVReg, ResType, I);
3887 case Intrinsic::spv_assume:
3888 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
3889 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAssumeTrueKHR))
3890 .addUse(RegNo: I.getOperand(i: 1).getReg())
3891 .constrainAllUses(TII, TRI, RBI);
3892 return true;
3893 }
3894 break;
3895 case Intrinsic::spv_expect:
3896 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
3897 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExpectKHR))
3898 .addDef(RegNo: ResVReg)
3899 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3900 .addUse(RegNo: I.getOperand(i: 2).getReg())
3901 .addUse(RegNo: I.getOperand(i: 3).getReg())
3902 .constrainAllUses(TII, TRI, RBI);
3903 return true;
3904 }
3905 break;
3906 case Intrinsic::arithmetic_fence:
3907 if (STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
3908 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpArithmeticFenceEXT))
3909 .addDef(RegNo: ResVReg)
3910 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3911 .addUse(RegNo: I.getOperand(i: 2).getReg())
3912 .constrainAllUses(TII, TRI, RBI);
3913 return true;
3914 } else
3915 return BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 2).getReg(), I);
3916 break;
3917 case Intrinsic::spv_thread_id:
3918 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
3919 // intrinsic in LLVM IR for SPIR-V backend.
3920 //
3921 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
3922 // `GlobalInvocationId` builtin variable
3923 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
3924 ResType, I);
3925 case Intrinsic::spv_thread_id_in_group:
3926 // The HLSL SV_GroupThreadId semantic is lowered to
3927 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
3928 //
3929 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
3930 // translated to a `LocalInvocationId` builtin variable
3931 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationId, ResVReg,
3932 ResType, I);
3933 case Intrinsic::spv_group_id:
3934 // The HLSL SV_GroupId semantic is lowered to
3935 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
3936 //
3937 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
3938 // builtin variable
3939 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
3940 I);
3941 case Intrinsic::spv_flattened_thread_id_in_group:
3942 // The HLSL SV_GroupIndex semantic is lowered to
3943 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
3944 // backend.
3945 //
3946 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
3947 // a `LocalInvocationIndex` builtin variable
3948 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
3949 ResType, I);
3950 case Intrinsic::spv_workgroup_size:
3951 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupSize, ResVReg,
3952 ResType, I);
3953 case Intrinsic::spv_global_size:
3954 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
3955 I);
3956 case Intrinsic::spv_global_offset:
3957 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalOffset, ResVReg,
3958 ResType, I);
3959 case Intrinsic::spv_num_workgroups:
3960 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumWorkgroups, ResVReg,
3961 ResType, I);
3962 case Intrinsic::spv_subgroup_size:
3963 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
3964 I);
3965 case Intrinsic::spv_num_subgroups:
3966 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
3967 I);
3968 case Intrinsic::spv_subgroup_id:
3969 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
3970 case Intrinsic::spv_subgroup_local_invocation_id:
3971 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupLocalInvocationId,
3972 ResVReg, ResType, I);
3973 case Intrinsic::spv_subgroup_max_size:
3974 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
3975 I);
3976 case Intrinsic::spv_fdot:
3977 return selectFloatDot(ResVReg, ResType, I);
3978 case Intrinsic::spv_udot:
3979 case Intrinsic::spv_sdot:
3980 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3981 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3982 return selectIntegerDot(ResVReg, ResType, I,
3983 /*Signed=*/IID == Intrinsic::spv_sdot);
3984 return selectIntegerDotExpansion(ResVReg, ResType, I);
3985 case Intrinsic::spv_dot4add_i8packed:
3986 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3987 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3988 return selectDot4AddPacked<true>(ResVReg, ResType, I);
3989 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
3990 case Intrinsic::spv_dot4add_u8packed:
3991 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3992 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
3993 return selectDot4AddPacked<false>(ResVReg, ResType, I);
3994 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
3995 case Intrinsic::spv_all:
3996 return selectAll(ResVReg, ResType, I);
3997 case Intrinsic::spv_any:
3998 return selectAny(ResVReg, ResType, I);
3999 case Intrinsic::spv_cross:
4000 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cross, GLInst: GL::Cross);
4001 case Intrinsic::spv_distance:
4002 return selectExtInst(ResVReg, ResType, I, CLInst: CL::distance, GLInst: GL::Distance);
4003 case Intrinsic::spv_lerp:
4004 return selectExtInst(ResVReg, ResType, I, CLInst: CL::mix, GLInst: GL::FMix);
4005 case Intrinsic::spv_length:
4006 return selectExtInst(ResVReg, ResType, I, CLInst: CL::length, GLInst: GL::Length);
4007 case Intrinsic::spv_degrees:
4008 return selectExtInst(ResVReg, ResType, I, CLInst: CL::degrees, GLInst: GL::Degrees);
4009 case Intrinsic::spv_faceforward:
4010 return selectExtInst(ResVReg, ResType, I, GLInst: GL::FaceForward);
4011 case Intrinsic::spv_frac:
4012 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fract, GLInst: GL::Fract);
4013 case Intrinsic::spv_isinf:
4014 return selectOpIsInf(ResVReg, ResType, I);
4015 case Intrinsic::spv_isnan:
4016 return selectOpIsNan(ResVReg, ResType, I);
4017 case Intrinsic::spv_normalize:
4018 return selectExtInst(ResVReg, ResType, I, CLInst: CL::normalize, GLInst: GL::Normalize);
4019 case Intrinsic::spv_refract:
4020 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Refract);
4021 case Intrinsic::spv_reflect:
4022 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Reflect);
4023 case Intrinsic::spv_rsqrt:
4024 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rsqrt, GLInst: GL::InverseSqrt);
4025 case Intrinsic::spv_sign:
4026 return selectSign(ResVReg, ResType, I);
4027 case Intrinsic::spv_smoothstep:
4028 return selectExtInst(ResVReg, ResType, I, CLInst: CL::smoothstep, GLInst: GL::SmoothStep);
4029 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
4030 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
4031 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
4032 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
4033 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
4034 return selectFirstBitLow(ResVReg, ResType, I);
4035 case Intrinsic::spv_group_memory_barrier_with_group_sync: {
4036 Register MemSemReg =
4037 buildI32Constant(Val: SPIRV::MemorySemantics::SequentiallyConsistent, I);
4038 Register ScopeReg = buildI32Constant(Val: SPIRV::Scope::Workgroup, I);
4039 MachineBasicBlock &BB = *I.getParent();
4040 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpControlBarrier))
4041 .addUse(RegNo: ScopeReg)
4042 .addUse(RegNo: ScopeReg)
4043 .addUse(RegNo: MemSemReg)
4044 .constrainAllUses(TII, TRI, RBI);
4045 return true;
4046 }
4047 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
4048 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 1).getReg();
4049 SPIRV::StorageClass::StorageClass ResSC =
4050 GR.getPointerStorageClass(Type: ResType);
4051 if (!isGenericCastablePtr(SC: ResSC))
4052 report_fatal_error(reason: "The target storage class is not castable from the "
4053 "Generic storage class");
4054 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtrExplicit))
4055 .addDef(RegNo: ResVReg)
4056 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4057 .addUse(RegNo: PtrReg)
4058 .addImm(Val: ResSC)
4059 .constrainAllUses(TII, TRI, RBI);
4060 return true;
4061 }
4062 case Intrinsic::spv_lifetime_start:
4063 case Intrinsic::spv_lifetime_end: {
4064 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
4065 : SPIRV::OpLifetimeStop;
4066 int64_t Size = I.getOperand(i: I.getNumExplicitDefs() + 1).getImm();
4067 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 2).getReg();
4068 if (Size == -1)
4069 Size = 0;
4070 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Op))
4071 .addUse(RegNo: PtrReg)
4072 .addImm(Val: Size)
4073 .constrainAllUses(TII, TRI, RBI);
4074 return true;
4075 }
4076 case Intrinsic::spv_saturate:
4077 return selectSaturate(ResVReg, ResType, I);
4078 case Intrinsic::spv_nclamp:
4079 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fclamp, GLInst: GL::NClamp);
4080 case Intrinsic::spv_uclamp:
4081 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_clamp, GLInst: GL::UClamp);
4082 case Intrinsic::spv_sclamp:
4083 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_clamp, GLInst: GL::SClamp);
4084 case Intrinsic::spv_subgroup_prefix_bit_count:
4085 return selectWavePrefixBitCount(ResVReg, ResType, I);
4086 case Intrinsic::spv_wave_active_countbits:
4087 return selectWaveActiveCountBits(ResVReg, ResType, I);
4088 case Intrinsic::spv_wave_all:
4089 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAll);
4090 case Intrinsic::spv_wave_any:
4091 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAny);
4092 case Intrinsic::spv_subgroup_ballot:
4093 return selectWaveOpInst(ResVReg, ResType, I,
4094 Opcode: SPIRV::OpGroupNonUniformBallot);
4095 case Intrinsic::spv_wave_is_first_lane:
4096 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformElect);
4097 case Intrinsic::spv_wave_reduce_umax:
4098 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
4099 case Intrinsic::spv_wave_reduce_max:
4100 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
4101 case Intrinsic::spv_wave_reduce_umin:
4102 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
4103 case Intrinsic::spv_wave_reduce_min:
4104 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
4105 case Intrinsic::spv_wave_reduce_sum:
4106 return selectWaveReduceSum(ResVReg, ResType, I);
4107 case Intrinsic::spv_wave_readlane:
4108 return selectWaveOpInst(ResVReg, ResType, I,
4109 Opcode: SPIRV::OpGroupNonUniformShuffle);
4110 case Intrinsic::spv_wave_prefix_sum:
4111 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
4112 case Intrinsic::spv_wave_prefix_product:
4113 return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
4114 case Intrinsic::spv_step:
4115 return selectExtInst(ResVReg, ResType, I, CLInst: CL::step, GLInst: GL::Step);
4116 case Intrinsic::spv_radians:
4117 return selectExtInst(ResVReg, ResType, I, CLInst: CL::radians, GLInst: GL::Radians);
4118 // Discard intrinsics which we do not expect to actually represent code after
4119 // lowering or intrinsics which are not implemented but should not crash when
4120 // found in a customer's LLVM IR input.
4121 case Intrinsic::instrprof_increment:
4122 case Intrinsic::instrprof_increment_step:
4123 case Intrinsic::instrprof_value_profile:
4124 break;
4125 // Discard internal intrinsics.
4126 case Intrinsic::spv_value_md:
4127 break;
4128 case Intrinsic::spv_resource_handlefrombinding: {
4129 return selectHandleFromBinding(ResVReg, ResType, I);
4130 }
4131 case Intrinsic::spv_resource_counterhandlefrombinding:
4132 return selectCounterHandleFromBinding(ResVReg, ResType, I);
4133 case Intrinsic::spv_resource_updatecounter:
4134 return selectUpdateCounter(ResVReg, ResType, I);
4135 case Intrinsic::spv_resource_store_typedbuffer: {
4136 return selectImageWriteIntrinsic(I);
4137 }
4138 case Intrinsic::spv_resource_load_typedbuffer: {
4139 return selectReadImageIntrinsic(ResVReg, ResType, I);
4140 }
4141 case Intrinsic::spv_resource_sample:
4142 case Intrinsic::spv_resource_sample_clamp:
4143 return selectSampleBasicIntrinsic(ResVReg, ResType, I);
4144 case Intrinsic::spv_resource_samplebias:
4145 case Intrinsic::spv_resource_samplebias_clamp:
4146 return selectSampleBiasIntrinsic(ResVReg, ResType, I);
4147 case Intrinsic::spv_resource_samplegrad:
4148 case Intrinsic::spv_resource_samplegrad_clamp:
4149 return selectSampleGradIntrinsic(ResVReg, ResType, I);
4150 case Intrinsic::spv_resource_samplelevel:
4151 return selectSampleLevelIntrinsic(ResVReg, ResType, I);
4152 case Intrinsic::spv_resource_samplecmp:
4153 case Intrinsic::spv_resource_samplecmp_clamp:
4154 return selectSampleCmpIntrinsic(ResVReg, ResType, I);
4155 case Intrinsic::spv_resource_samplecmplevelzero:
4156 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I);
4157 case Intrinsic::spv_resource_getpointer: {
4158 return selectResourceGetPointer(ResVReg, ResType, I);
4159 }
4160 case Intrinsic::spv_pushconstant_getpointer: {
4161 return selectPushConstantGetPointer(ResVReg, ResType, I);
4162 }
4163 case Intrinsic::spv_discard: {
4164 return selectDiscard(ResVReg, ResType, I);
4165 }
4166 case Intrinsic::spv_resource_nonuniformindex: {
4167 return selectResourceNonUniformIndex(ResVReg, ResType, I);
4168 }
4169 case Intrinsic::spv_unpackhalf2x16: {
4170 return selectExtInst(ResVReg, ResType, I, GLInst: GL::UnpackHalf2x16);
4171 }
4172 case Intrinsic::spv_packhalf2x16: {
4173 return selectExtInst(ResVReg, ResType, I, GLInst: GL::PackHalf2x16);
4174 }
4175 case Intrinsic::spv_ddx:
4176 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdx);
4177 case Intrinsic::spv_ddy:
4178 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdy);
4179 case Intrinsic::spv_ddx_coarse:
4180 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxCoarse);
4181 case Intrinsic::spv_ddy_coarse:
4182 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyCoarse);
4183 case Intrinsic::spv_ddx_fine:
4184 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxFine);
4185 case Intrinsic::spv_ddy_fine:
4186 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyFine);
4187 case Intrinsic::spv_fwidth:
4188 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpFwidth);
4189 default: {
4190 std::string DiagMsg;
4191 raw_string_ostream OS(DiagMsg);
4192 I.print(OS);
4193 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
4194 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
4195 }
4196 }
4197 return true;
4198}
4199
4200bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
4201 SPIRVTypeInst ResType,
4202 MachineInstr &I) const {
4203 // The images need to be loaded in the same basic block as their use. We defer
4204 // loading the image to the intrinsic that uses it.
4205 if (ResType->getOpcode() == SPIRV::OpTypeImage)
4206 return true;
4207
4208 return loadHandleBeforePosition(HandleReg&: ResVReg, ResType: GR.getSPIRVTypeForVReg(VReg: ResVReg),
4209 HandleDef&: *cast<GIntrinsic>(Val: &I), Pos&: I);
4210}
4211
4212bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
4213 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4214 auto &Intr = cast<GIntrinsic>(Val&: I);
4215 assert(Intr.getIntrinsicID() ==
4216 Intrinsic::spv_resource_counterhandlefrombinding);
4217
4218 // Extract information from the intrinsic call.
4219 Register MainHandleReg = Intr.getOperand(i: 2).getReg();
4220 auto *MainHandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: MainHandleReg));
4221 assert(MainHandleDef->getIntrinsicID() ==
4222 Intrinsic::spv_resource_handlefrombinding);
4223
4224 uint32_t Set = getIConstVal(ConstReg: Intr.getOperand(i: 4).getReg(), MRI);
4225 uint32_t Binding = getIConstVal(ConstReg: Intr.getOperand(i: 3).getReg(), MRI);
4226 uint32_t ArraySize = getIConstVal(ConstReg: MainHandleDef->getOperand(i: 4).getReg(), MRI);
4227 Register IndexReg = MainHandleDef->getOperand(i: 5).getReg();
4228 std::string CounterName =
4229 getStringValueFromReg(Reg: MainHandleDef->getOperand(i: 6).getReg(), MRI&: *MRI) +
4230 ".counter";
4231
4232 // Create the counter variable.
4233 MachineIRBuilder MIRBuilder(I);
4234 Register CounterVarReg =
4235 buildPointerToResource(ResType: SPIRVTypeInst(GR.getPointeeType(PtrType: ResType)),
4236 SC: GR.getPointerStorageClass(Type: ResType), Set, Binding,
4237 ArraySize, IndexReg, Name: CounterName, MIRBuilder);
4238
4239 return BuildCOPY(DestReg: ResVReg, SrcReg: CounterVarReg, I);
4240}
4241
4242bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
4243 SPIRVTypeInst ResType,
4244 MachineInstr &I) const {
4245 auto &Intr = cast<GIntrinsic>(Val&: I);
4246 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
4247
4248 Register CounterHandleReg = Intr.getOperand(i: 2).getReg();
4249 Register IncrReg = Intr.getOperand(i: 3).getReg();
4250
4251 // The counter handle is a pointer to the counter variable (which is a struct
4252 // containing an i32). We need to get a pointer to that i32 member to do the
4253 // atomic operation.
4254#ifndef NDEBUG
4255 SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
4256 SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType);
4257 assert(CounterVarPointeeType &&
4258 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
4259 "Counter variable must be a struct");
4260 assert(GR.getPointerStorageClass(CounterVarType) ==
4261 SPIRV::StorageClass::StorageBuffer &&
4262 "Counter variable must be in the storage buffer storage class");
4263 assert(CounterVarPointeeType->getNumOperands() == 2 &&
4264 "Counter variable must have exactly 1 member in the struct");
4265 const SPIRVTypeInst MemberType =
4266 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
4267 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
4268 "Counter variable struct must have a single i32 member");
4269#endif
4270
4271 // The struct has a single i32 member.
4272 MachineIRBuilder MIRBuilder(I);
4273 const Type *LLVMIntType =
4274 Type::getInt32Ty(C&: I.getMF()->getFunction().getContext());
4275
4276 SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType(
4277 BaseType: LLVMIntType, MIRBuilder, SC: SPIRV::StorageClass::StorageBuffer);
4278
4279 Register Zero = buildI32Constant(Val: 0, I);
4280
4281 Register PtrToCounter =
4282 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: IntPtrType));
4283 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAccessChain))
4284 .addDef(RegNo: PtrToCounter)
4285 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntPtrType))
4286 .addUse(RegNo: CounterHandleReg)
4287 .addUse(RegNo: Zero)
4288 .constrainAllUses(TII, TRI, RBI);
4289
4290 // For UAV/SSBO counters, the scope is Device. The counter variable is not
4291 // used as a flag. So the memory semantics can be None.
4292 Register Scope = buildI32Constant(Val: SPIRV::Scope::Device, I);
4293 Register Semantics = buildI32Constant(Val: SPIRV::MemorySemantics::None, I);
4294
4295 int64_t IncrVal = getIConstValSext(ConstReg: IncrReg, MRI);
4296 Register Incr = buildI32Constant(Val: static_cast<uint32_t>(IncrVal), I);
4297
4298 Register AtomicRes = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4299 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAtomicIAdd))
4300 .addDef(RegNo: AtomicRes)
4301 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4302 .addUse(RegNo: PtrToCounter)
4303 .addUse(RegNo: Scope)
4304 .addUse(RegNo: Semantics)
4305 .addUse(RegNo: Incr)
4306 .constrainAllUses(TII, TRI, RBI);
4307 if (IncrVal >= 0) {
4308 return BuildCOPY(DestReg: ResVReg, SrcReg: AtomicRes, I);
4309 }
4310
4311 // In HLSL, IncrementCounter returns the value *before* the increment, while
4312 // DecrementCounter returns the value *after* the decrement. Both are lowered
4313 // to the same atomic intrinsic which returns the value *before* the
4314 // operation. So for decrements (negative IncrVal), we must subtract the
4315 // increment value from the result to get the post-decrement value.
4316 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
4317 .addDef(RegNo: ResVReg)
4318 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4319 .addUse(RegNo: AtomicRes)
4320 .addUse(RegNo: Incr)
4321 .constrainAllUses(TII, TRI, RBI);
4322 return true;
4323}
4324bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg,
4325 SPIRVTypeInst ResType,
4326 MachineInstr &I) const {
4327
4328 // If the load of the image is in a different basic block, then
4329 // this will generate invalid code. A proper solution is to move
4330 // the OpLoad from selectHandleFromBinding here. However, to do
4331 // that we will need to change the return type of the intrinsic.
4332 // We will do that when we can, but for now trying to move forward with other
4333 // issues.
4334 Register ImageReg = I.getOperand(i: 2).getReg();
4335 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4336 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4337 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4338 HandleDef&: *ImageDef, Pos&: I)) {
4339 return false;
4340 }
4341
4342 Register IdxReg = I.getOperand(i: 3).getReg();
4343 DebugLoc Loc = I.getDebugLoc();
4344 MachineInstr &Pos = I;
4345
4346 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewImageReg, IdxReg, Loc,
4347 Pos);
4348}
4349
4350bool SPIRVInstructionSelector::generateSampleImage(
4351 Register ResVReg, SPIRVTypeInst ResType, Register ImageReg,
4352 Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps,
4353 DebugLoc Loc, MachineInstr &Pos) const {
4354 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4355 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4356 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4357 HandleDef&: *ImageDef, Pos)) {
4358 return false;
4359 }
4360
4361 auto *SamplerDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: SamplerReg));
4362 Register NewSamplerReg =
4363 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: SamplerReg));
4364 if (!loadHandleBeforePosition(HandleReg&: NewSamplerReg,
4365 ResType: GR.getSPIRVTypeForVReg(VReg: SamplerReg), HandleDef&: *SamplerDef,
4366 Pos)) {
4367 return false;
4368 }
4369
4370 MachineIRBuilder MIRBuilder(Pos);
4371 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
4372 ImageType: GR.getSPIRVTypeForVReg(VReg: ImageReg), MIRBuilder);
4373 Register SampledImageReg =
4374 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: SampledImageType));
4375
4376 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpSampledImage))
4377 .addDef(RegNo: SampledImageReg)
4378 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SampledImageType))
4379 .addUse(RegNo: NewImageReg)
4380 .addUse(RegNo: NewSamplerReg)
4381 .constrainAllUses(TII, TRI, RBI);
4382
4383 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
4384 ImOps.Lod.has_value();
4385 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
4386 : SPIRV::OpImageSampleImplicitLod;
4387 if (ImOps.Compare)
4388 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
4389 : SPIRV::OpImageSampleDrefImplicitLod;
4390
4391 auto MIB = BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode))
4392 .addDef(RegNo: ResVReg)
4393 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4394 .addUse(RegNo: SampledImageReg)
4395 .addUse(RegNo: CoordinateReg);
4396
4397 if (ImOps.Compare)
4398 MIB.addUse(RegNo: *ImOps.Compare);
4399
4400 uint32_t ImageOperands = 0;
4401 if (ImOps.Bias)
4402 ImageOperands |= SPIRV::ImageOperand::Bias;
4403 if (ImOps.Lod)
4404 ImageOperands |= SPIRV::ImageOperand::Lod;
4405 if (ImOps.GradX && ImOps.GradY)
4406 ImageOperands |= SPIRV::ImageOperand::Grad;
4407 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(Reg: *ImOps.Offset)) {
4408 if (isConstReg(MRI, OpReg: *ImOps.Offset))
4409 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
4410 else {
4411 Pos.emitGenericError(
4412 ErrMsg: "Non-constant offsets are not supported in sample instructions.");
4413 }
4414 }
4415 if (ImOps.MinLod)
4416 ImageOperands |= SPIRV::ImageOperand::MinLod;
4417
4418 if (ImageOperands != 0) {
4419 MIB.addImm(Val: ImageOperands);
4420 if (ImageOperands & SPIRV::ImageOperand::Bias)
4421 MIB.addUse(RegNo: *ImOps.Bias);
4422 if (ImageOperands & SPIRV::ImageOperand::Lod)
4423 MIB.addUse(RegNo: *ImOps.Lod);
4424 if (ImageOperands & SPIRV::ImageOperand::Grad) {
4425 MIB.addUse(RegNo: *ImOps.GradX);
4426 MIB.addUse(RegNo: *ImOps.GradY);
4427 }
4428 if (ImageOperands &
4429 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
4430 MIB.addUse(RegNo: *ImOps.Offset);
4431 if (ImageOperands & SPIRV::ImageOperand::MinLod)
4432 MIB.addUse(RegNo: *ImOps.MinLod);
4433 }
4434
4435 MIB.constrainAllUses(TII, TRI, RBI);
4436 return true;
4437}
4438
4439bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
4440 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4441 Register ImageReg = I.getOperand(i: 2).getReg();
4442 Register SamplerReg = I.getOperand(i: 3).getReg();
4443 Register CoordinateReg = I.getOperand(i: 4).getReg();
4444 ImageOperands ImOps;
4445 if (I.getNumOperands() > 5)
4446 ImOps.Offset = I.getOperand(i: 5).getReg();
4447 if (I.getNumOperands() > 6)
4448 ImOps.MinLod = I.getOperand(i: 6).getReg();
4449 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4450 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4451}
4452
4453bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
4454 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4455 Register ImageReg = I.getOperand(i: 2).getReg();
4456 Register SamplerReg = I.getOperand(i: 3).getReg();
4457 Register CoordinateReg = I.getOperand(i: 4).getReg();
4458 ImageOperands ImOps;
4459 ImOps.Bias = I.getOperand(i: 5).getReg();
4460 if (I.getNumOperands() > 6)
4461 ImOps.Offset = I.getOperand(i: 6).getReg();
4462 if (I.getNumOperands() > 7)
4463 ImOps.MinLod = I.getOperand(i: 7).getReg();
4464 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4465 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4466}
4467
4468bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
4469 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4470 Register ImageReg = I.getOperand(i: 2).getReg();
4471 Register SamplerReg = I.getOperand(i: 3).getReg();
4472 Register CoordinateReg = I.getOperand(i: 4).getReg();
4473 ImageOperands ImOps;
4474 ImOps.GradX = I.getOperand(i: 5).getReg();
4475 ImOps.GradY = I.getOperand(i: 6).getReg();
4476 if (I.getNumOperands() > 7)
4477 ImOps.Offset = I.getOperand(i: 7).getReg();
4478 if (I.getNumOperands() > 8)
4479 ImOps.MinLod = I.getOperand(i: 8).getReg();
4480 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4481 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4482}
4483
4484bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
4485 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4486 Register ImageReg = I.getOperand(i: 2).getReg();
4487 Register SamplerReg = I.getOperand(i: 3).getReg();
4488 Register CoordinateReg = I.getOperand(i: 4).getReg();
4489 ImageOperands ImOps;
4490 ImOps.Lod = I.getOperand(i: 5).getReg();
4491 if (I.getNumOperands() > 6)
4492 ImOps.Offset = I.getOperand(i: 6).getReg();
4493 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4494 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4495}
4496
4497bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg,
4498 SPIRVTypeInst ResType,
4499 MachineInstr &I) const {
4500 Register ImageReg = I.getOperand(i: 2).getReg();
4501 Register SamplerReg = I.getOperand(i: 3).getReg();
4502 Register CoordinateReg = I.getOperand(i: 4).getReg();
4503 ImageOperands ImOps;
4504 ImOps.Compare = I.getOperand(i: 5).getReg();
4505 if (I.getNumOperands() > 6)
4506 ImOps.Offset = I.getOperand(i: 6).getReg();
4507 if (I.getNumOperands() > 7)
4508 ImOps.MinLod = I.getOperand(i: 7).getReg();
4509 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4510 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4511}
4512
4513bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
4514 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4515 Register ImageReg = I.getOperand(i: 2).getReg();
4516 Register SamplerReg = I.getOperand(i: 3).getReg();
4517 Register CoordinateReg = I.getOperand(i: 4).getReg();
4518 ImageOperands ImOps;
4519 ImOps.Compare = I.getOperand(i: 5).getReg();
4520 if (I.getNumOperands() > 6)
4521 ImOps.Offset = I.getOperand(i: 6).getReg();
4522 SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(BitWidth: 32, I, TII);
4523 ImOps.Lod = GR.getOrCreateConstFP(Val: APFloat(0.0f), I, SpvType: FloatTy, TII);
4524 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4525 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4526}
4527
4528bool SPIRVInstructionSelector::generateImageReadOrFetch(
4529 Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg,
4530 Register IdxReg, DebugLoc Loc, MachineInstr &Pos) const {
4531 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(VReg: ImageReg);
4532 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
4533 "ImageReg is not an image type.");
4534
4535 bool IsSignedInteger =
4536 sampledTypeIsSignedInteger(HandleType: GR.getTypeForSPIRVType(Ty: ImageType));
4537 // Check if the "sampled" operand of the image type is 1.
4538 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
4539 auto SampledOp = ImageType->getOperand(i: 6);
4540 bool IsFetch = (SampledOp.getImm() == 1);
4541
4542 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
4543 if (ResultSize == 4) {
4544 auto BMI =
4545 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
4546 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
4547 .addDef(RegNo: ResVReg)
4548 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4549 .addUse(RegNo: ImageReg)
4550 .addUse(RegNo: IdxReg);
4551
4552 if (IsSignedInteger)
4553 BMI.addImm(Val: 0x1000); // SignExtend
4554 BMI.constrainAllUses(TII, TRI, RBI);
4555 return true;
4556 }
4557
4558 SPIRVTypeInst ReadType = widenTypeToVec4(Type: ResType, I&: Pos);
4559 Register ReadReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ReadType));
4560 auto BMI =
4561 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
4562 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
4563 .addDef(RegNo: ReadReg)
4564 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ReadType))
4565 .addUse(RegNo: ImageReg)
4566 .addUse(RegNo: IdxReg);
4567 if (IsSignedInteger)
4568 BMI.addImm(Val: 0x1000); // SignExtend
4569 BMI.constrainAllUses(TII, TRI, RBI);
4570
4571 if (ResultSize == 1) {
4572 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
4573 .addDef(RegNo: ResVReg)
4574 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4575 .addUse(RegNo: ReadReg)
4576 .addImm(Val: 0)
4577 .constrainAllUses(TII, TRI, RBI);
4578 return true;
4579 }
4580 return extractSubvector(ResVReg, ResType, ReadReg, InsertionPoint&: Pos);
4581}
4582
4583bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg,
4584 SPIRVTypeInst ResType,
4585 MachineInstr &I) const {
4586 Register ResourcePtr = I.getOperand(i: 2).getReg();
4587 SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(VReg: ResourcePtr, MF: I.getMF());
4588 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
4589 // For texel buffers, the index into the image is part of the OpImageRead or
4590 // OpImageWrite instructions. So we will do nothing in this case. This
4591 // intrinsic will be combined with the load or store when selecting the load
4592 // or store.
4593 return true;
4594 }
4595
4596 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
4597 MachineIRBuilder MIRBuilder(I);
4598
4599 Register IndexReg = I.getOperand(i: 3).getReg();
4600 Register ZeroReg =
4601 buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
4602 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAccessChain))
4603 .addDef(RegNo: ResVReg)
4604 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4605 .addUse(RegNo: ResourcePtr)
4606 .addUse(RegNo: ZeroReg)
4607 .addUse(RegNo: IndexReg)
4608 .constrainAllUses(TII, TRI, RBI);
4609 return true;
4610}
4611
4612bool SPIRVInstructionSelector::selectPushConstantGetPointer(
4613 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4614 MRI->replaceRegWith(FromReg: ResVReg, ToReg: I.getOperand(i: 2).getReg());
4615 return true;
4616}
4617
4618bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
4619 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4620 Register ObjReg = I.getOperand(i: 2).getReg();
4621 if (!BuildCOPY(DestReg: ResVReg, SrcReg: ObjReg, I))
4622 return false;
4623
4624 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
4625 // Check for the registers that use the index marked as non-uniform
4626 // and recursively mark them as non-uniform.
4627 // Per the spec, it's necessary that the final argument used for
4628 // load/store/sample/atomic must be decorated, so we need to propagate the
4629 // decoration through access chains and copies.
4630 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
4631 decorateUsesAsNonUniform(NonUniformReg&: ResVReg);
4632 return true;
4633}
4634
4635void SPIRVInstructionSelector::decorateUsesAsNonUniform(
4636 Register &NonUniformReg) const {
4637 llvm::SmallVector<Register> WorkList = {NonUniformReg};
4638 while (WorkList.size() > 0) {
4639 Register CurrentReg = WorkList.back();
4640 WorkList.pop_back();
4641
4642 bool IsDecorated = false;
4643 for (MachineInstr &Use : MRI->use_instructions(Reg: CurrentReg)) {
4644 if (Use.getOpcode() == SPIRV::OpDecorate &&
4645 Use.getOperand(i: 1).getImm() == SPIRV::Decoration::NonUniformEXT) {
4646 IsDecorated = true;
4647 continue;
4648 }
4649 // Check if the instruction has the result register and add it to the
4650 // worklist.
4651 if (Use.getOperand(i: 0).isReg() && Use.getOperand(i: 0).isDef()) {
4652 Register ResultReg = Use.getOperand(i: 0).getReg();
4653 if (ResultReg == CurrentReg)
4654 continue;
4655 WorkList.push_back(Elt: ResultReg);
4656 }
4657 }
4658
4659 if (!IsDecorated) {
4660 buildOpDecorate(Reg: CurrentReg, I&: *MRI->getVRegDef(Reg: CurrentReg), TII,
4661 Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
4662 }
4663 }
4664}
4665
4666bool SPIRVInstructionSelector::extractSubvector(
4667 Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg,
4668 MachineInstr &InsertionPoint) const {
4669 SPIRVTypeInst InputType = GR.getResultType(VReg: ReadReg);
4670 [[maybe_unused]] uint64_t InputSize =
4671 GR.getScalarOrVectorComponentCount(Type: InputType);
4672 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
4673 assert(InputSize > 1 && "The input must be a vector.");
4674 assert(ResultSize > 1 && "The result must be a vector.");
4675 assert(ResultSize < InputSize &&
4676 "Cannot extract more element than there are in the input.");
4677 SmallVector<Register> ComponentRegisters;
4678 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(Type: ResType);
4679 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(SpvType: ScalarType);
4680 for (uint64_t I = 0; I < ResultSize; I++) {
4681 Register ComponentReg = MRI->createVirtualRegister(RegClass: ScalarRegClass);
4682 BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
4683 MIMD: InsertionPoint.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
4684 .addDef(RegNo: ComponentReg)
4685 .addUse(RegNo: ScalarType->getOperand(i: 0).getReg())
4686 .addUse(RegNo: ReadReg)
4687 .addImm(Val: I)
4688 .constrainAllUses(TII, TRI, RBI);
4689 ComponentRegisters.emplace_back(Args&: ComponentReg);
4690 }
4691
4692 MachineInstrBuilder MIB = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
4693 MIMD: InsertionPoint.getDebugLoc(),
4694 MCID: TII.get(Opcode: SPIRV::OpCompositeConstruct))
4695 .addDef(RegNo: ResVReg)
4696 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
4697
4698 for (Register ComponentReg : ComponentRegisters)
4699 MIB.addUse(RegNo: ComponentReg);
4700 MIB.constrainAllUses(TII, TRI, RBI);
4701 return true;
4702}
4703
4704bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
4705 MachineInstr &I) const {
4706 // If the load of the image is in a different basic block, then
4707 // this will generate invalid code. A proper solution is to move
4708 // the OpLoad from selectHandleFromBinding here. However, to do
4709 // that we will need to change the return type of the intrinsic.
4710 // We will do that when we can, but for now trying to move forward with other
4711 // issues.
4712 Register ImageReg = I.getOperand(i: 1).getReg();
4713 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4714 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4715 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4716 HandleDef&: *ImageDef, Pos&: I)) {
4717 return false;
4718 }
4719
4720 Register CoordinateReg = I.getOperand(i: 2).getReg();
4721 Register DataReg = I.getOperand(i: 3).getReg();
4722 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
4723 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
4724 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpImageWrite))
4725 .addUse(RegNo: NewImageReg)
4726 .addUse(RegNo: CoordinateReg)
4727 .addUse(RegNo: DataReg)
4728 .constrainAllUses(TII, TRI, RBI);
4729 return true;
4730}
4731
4732Register SPIRVInstructionSelector::buildPointerToResource(
4733 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
4734 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
4735 StringRef Name, MachineIRBuilder MIRBuilder) const {
4736 const Type *ResType = GR.getTypeForSPIRVType(Ty: SpirvResType);
4737 if (ArraySize == 1) {
4738 SPIRVTypeInst PtrType =
4739 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
4740 assert(GR.getPointeeType(PtrType) == SpirvResType &&
4741 "SpirvResType did not have an explicit layout.");
4742 return GR.getOrCreateGlobalVariableWithBinding(VarType: PtrType, Set, Binding, Name,
4743 MIRBuilder);
4744 }
4745
4746 const Type *VarType = ArrayType::get(ElementType: const_cast<Type *>(ResType), NumElements: ArraySize);
4747 SPIRVTypeInst VarPointerType =
4748 GR.getOrCreateSPIRVPointerType(BaseType: VarType, MIRBuilder, SC);
4749 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
4750 VarType: VarPointerType, Set, Binding, Name, MIRBuilder);
4751
4752 SPIRVTypeInst ResPointerType =
4753 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
4754 Register AcReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResPointerType));
4755
4756 MIRBuilder.buildInstr(Opcode: SPIRV::OpAccessChain)
4757 .addDef(RegNo: AcReg)
4758 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResPointerType))
4759 .addUse(RegNo: VarReg)
4760 .addUse(RegNo: IndexReg);
4761
4762 return AcReg;
4763}
4764
4765bool SPIRVInstructionSelector::selectFirstBitSet16(
4766 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
4767 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
4768 Register ExtReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4769 if (!selectOpWithSrcs(ResVReg: ExtReg, ResType, I, Srcs: {I.getOperand(i: 2).getReg()},
4770 Opcode: ExtendOpcode))
4771 return false;
4772
4773 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: ExtReg, BitSetOpcode);
4774}
4775
4776bool SPIRVInstructionSelector::selectFirstBitSet32(
4777 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
4778 unsigned BitSetOpcode) const {
4779 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
4780 .addDef(RegNo: ResVReg)
4781 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4782 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
4783 .addImm(Val: BitSetOpcode)
4784 .addUse(RegNo: SrcReg)
4785 .constrainAllUses(TII, TRI, RBI);
4786 return true;
4787}
4788
4789bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
4790 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
4791 unsigned BitSetOpcode, bool SwapPrimarySide) const {
4792
4793 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
4794 // requires creating a param register and return register with an invalid
4795 // vector size. If that is resolved, then this function can be used for
4796 // vectors of any component size.
4797 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
4798 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
4799
4800 MachineIRBuilder MIRBuilder(I);
4801 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
4802 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 64, MIRBuilder);
4803 SPIRVTypeInst I64x2Type =
4804 GR.getOrCreateSPIRVVectorType(BaseType: I64Type, NumElements: 2, MIRBuilder, EmitIR: false);
4805 SPIRVTypeInst Vec2ResType =
4806 GR.getOrCreateSPIRVVectorType(BaseType, NumElements: 2, MIRBuilder, EmitIR: false);
4807
4808 std::vector<Register> PartialRegs;
4809
4810 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
4811 unsigned CurrentComponent = 0;
4812 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
4813 // This register holds the firstbitX result for each of the i64x2 vectors
4814 // extracted from SrcReg
4815 Register BitSetResult =
4816 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64x2Type));
4817
4818 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4819 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4820 .addDef(RegNo: BitSetResult)
4821 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: I64x2Type))
4822 .addUse(RegNo: SrcReg)
4823 .addUse(RegNo: SrcReg)
4824 .addImm(Val: CurrentComponent)
4825 .addImm(Val: CurrentComponent + 1);
4826
4827 MIB.constrainAllUses(TII, TRI, RBI);
4828
4829 Register SubVecBitSetReg =
4830 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Vec2ResType));
4831
4832 if (!selectFirstBitSet64(ResVReg: SubVecBitSetReg, ResType: Vec2ResType, I, SrcReg: BitSetResult,
4833 BitSetOpcode, SwapPrimarySide))
4834 return false;
4835
4836 PartialRegs.push_back(x: SubVecBitSetReg);
4837 }
4838
4839 // On odd component counts we need to handle one more component
4840 if (CurrentComponent != ComponentCount) {
4841 bool ZeroAsNull = !STI.isShader();
4842 Register FinalElemReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64Type));
4843 Register ConstIntLastIdx = GR.getOrCreateConstInt(
4844 Val: ComponentCount - 1, I, SpvType: BaseType, TII, ZeroAsNull);
4845
4846 if (!selectOpWithSrcs(ResVReg: FinalElemReg, ResType: I64Type, I, Srcs: {SrcReg, ConstIntLastIdx},
4847 Opcode: SPIRV::OpVectorExtractDynamic))
4848 return false;
4849
4850 Register FinalElemBitSetReg =
4851 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BaseType));
4852
4853 if (!selectFirstBitSet64(ResVReg: FinalElemBitSetReg, ResType: BaseType, I, SrcReg: FinalElemReg,
4854 BitSetOpcode, SwapPrimarySide))
4855 return false;
4856
4857 PartialRegs.push_back(x: FinalElemBitSetReg);
4858 }
4859
4860 // Join all the resulting registers back into the return type in order
4861 // (ie i32x2, i32x2, i32x1 -> i32x5)
4862 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: std::move(PartialRegs),
4863 Opcode: SPIRV::OpCompositeConstruct);
4864}
4865
4866bool SPIRVInstructionSelector::selectFirstBitSet64(
4867 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
4868 unsigned BitSetOpcode, bool SwapPrimarySide) const {
4869 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
4870 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
4871 bool ZeroAsNull = !STI.isShader();
4872 Register ConstIntZero =
4873 GR.getOrCreateConstInt(Val: 0, I, SpvType: BaseType, TII, ZeroAsNull);
4874 Register ConstIntOne =
4875 GR.getOrCreateConstInt(Val: 1, I, SpvType: BaseType, TII, ZeroAsNull);
4876
4877 // SPIRV doesn't support vectors with more than 4 components. Since the
4878 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
4879 // operate on vectors with 2 or less components. When largers vectors are
4880 // seen. Split them, recurse, then recombine them.
4881 if (ComponentCount > 2) {
4882 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
4883 BitSetOpcode, SwapPrimarySide);
4884 }
4885
4886 // 1. Split int64 into 2 pieces using a bitcast
4887 MachineIRBuilder MIRBuilder(I);
4888 SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType(
4889 BaseType, NumElements: 2 * ComponentCount, MIRBuilder, EmitIR: false);
4890 Register BitcastReg =
4891 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
4892
4893 if (!selectOpWithSrcs(ResVReg: BitcastReg, ResType: PostCastType, I, Srcs: {SrcReg},
4894 Opcode: SPIRV::OpBitcast))
4895 return false;
4896
4897 // 2. Find the first set bit from the primary side for all the pieces in #1
4898 Register FBSReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
4899 if (!selectFirstBitSet32(ResVReg: FBSReg, ResType: PostCastType, I, SrcReg: BitcastReg, BitSetOpcode))
4900 return false;
4901
4902 // 3. Split result vector into high bits and low bits
4903 Register HighReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4904 Register LowReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4905
4906 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
4907 if (IsScalarRes) {
4908 // if scalar do a vector extract
4909 if (!selectOpWithSrcs(ResVReg: HighReg, ResType, I, Srcs: {FBSReg, ConstIntZero},
4910 Opcode: SPIRV::OpVectorExtractDynamic))
4911 return false;
4912 if (!selectOpWithSrcs(ResVReg: LowReg, ResType, I, Srcs: {FBSReg, ConstIntOne},
4913 Opcode: SPIRV::OpVectorExtractDynamic))
4914 return false;
4915 } else {
4916 // if vector do a shufflevector
4917 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4918 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4919 .addDef(RegNo: HighReg)
4920 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4921 .addUse(RegNo: FBSReg)
4922 // Per the spec, repeat the vector if only one vec is needed
4923 .addUse(RegNo: FBSReg);
4924
4925 // high bits are stored in even indexes. Extract them from FBSReg
4926 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
4927 MIB.addImm(Val: J);
4928 }
4929
4930 MIB.constrainAllUses(TII, TRI, RBI);
4931
4932 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4933 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
4934 .addDef(RegNo: LowReg)
4935 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4936 .addUse(RegNo: FBSReg)
4937 // Per the spec, repeat the vector if only one vec is needed
4938 .addUse(RegNo: FBSReg);
4939
4940 // low bits are stored in odd indexes. Extract them from FBSReg
4941 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
4942 MIB.addImm(Val: J);
4943 }
4944 MIB.constrainAllUses(TII, TRI, RBI);
4945 }
4946
4947 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
4948 // primary
4949 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
4950 Register NegOneReg;
4951 Register Reg0;
4952 Register Reg32;
4953 unsigned SelectOp;
4954 unsigned AddOp;
4955
4956 if (IsScalarRes) {
4957 NegOneReg =
4958 GR.getOrCreateConstInt(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
4959 Reg0 = GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
4960 Reg32 = GR.getOrCreateConstInt(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
4961 SelectOp = SPIRV::OpSelectSISCond;
4962 AddOp = SPIRV::OpIAddS;
4963 } else {
4964 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: ComponentCount,
4965 MIRBuilder, EmitIR: false);
4966 NegOneReg =
4967 GR.getOrCreateConstVector(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
4968 Reg0 = GR.getOrCreateConstVector(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
4969 Reg32 = GR.getOrCreateConstVector(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
4970 SelectOp = SPIRV::OpSelectVIVCond;
4971 AddOp = SPIRV::OpIAddV;
4972 }
4973
4974 Register PrimaryReg = HighReg;
4975 Register SecondaryReg = LowReg;
4976 Register PrimaryShiftReg = Reg32;
4977 Register SecondaryShiftReg = Reg0;
4978
4979 // By default the emitted opcodes check for the set bit from the MSB side.
4980 // Setting SwapPrimarySide checks the set bit from the LSB side
4981 if (SwapPrimarySide) {
4982 PrimaryReg = LowReg;
4983 SecondaryReg = HighReg;
4984 PrimaryShiftReg = Reg0;
4985 SecondaryShiftReg = Reg32;
4986 }
4987
4988 // Check if the primary bits are == -1
4989 Register BReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BoolType));
4990 if (!selectOpWithSrcs(ResVReg: BReg, ResType: BoolType, I, Srcs: {PrimaryReg, NegOneReg},
4991 Opcode: SPIRV::OpIEqual))
4992 return false;
4993
4994 // Select secondary bits if true in BReg, otherwise primary bits
4995 Register TmpReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4996 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {BReg, SecondaryReg, PrimaryReg},
4997 Opcode: SelectOp))
4998 return false;
4999
5000 // 5. Add 32 when high bits are used, otherwise 0 for low bits
5001 Register ValReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5002 if (!selectOpWithSrcs(ResVReg: ValReg, ResType, I,
5003 Srcs: {BReg, SecondaryShiftReg, PrimaryShiftReg}, Opcode: SelectOp))
5004 return false;
5005
5006 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {ValReg, TmpReg}, Opcode: AddOp);
5007}
5008
5009bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
5010 SPIRVTypeInst ResType,
5011 MachineInstr &I,
5012 bool IsSigned) const {
5013 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
5014 Register OpReg = I.getOperand(i: 2).getReg();
5015 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
5016 // zero or sign extend
5017 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5018 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
5019
5020 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
5021 case 16:
5022 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
5023 case 32:
5024 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
5025 case 64:
5026 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
5027 /*SwapPrimarySide=*/false);
5028 default:
5029 report_fatal_error(
5030 reason: "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
5031 }
5032}
5033
5034bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
5035 SPIRVTypeInst ResType,
5036 MachineInstr &I) const {
5037 // FindILsb intrinsic only supports 32 bit integers
5038 Register OpReg = I.getOperand(i: 2).getReg();
5039 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
5040 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
5041 // to an unsigned i32. As this leaves all the least significant bits unchanged
5042 // so the first set bit from the LSB side doesn't change.
5043 unsigned ExtendOpcode = SPIRV::OpUConvert;
5044 unsigned BitSetOpcode = GL::FindILsb;
5045
5046 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
5047 case 16:
5048 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
5049 case 32:
5050 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
5051 case 64:
5052 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
5053 /*SwapPrimarySide=*/true);
5054 default:
5055 report_fatal_error(reason: "spv_firstbitlow only supports 16,32,64 bits.");
5056 }
5057}
5058
5059bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
5060 SPIRVTypeInst ResType,
5061 MachineInstr &I) const {
5062 // there was an allocation size parameter to the allocation instruction
5063 // that is not 1
5064 MachineBasicBlock &BB = *I.getParent();
5065 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariableLengthArrayINTEL))
5066 .addDef(RegNo: ResVReg)
5067 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5068 .addUse(RegNo: I.getOperand(i: 2).getReg())
5069 .constrainAllUses(TII, TRI, RBI);
5070 if (!STI.isShader()) {
5071 unsigned Alignment = I.getOperand(i: 3).getImm();
5072 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::Alignment, DecArgs: {Alignment});
5073 }
5074 return true;
5075}
5076
5077bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
5078 SPIRVTypeInst ResType,
5079 MachineInstr &I) const {
5080 // Change order of instructions if needed: all OpVariable instructions in a
5081 // function must be the first instructions in the first block
5082 auto It = getOpVariableMBBIt(I);
5083 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
5084 .addDef(RegNo: ResVReg)
5085 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5086 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
5087 .constrainAllUses(TII, TRI, RBI);
5088 if (!STI.isShader()) {
5089 unsigned Alignment = I.getOperand(i: 2).getImm();
5090 buildOpDecorate(Reg: ResVReg, I&: *It, TII, Dec: SPIRV::Decoration::Alignment,
5091 DecArgs: {Alignment});
5092 }
5093 return true;
5094}
5095
5096bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
5097 // InstructionSelector walks backwards through the instructions. We can use
5098 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
5099 // first, so can generate an OpBranchConditional here. If there is no
5100 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
5101 const MachineInstr *PrevI = I.getPrevNode();
5102 MachineBasicBlock &MBB = *I.getParent();
5103 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
5104 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
5105 .addUse(RegNo: PrevI->getOperand(i: 0).getReg())
5106 .addMBB(MBB: PrevI->getOperand(i: 1).getMBB())
5107 .addMBB(MBB: I.getOperand(i: 0).getMBB())
5108 .constrainAllUses(TII, TRI, RBI);
5109 return true;
5110 }
5111 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranch))
5112 .addMBB(MBB: I.getOperand(i: 0).getMBB())
5113 .constrainAllUses(TII, TRI, RBI);
5114 return true;
5115}
5116
5117bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
5118 // InstructionSelector walks backwards through the instructions. For an
5119 // explicit conditional branch with no fallthrough, we use both a G_BR and a
5120 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
5121 // generate the OpBranchConditional in selectBranch above.
5122 //
5123 // If an OpBranchConditional has been generated, we simply return, as the work
5124 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
5125 // implicit fallthrough to the next basic block, so we need to create an
5126 // OpBranchConditional with an explicit "false" argument pointing to the next
5127 // basic block that LLVM would fall through to.
5128 const MachineInstr *NextI = I.getNextNode();
5129 // Check if this has already been successfully selected.
5130 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
5131 return true;
5132 // Must be relying on implicit block fallthrough, so generate an
5133 // OpBranchConditional with the "next" basic block as the "false" target.
5134 MachineBasicBlock &MBB = *I.getParent();
5135 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
5136 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(N: NextMBBNum);
5137 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
5138 .addUse(RegNo: I.getOperand(i: 0).getReg())
5139 .addMBB(MBB: I.getOperand(i: 1).getMBB())
5140 .addMBB(MBB: NextMBB)
5141 .constrainAllUses(TII, TRI, RBI);
5142 return true;
5143}
5144
5145bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
5146 SPIRVTypeInst ResType,
5147 MachineInstr &I) const {
5148 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpPhi))
5149 .addDef(RegNo: ResVReg)
5150 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
5151 const unsigned NumOps = I.getNumOperands();
5152 for (unsigned i = 1; i < NumOps; i += 2) {
5153 MIB.addUse(RegNo: I.getOperand(i: i + 0).getReg());
5154 MIB.addMBB(MBB: I.getOperand(i: i + 1).getMBB());
5155 }
5156 MIB.constrainAllUses(TII, TRI, RBI);
5157 MIB->setDesc(TII.get(Opcode: TargetOpcode::PHI));
5158 MIB->removeOperand(OpNo: 1);
5159 return true;
5160}
5161
5162bool SPIRVInstructionSelector::selectGlobalValue(
5163 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
5164 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
5165 MachineIRBuilder MIRBuilder(I);
5166 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
5167 Type *GVType = toTypedPointer(Ty: GR.getDeducedGlobalValueType(Global: GV));
5168
5169 std::string GlobalIdent;
5170 if (!GV->hasName()) {
5171 unsigned &ID = UnnamedGlobalIDs[GV];
5172 if (ID == 0)
5173 ID = UnnamedGlobalIDs.size();
5174 GlobalIdent = "__unnamed_" + Twine(ID).str();
5175 } else {
5176 GlobalIdent = GV->getName();
5177 }
5178
5179 // Behaviour of functions as operands depends on availability of the
5180 // corresponding extension (SPV_INTEL_function_pointers):
5181 // - If there is an extension to operate with functions as operands:
5182 // We create a proper constant operand and evaluate a correct type for a
5183 // function pointer.
5184 // - Without the required extension:
5185 // We have functions as operands in tests with blocks of instruction e.g. in
5186 // transcoding/global_block.ll. These operands are not used and should be
5187 // substituted by zero constants. Their type is expected to be always
5188 // OpTypePointer Function %uchar.
5189 if (isa<Function>(Val: GV)) {
5190 const Constant *ConstVal = GV;
5191 MachineBasicBlock &BB = *I.getParent();
5192 Register NewReg = GR.find(V: ConstVal, MF: GR.CurMF);
5193 if (!NewReg.isValid()) {
5194 Register NewReg = ResVReg;
5195 const Function *GVFun =
5196 STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
5197 ? dyn_cast<Function>(Val: GV)
5198 : nullptr;
5199 SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType(
5200 BaseType: GVType, I,
5201 SC: GVFun ? SPIRV::StorageClass::CodeSectionINTEL
5202 : addressSpaceToStorageClass(AddrSpace: GV->getAddressSpace(), STI));
5203 if (GVFun) {
5204 // References to a function via function pointers generate virtual
5205 // registers without a definition. We will resolve it later, during
5206 // module analysis stage.
5207 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
5208 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5209 Register FuncVReg =
5210 MRI->createGenericVirtualRegister(Ty: GR.getRegType(SpvType: ResType));
5211 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::pIDRegClass);
5212 MachineInstrBuilder MIB1 =
5213 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
5214 .addDef(RegNo: FuncVReg)
5215 .addUse(RegNo: ResTypeReg);
5216 MachineInstrBuilder MIB2 =
5217 BuildMI(BB, I, MIMD: I.getDebugLoc(),
5218 MCID: TII.get(Opcode: SPIRV::OpConstantFunctionPointerINTEL))
5219 .addDef(RegNo: NewReg)
5220 .addUse(RegNo: ResTypeReg)
5221 .addUse(RegNo: FuncVReg);
5222 GR.add(V: ConstVal, MI: MIB2);
5223 // mapping the function pointer to the used Function
5224 GR.recordFunctionPointer(MO: &MIB2.getInstr()->getOperand(i: 2), F: GVFun);
5225 MIB1.constrainAllUses(TII, TRI, RBI);
5226 MIB2.constrainAllUses(TII, TRI, RBI);
5227 return true;
5228 }
5229 MachineInstrBuilder MIB3 =
5230 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
5231 .addDef(RegNo: NewReg)
5232 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
5233 GR.add(V: ConstVal, MI: MIB3);
5234 MIB3.constrainAllUses(TII, TRI, RBI);
5235 return true;
5236 }
5237 assert(NewReg != ResVReg);
5238 return BuildCOPY(DestReg: ResVReg, SrcReg: NewReg, I);
5239 }
5240 auto GlobalVar = cast<GlobalVariable>(Val: GV);
5241 assert(GlobalVar->getName() != "llvm.global.annotations");
5242
5243 // Skip empty declaration for GVs with initializers till we get the decl with
5244 // passed initializer.
5245 if (hasInitializer(GV: GlobalVar) && !Init)
5246 return true;
5247
5248 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
5249 getSpirvLinkageTypeFor(ST: STI, GV: *GV);
5250
5251 const unsigned AddrSpace = GV->getAddressSpace();
5252 SPIRV::StorageClass::StorageClass StorageClass =
5253 addressSpaceToStorageClass(AddrSpace, STI);
5254 SPIRVTypeInst ResType =
5255 GR.getOrCreateSPIRVPointerType(BaseType: GVType, I, SC: StorageClass);
5256 Register Reg = GR.buildGlobalVariable(
5257 Reg: ResVReg, BaseType: ResType, Name: GlobalIdent, GV, Storage: StorageClass, Init,
5258 IsConst: GlobalVar->isConstant(), LinkageType: LnkType, MIRBuilder, IsInstSelector: true);
5259 // TODO: For AMDGCN, we pipe externally_initialized through via
5260 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
5261 // reverse translation. We should remove this once SPIR-V gains the ability to
5262 // express the concept.
5263 if (GlobalVar->isExternallyInitialized() &&
5264 STI.getTargetTriple().getVendor() == Triple::AMD) {
5265 constexpr unsigned ReadWriteINTEL = 3u;
5266 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::HostAccessINTEL,
5267 DecArgs: {ReadWriteINTEL});
5268 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
5269 addStringImm(Str: GV->getName(), MIB);
5270 }
5271 return Reg.isValid();
5272}
5273
5274bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
5275 SPIRVTypeInst ResType,
5276 MachineInstr &I) const {
5277 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5278 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log10);
5279 }
5280
5281 // There is no log10 instruction in the GLSL Extended Instruction set, so it
5282 // is implemented as:
5283 // log10(x) = log2(x) * (1 / log2(10))
5284 // = log2(x) * 0.30103
5285
5286 MachineIRBuilder MIRBuilder(I);
5287 MachineBasicBlock &BB = *I.getParent();
5288
5289 // Build log2(x).
5290 Register VarReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5291 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5292 .addDef(RegNo: VarReg)
5293 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5294 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
5295 .addImm(Val: GL::Log2)
5296 .add(MO: I.getOperand(i: 1))
5297 .constrainAllUses(TII, TRI, RBI);
5298
5299 // Build 0.30103.
5300 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
5301 ResType->getOpcode() == SPIRV::OpTypeFloat);
5302 // TODO: Add matrix implementation once supported by the HLSL frontend.
5303 SPIRVTypeInst SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector
5304 ? SPIRVTypeInst(GR.getSPIRVTypeForVReg(
5305 VReg: ResType->getOperand(i: 1).getReg()))
5306 : ResType;
5307 Register ScaleReg =
5308 GR.buildConstantFP(Val: APFloat(0.30103f), MIRBuilder, SpvType: SpirvScalarType);
5309
5310 // Multiply log2(x) by 0.30103 to get log10(x) result.
5311 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
5312 ? SPIRV::OpVectorTimesScalar
5313 : SPIRV::OpFMulS;
5314 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
5315 .addDef(RegNo: ResVReg)
5316 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5317 .addUse(RegNo: VarReg)
5318 .addUse(RegNo: ScaleReg)
5319 .constrainAllUses(TII, TRI, RBI);
5320 return true;
5321}
5322
5323bool SPIRVInstructionSelector::selectModf(Register ResVReg,
5324 SPIRVTypeInst ResType,
5325 MachineInstr &I) const {
5326 // llvm.modf has a single arg --the number to be decomposed-- and returns a
5327 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
5328 // number to be decomposed and a pointer--, returns the fractional part and
5329 // the integral part is stored in the pointer argument. Therefore, we can't
5330 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
5331 // scaffolding to make it work. The idea is to create an alloca instruction
5332 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
5333 // from this ptr to place it in the struct. llvm.modf returns the fractional
5334 // part as the first element of the result, and the integral part as the
5335 // second element of the result.
5336
5337 // At this point, the return type is not a struct anymore, but rather two
5338 // independent elements of SPIRVResType. We can get each independent element
5339 // from I.getDefs() or I.getOperands().
5340 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5341 MachineIRBuilder MIRBuilder(I);
5342 // Get pointer type for alloca variable.
5343 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5344 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Function);
5345 // Create new register for the pointer type of alloca variable.
5346 Register PtrTyReg =
5347 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5348 MIRBuilder.getMRI()->setType(
5349 VReg: PtrTyReg,
5350 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Function),
5351 SizeInBits: GR.getPointerSize()));
5352
5353 // Assign SPIR-V type of the pointer type of the alloca variable to the
5354 // new register.
5355 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: PtrTyReg, MF: MIRBuilder.getMF());
5356 MachineBasicBlock &EntryBB = I.getMF()->front();
5357 MachineBasicBlock::iterator VarPos =
5358 getFirstValidInstructionInsertPoint(BB&: EntryBB);
5359 auto AllocaMIB =
5360 BuildMI(BB&: EntryBB, I: VarPos, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
5361 .addDef(RegNo: PtrTyReg)
5362 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PtrType))
5363 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function));
5364 Register Variable = AllocaMIB->getOperand(i: 0).getReg();
5365
5366 MachineBasicBlock &BB = *I.getParent();
5367 // Create the OpenCLLIB::modf instruction.
5368 auto MIB =
5369 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5370 .addDef(RegNo: ResVReg)
5371 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5372 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
5373 .addImm(Val: CL::modf)
5374 .setMIFlags(I.getFlags())
5375 .add(MO: I.getOperand(i: I.getNumExplicitDefs())) // Floating point value.
5376 .addUse(RegNo: Variable); // Pointer to integral part.
5377 // Assign the integral part stored in the ptr to the second element of the
5378 // result.
5379 Register IntegralPartReg = I.getOperand(i: 1).getReg();
5380 if (IntegralPartReg.isValid()) {
5381 // Load the value from the pointer to integral part.
5382 auto LoadMIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5383 .addDef(RegNo: IntegralPartReg)
5384 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5385 .addUse(RegNo: Variable);
5386 LoadMIB.constrainAllUses(TII, TRI, RBI);
5387 return true;
5388 }
5389
5390 MIB.constrainAllUses(TII, TRI, RBI);
5391 return true;
5392 } else if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
5393 assert(false && "GLSL::Modf is deprecated.");
5394 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
5395 return false;
5396 }
5397 return false;
5398}
5399
5400// Generate the instructions to load 3-element vector builtin input
5401// IDs/Indices.
5402// Like: GlobalInvocationId, LocalInvocationId, etc....
5403
5404bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
5405 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5406 SPIRVTypeInst ResType, MachineInstr &I) const {
5407 MachineIRBuilder MIRBuilder(I);
5408 const SPIRVTypeInst Vec3Ty =
5409 GR.getOrCreateSPIRVVectorType(BaseType: ResType, NumElements: 3, MIRBuilder, EmitIR: false);
5410 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5411 BaseType: Vec3Ty, MIRBuilder, SC: SPIRV::StorageClass::Input);
5412
5413 // Create new register for the input ID builtin variable.
5414 Register NewRegister =
5415 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5416 MIRBuilder.getMRI()->setType(VReg: NewRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5417 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5418
5419 // Build global variable with the necessary decorations for the input ID
5420 // builtin variable.
5421 Register Variable = GR.buildGlobalVariable(
5422 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5423 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5424 IsInstSelector: false);
5425
5426 // Create new register for loading value.
5427 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5428 Register LoadedRegister = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5429 MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5430 GR.assignSPIRVTypeToVReg(Type: Vec3Ty, VReg: LoadedRegister, MF: MIRBuilder.getMF());
5431
5432 // Load v3uint value from the global variable.
5433 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5434 .addDef(RegNo: LoadedRegister)
5435 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: Vec3Ty))
5436 .addUse(RegNo: Variable);
5437
5438 // Get the input ID index. Expecting operand is a constant immediate value,
5439 // wrapped in a type assignment.
5440 assert(I.getOperand(2).isReg());
5441 const uint32_t ThreadId = foldImm(MO: I.getOperand(i: 2), MRI);
5442
5443 // Extract the input ID from the loaded vector value.
5444 MachineBasicBlock &BB = *I.getParent();
5445 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
5446 .addDef(RegNo: ResVReg)
5447 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5448 .addUse(RegNo: LoadedRegister)
5449 .addImm(Val: ThreadId);
5450 MIB.constrainAllUses(TII, TRI, RBI);
5451 return true;
5452}
5453
5454// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
5455// Like LocalInvocationIndex
5456bool SPIRVInstructionSelector::loadBuiltinInputID(
5457 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5458 SPIRVTypeInst ResType, MachineInstr &I) const {
5459 MachineIRBuilder MIRBuilder(I);
5460 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5461 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Input);
5462
5463 // Create new register for the input ID builtin variable.
5464 Register NewRegister =
5465 MIRBuilder.getMRI()->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PtrType));
5466 MIRBuilder.getMRI()->setType(
5467 VReg: NewRegister,
5468 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Input),
5469 SizeInBits: GR.getPointerSize()));
5470 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5471
5472 // Build global variable with the necessary decorations for the input ID
5473 // builtin variable.
5474 Register Variable = GR.buildGlobalVariable(
5475 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5476 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5477 IsInstSelector: false);
5478
5479 // Load uint value from the global variable.
5480 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5481 .addDef(RegNo: ResVReg)
5482 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5483 .addUse(RegNo: Variable);
5484
5485 MIB.constrainAllUses(TII, TRI, RBI);
5486 return true;
5487}
5488
5489SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type,
5490 MachineInstr &I) const {
5491 MachineIRBuilder MIRBuilder(I);
5492 if (Type->getOpcode() != SPIRV::OpTypeVector)
5493 return GR.getOrCreateSPIRVVectorType(BaseType: Type, NumElements: 4, MIRBuilder, EmitIR: false);
5494
5495 uint64_t VectorSize = Type->getOperand(i: 2).getImm();
5496 if (VectorSize == 4)
5497 return Type;
5498
5499 Register ScalarTypeReg = Type->getOperand(i: 1).getReg();
5500 const SPIRVTypeInst ScalarType = GR.getSPIRVTypeForVReg(VReg: ScalarTypeReg);
5501 return GR.getOrCreateSPIRVVectorType(BaseType: ScalarType, NumElements: 4, MIRBuilder, EmitIR: false);
5502}
5503
5504bool SPIRVInstructionSelector::loadHandleBeforePosition(
5505 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
5506 MachineInstr &Pos) const {
5507
5508 assert(HandleDef.getIntrinsicID() ==
5509 Intrinsic::spv_resource_handlefrombinding);
5510 uint32_t Set = foldImm(MO: HandleDef.getOperand(i: 2), MRI);
5511 uint32_t Binding = foldImm(MO: HandleDef.getOperand(i: 3), MRI);
5512 uint32_t ArraySize = foldImm(MO: HandleDef.getOperand(i: 4), MRI);
5513 Register IndexReg = HandleDef.getOperand(i: 5).getReg();
5514 std::string Name =
5515 getStringValueFromReg(Reg: HandleDef.getOperand(i: 6).getReg(), MRI&: *MRI);
5516
5517 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
5518 MachineIRBuilder MIRBuilder(HandleDef);
5519 SPIRVTypeInst VarType = ResType;
5520 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
5521
5522 if (IsStructuredBuffer) {
5523 VarType = GR.getPointeeType(PtrType: ResType);
5524 SC = GR.getPointerStorageClass(Type: ResType);
5525 }
5526
5527 Register VarReg =
5528 buildPointerToResource(SpirvResType: SPIRVTypeInst(VarType), SC, Set, Binding,
5529 ArraySize, IndexReg, Name, MIRBuilder);
5530
5531 // The handle for the buffer is the pointer to the resource. For an image, the
5532 // handle is the image object. So images get an extra load.
5533 uint32_t LoadOpcode =
5534 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
5535 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: HandleReg, MF: *Pos.getMF());
5536 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: HandleDef.getDebugLoc(), MCID: TII.get(Opcode: LoadOpcode))
5537 .addDef(RegNo: HandleReg)
5538 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5539 .addUse(RegNo: VarReg)
5540 .constrainAllUses(TII, TRI, RBI);
5541 return true;
5542}
5543
5544void SPIRVInstructionSelector::errorIfInstrOutsideShader(
5545 MachineInstr &I) const {
5546 if (!STI.isShader()) {
5547 std::string DiagMsg;
5548 raw_string_ostream OS(DiagMsg);
5549 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
5550 DiagMsg += " is only supported in shaders.\n";
5551 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
5552 }
5553}
5554
5555namespace llvm {
5556InstructionSelector *
5557createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
5558 const SPIRVSubtarget &Subtarget,
5559 const RegisterBankInfo &RBI) {
5560 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
5561}
5562} // namespace llvm
5563