1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVUtils.h"
23#include "llvm/ADT/APFloat.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/Register.h"
31#include "llvm/CodeGen/TargetOpcodes.h"
32#include "llvm/IR/DiagnosticInfo.h"
33#include "llvm/IR/IntrinsicsSPIRV.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36
37#define DEBUG_TYPE "spirv-isel"
38
39using namespace llvm;
40namespace CL = SPIRV::OpenCLExtInst;
41namespace GL = SPIRV::GLSLExtInst;
42
43using ExtInstList =
44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
45
46namespace {
47
48struct ImageOperands {
49 std::optional<Register> Bias;
50 std::optional<Register> Offset;
51 std::optional<Register> MinLod;
52 std::optional<Register> GradX;
53 std::optional<Register> GradY;
54 std::optional<Register> Lod;
55 std::optional<Register> Compare;
56};
57
58llvm::SPIRV::SelectionControl::SelectionControl
59getSelectionOperandForImm(int Imm) {
60 if (Imm == 2)
61 return SPIRV::SelectionControl::Flatten;
62 if (Imm == 1)
63 return SPIRV::SelectionControl::DontFlatten;
64 if (Imm == 0)
65 return SPIRV::SelectionControl::None;
66 llvm_unreachable("Invalid immediate");
67}
68
69#define GET_GLOBALISEL_PREDICATE_BITSET
70#include "SPIRVGenGlobalISel.inc"
71#undef GET_GLOBALISEL_PREDICATE_BITSET
72
73class SPIRVInstructionSelector : public InstructionSelector {
74 const SPIRVSubtarget &STI;
75 const SPIRVInstrInfo &TII;
76 const SPIRVRegisterInfo &TRI;
77 const RegisterBankInfo &RBI;
78 SPIRVGlobalRegistry &GR;
79 MachineRegisterInfo *MRI;
80 MachineFunction *HasVRegsReset = nullptr;
81
82 /// We need to keep track of the number we give to anonymous global values to
83 /// generate the same name every time when this is needed.
84 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
85 SmallPtrSet<MachineInstr *, 8> DeadMIs;
86
87public:
88 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
89 const SPIRVSubtarget &ST,
90 const RegisterBankInfo &RBI);
91 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
92 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
93 BlockFrequencyInfo *BFI) override;
94 // Common selection code. Instruction-specific selection occurs in spvSelect.
95 bool select(MachineInstr &I) override;
96 static const char *getName() { return DEBUG_TYPE; }
97
98#define GET_GLOBALISEL_PREDICATES_DECL
99#include "SPIRVGenGlobalISel.inc"
100#undef GET_GLOBALISEL_PREDICATES_DECL
101
102#define GET_GLOBALISEL_TEMPORARIES_DECL
103#include "SPIRVGenGlobalISel.inc"
104#undef GET_GLOBALISEL_TEMPORARIES_DECL
105
106private:
107 void resetVRegsType(MachineFunction &MF);
108 void removeDeadInstruction(MachineInstr &MI) const;
109 void removeOpNamesForDeadMI(MachineInstr &MI) const;
110
111 // tblgen-erated 'select' implementation, used as the initial selector for
112 // the patterns that don't require complex C++.
113 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
114
115 // All instruction-specific selection that didn't happen in "select()".
116 // Is basically a large Switch/Case delegating to all other select method.
117 bool spvSelect(Register ResVReg, SPIRVTypeInst ResType,
118 MachineInstr &I) const;
119
120 bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType,
121 MachineInstr &I, bool IsSigned) const;
122
123 bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType,
124 MachineInstr &I) const;
125
126 bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType,
127 MachineInstr &I, unsigned ExtendOpcode,
128 unsigned BitSetOpcode) const;
129
130 bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType,
131 MachineInstr &I, Register SrcReg,
132 unsigned BitSetOpcode) const;
133
134 bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType,
135 MachineInstr &I, Register SrcReg,
136 unsigned BitSetOpcode, bool SwapPrimarySide) const;
137
138 bool selectFirstBitSet64Overflow(Register ResVReg, SPIRVTypeInst ResType,
139 MachineInstr &I, Register SrcReg,
140 unsigned BitSetOpcode,
141 bool SwapPrimarySide) const;
142
143 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
144 const MachineInstr *Init = nullptr) const;
145
146 bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType,
147 MachineInstr &I, std::vector<Register> SrcRegs,
148 unsigned Opcode) const;
149
150 bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
151 unsigned Opcode) const;
152
153 bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType,
154 MachineInstr &I) const;
155
156 bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
157 MachineInstr &I) const;
158 bool selectStore(MachineInstr &I) const;
159
160 bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
161 MachineInstr &I) const;
162 bool selectStackRestore(MachineInstr &I) const;
163
164 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
165 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
166 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
167 bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
168
169 bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
170 unsigned NewOpcode, unsigned NegateOpcode = 0) const;
171
172 bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType,
173 MachineInstr &I) const;
174
175 bool selectFence(MachineInstr &I) const;
176
177 bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType,
178 MachineInstr &I) const;
179
180 bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
181 unsigned OpType) const;
182
183 bool selectAll(Register ResVReg, SPIRVTypeInst ResType,
184 MachineInstr &I) const;
185
186 bool selectAny(Register ResVReg, SPIRVTypeInst ResType,
187 MachineInstr &I) const;
188
189 bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType,
190 MachineInstr &I) const;
191
192 bool selectBitreverse16(Register ResVReg, SPIRVTypeInst ResType,
193 MachineInstr &I, Register Op) const;
194
195 bool selectBitreverse32(Register ResVReg, SPIRVTypeInst ResType,
196 MachineInstr &I, Register Op) const;
197
198 bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType,
199 MachineInstr &I) const;
200 bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType,
201 MachineInstr &I) const;
202
203 bool selectCmp(Register ResVReg, SPIRVTypeInst ResType,
204 unsigned comparisonOpcode, MachineInstr &I) const;
205 bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType,
206 MachineInstr &I) const;
207
208 bool selectICmp(Register ResVReg, SPIRVTypeInst ResType,
209 MachineInstr &I) const;
210 bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType,
211 MachineInstr &I) const;
212
213 bool selectSign(Register ResVReg, SPIRVTypeInst ResType,
214 MachineInstr &I) const;
215
216 bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType,
217 MachineInstr &I) const;
218
219 bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType,
220 MachineInstr &I, unsigned Opcode) const;
221 bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType,
222 MachineInstr &I) const;
223
224 bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType,
225 MachineInstr &I, bool Signed) const;
226
227 bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType,
228 MachineInstr &I) const;
229
230 bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType,
231 MachineInstr &I) const;
232
233 bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType,
234 MachineInstr &I) const;
235
236 template <bool Signed>
237 bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType,
238 MachineInstr &I) const;
239 template <bool Signed>
240 bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType,
241 MachineInstr &I) const;
242
243 bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType,
244 MachineInstr &I) const;
245
246 template <typename PickOpcodeFn>
247 bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType,
248 MachineInstr &I, bool IsUnsigned,
249 PickOpcodeFn &&PickOpcode) const;
250
251 bool selectWaveReduceOp(Register ResVReg, SPIRVTypeInst ResType,
252 MachineInstr &I, unsigned Opcode) const;
253
254 bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType,
255 MachineInstr &I, bool IsUnsigned) const;
256
257 bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType,
258 MachineInstr &I, bool IsUnsigned) const;
259
260 bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType,
261 MachineInstr &I) const;
262
263 bool selectWaveReduceProduct(Register ResVReg, const SPIRVTypeInst ResType,
264 MachineInstr &I) const;
265
266 template <typename PickOpcodeFn>
267 bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType,
268 MachineInstr &I, bool IsUnsigned,
269 PickOpcodeFn &&PickOpcode) const;
270
271 bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType,
272 MachineInstr &I) const;
273
274 bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
275 MachineInstr &I) const;
276
277 bool selectConst(Register ResVReg, SPIRVTypeInst ResType,
278 MachineInstr &I) const;
279
280 bool selectSelect(Register ResVReg, SPIRVTypeInst ResType,
281 MachineInstr &I) const;
282 bool selectBoolToInt(Register ResVReg, SPIRVTypeInst ResType,
283 Register BooleanVReg, MachineInstr &InsertAt,
284 bool IsSigned) const;
285 bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
286 bool IsSigned, unsigned Opcode) const;
287 bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
288 bool IsSigned) const;
289
290 bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType,
291 MachineInstr &I) const;
292
293 bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
294 bool IsSigned) const;
295
296 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
297 SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const;
298
299 bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType,
300 MachineInstr &I) const;
301 bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType,
302 MachineInstr &I) const;
303 bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType,
304 MachineInstr &I) const;
305 bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType,
306 MachineInstr &I) const;
307 bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType,
308 MachineInstr &I) const;
309 bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType,
310 MachineInstr &I) const;
311 bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType,
312 MachineInstr &I) const;
313 bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
314 MachineInstr &I) const;
315
316 bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
317 MachineInstr &I) const;
318 bool selectMaskedScatter(MachineInstr &I) const;
319
320 bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const;
321
322 bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
323 MachineInstr &I) const;
324 bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
325 MachineInstr &I) const;
326
327 bool selectBranch(MachineInstr &I) const;
328 bool selectBranchCond(MachineInstr &I) const;
329
330 bool selectPhi(Register ResVReg, MachineInstr &I) const;
331
332 bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I,
333 GL::GLSLExtInst GLInst, bool setMIFlags = true,
334 bool useMISrc = true,
335 ArrayRef<Register> SrcRegs = {}) const;
336 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
337 CL::OpenCLExtInst CLInst, bool setMIFlags = true,
338 bool useMISrc = true,
339 ArrayRef<Register> SrcRegs = {}) const;
340 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
341 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
342 bool setMIFlags = true, bool useMISrc = true,
343 ArrayRef<Register> SrcRegs = {}) const;
344 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
345 const ExtInstList &ExtInsts, bool setMIFlags = true,
346 bool useMISrc = true,
347 ArrayRef<Register> SrcRegs = {}) const;
348
349 bool selectLog10(Register ResVReg, SPIRVTypeInst ResType,
350 MachineInstr &I) const;
351
352 bool selectFpowi(Register ResVReg, SPIRVTypeInst ResType,
353 MachineInstr &I) const;
354
355 bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType,
356 MachineInstr &I) const;
357
358 bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType,
359 MachineInstr &I, unsigned Opcode) const;
360
361 bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType,
362 MachineInstr &I) const;
363
364 bool selectWaveActiveAllEqual(Register ResVReg, SPIRVTypeInst ResType,
365 MachineInstr &I) const;
366
367 bool selectUnmergeValues(MachineInstr &I) const;
368
369 bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
370 MachineInstr &I) const;
371
372 bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
373 MachineInstr &I) const;
374
375 bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
376 MachineInstr &I) const;
377 bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
378 MachineInstr &I) const;
379 bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
380 MachineInstr &I) const;
381 bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
382 MachineInstr &I) const;
383 bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
384 MachineInstr &I) const;
385 bool selectLoadLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
386 MachineInstr &I) const;
387 bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
388 MachineInstr &I) const;
389 bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg,
390 SPIRVTypeInst ResType,
391 MachineInstr &I) const;
392 bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
393 MachineInstr &I) const;
394 bool selectImageWriteIntrinsic(MachineInstr &I) const;
395 bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
396 MachineInstr &I) const;
397 bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
398 MachineInstr &I) const;
399 bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType,
400 MachineInstr &I) const;
401 bool selectModf(Register ResVReg, SPIRVTypeInst ResType,
402 MachineInstr &I) const;
403 bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType,
404 MachineInstr &I) const;
405 bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType,
406 MachineInstr &I) const;
407 bool selectSincos(Register ResVReg, SPIRVTypeInst ResType,
408 MachineInstr &I) const;
409 bool selectExp10(Register ResVReg, SPIRVTypeInst ResType,
410 MachineInstr &I) const;
411 bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType,
412 MachineInstr &I, const unsigned DPdOpCode) const;
413 // Utilities
414 Register buildI32Constant(uint32_t Val, MachineInstr &I,
415 SPIRVTypeInst ResType = nullptr) const;
416
417 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const;
418 bool isScalarOrVectorIntConstantZero(Register Reg) const;
419 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const;
420 Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
421 MachineInstr &I) const;
422 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
423
424 bool wrapIntoSpecConstantOp(MachineInstr &I,
425 SmallVector<Register> &CompositeArgs) const;
426
427 Register getUcharPtrTypeReg(MachineInstr &I,
428 SPIRV::StorageClass::StorageClass SC) const;
429 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
430 Register Src, Register DestType,
431 uint32_t Opcode) const;
432 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
433 SPIRVTypeInst SrcPtrTy) const;
434 Register buildPointerToResource(SPIRVTypeInst ResType,
435 SPIRV::StorageClass::StorageClass SC,
436 uint32_t Set, uint32_t Binding,
437 uint32_t ArraySize, Register IndexReg,
438 StringRef Name,
439 MachineIRBuilder MIRBuilder) const;
440 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const;
441 bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType,
442 Register &ReadReg, MachineInstr &InsertionPoint) const;
443 bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType,
444 Register ImageReg, Register IdxReg,
445 DebugLoc Loc, MachineInstr &Pos,
446 const ImageOperands *ImOps = nullptr) const;
447 bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType,
448 Register ImageReg, Register SamplerReg,
449 Register CoordinateReg, const ImageOperands &ImOps,
450 DebugLoc Loc, MachineInstr &I) const;
451 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
452 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
453 Register ResVReg, SPIRVTypeInst ResType,
454 MachineInstr &I) const;
455 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
456 Register ResVReg, SPIRVTypeInst ResType,
457 MachineInstr &I) const;
458 bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType,
459 GIntrinsic &HandleDef, MachineInstr &Pos) const;
460 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
461 void errorIfInstrOutsideShader(MachineInstr &I) const;
462};
463
464bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
465 const TargetExtType *TET = cast<TargetExtType>(Val: HandleType);
466 if (TET->getTargetExtName() == "spirv.Image") {
467 return false;
468 }
469 assert(TET->getTargetExtName() == "spirv.SignedImage");
470 return TET->getTypeParameter(i: 0)->isIntegerTy();
471}
472} // end anonymous namespace
473
474#define GET_GLOBALISEL_IMPL
475#include "SPIRVGenGlobalISel.inc"
476#undef GET_GLOBALISEL_IMPL
477
478SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
479 const SPIRVSubtarget &ST,
480 const RegisterBankInfo &RBI)
481 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
482 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
483 MRI(nullptr),
484#define GET_GLOBALISEL_PREDICATES_INIT
485#include "SPIRVGenGlobalISel.inc"
486#undef GET_GLOBALISEL_PREDICATES_INIT
487#define GET_GLOBALISEL_TEMPORARIES_INIT
488#include "SPIRVGenGlobalISel.inc"
489#undef GET_GLOBALISEL_TEMPORARIES_INIT
490{
491}
492
493void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
494 GISelValueTracking *VT,
495 CodeGenCoverage *CoverageInfo,
496 ProfileSummaryInfo *PSI,
497 BlockFrequencyInfo *BFI) {
498 MRI = &MF.getRegInfo();
499 GR.setCurrentFunc(MF);
500 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
501}
502
503// Ensure that register classes correspond to pattern matching rules.
504void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
505 if (HasVRegsReset == &MF)
506 return;
507 HasVRegsReset = &MF;
508
509 MachineRegisterInfo &MRI = MF.getRegInfo();
510 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
511 Register Reg = Register::index2VirtReg(Index: I);
512 LLT RegType = MRI.getType(Reg);
513 if (RegType.isScalar())
514 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: 64));
515 else if (RegType.isPointer())
516 MRI.setType(VReg: Reg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
517 else if (RegType.isVector())
518 MRI.setType(VReg: Reg, Ty: LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::scalar(SizeInBits: 64)));
519 }
520 for (const auto &MBB : MF) {
521 for (const auto &MI : MBB) {
522 if (isPreISelGenericOpcode(Opcode: MI.getOpcode()))
523 GR.erase(MI: &MI);
524 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
525 continue;
526
527 Register DstReg = MI.getOperand(i: 0).getReg();
528 LLT DstType = MRI.getType(Reg: DstReg);
529 Register SrcReg = MI.getOperand(i: 1).getReg();
530 LLT SrcType = MRI.getType(Reg: SrcReg);
531 if (DstType != SrcType)
532 MRI.setType(VReg: DstReg, Ty: MRI.getType(Reg: SrcReg));
533
534 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(Reg: DstReg);
535 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(Reg: SrcReg);
536 if (DstRC != SrcRC && SrcRC)
537 MRI.setRegClass(Reg: DstReg, RC: SrcRC);
538 }
539 }
540}
541
542// Return true if the MachineInstr represents a constant register
543static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef) {
544
545 SmallVector<MachineInstr *> Stack = {OpDef};
546 SmallPtrSet<MachineInstr *, 4> Visited;
547
548 while (!Stack.empty()) {
549 MachineInstr *MI = Stack.pop_back_val();
550 MI = passCopy(Def: MI, MRI);
551 if (!Visited.insert(Ptr: MI).second)
552 continue;
553 switch (MI->getOpcode()) {
554 case TargetOpcode::G_INTRINSIC:
555 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
556 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
557 GIntrinsic *GIntr = cast<GIntrinsic>(Val: MI);
558 unsigned IntrID = GIntr->getIntrinsicID();
559 if (IntrID != Intrinsic::spv_const_composite &&
560 IntrID != Intrinsic::spv_undef)
561 return false;
562 continue;
563 }
564 case TargetOpcode::G_BUILD_VECTOR:
565 case TargetOpcode::G_SPLAT_VECTOR:
566 for (unsigned i = OpDef->getNumExplicitDefs();
567 i < OpDef->getNumOperands(); i++) {
568 if (!OpDef->getOperand(i).isReg())
569 continue;
570 MachineInstr *OpNestedDef =
571 MRI->getVRegDef(Reg: OpDef->getOperand(i).getReg());
572 Stack.push_back(Elt: OpNestedDef);
573 }
574 continue;
575 case TargetOpcode::G_CONSTANT:
576 case TargetOpcode::G_FCONSTANT:
577 case TargetOpcode::G_IMPLICIT_DEF:
578 case SPIRV::OpConstantTrue:
579 case SPIRV::OpConstantFalse:
580 case SPIRV::OpConstantI:
581 case SPIRV::OpConstantF:
582 case SPIRV::OpConstantComposite:
583 case SPIRV::OpConstantCompositeContinuedINTEL:
584 case SPIRV::OpConstantSampler:
585 case SPIRV::OpConstantNull:
586 case SPIRV::OpUndef:
587 case SPIRV::OpConstantFunctionPointerINTEL:
588 continue;
589 default:
590 return false;
591 }
592 }
593 return true;
594}
595
596// Return true if the virtual register represents a constant
597static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
598 if (MachineInstr *OpDef = MRI->getVRegDef(Reg: OpReg))
599 return isConstReg(MRI, OpDef);
600 return false;
601}
602
603// TODO(168736): We should make this either a flag in tabelgen
604// or reduce our dependence on the global registry, so we can remove this
605// function. It can easily be missed when new intrinsics are added.
606
607// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
608// definition because they are referenced in the global registry. This is a list
609// of intrinsics that have no side effects other than their references in the
610// global registry.
611static bool intrinsicHasSideEffects(Intrinsic::ID ID) {
612 switch (ID) {
613 // This is not an exhaustive list and may need to be updated.
614 case Intrinsic::spv_all:
615 case Intrinsic::spv_alloca:
616 case Intrinsic::spv_any:
617 case Intrinsic::spv_bitcast:
618 case Intrinsic::spv_const_composite:
619 case Intrinsic::spv_cross:
620 case Intrinsic::spv_degrees:
621 case Intrinsic::spv_distance:
622 case Intrinsic::spv_extractelt:
623 case Intrinsic::spv_extractv:
624 case Intrinsic::spv_faceforward:
625 case Intrinsic::spv_fdot:
626 case Intrinsic::spv_firstbitlow:
627 case Intrinsic::spv_firstbitshigh:
628 case Intrinsic::spv_firstbituhigh:
629 case Intrinsic::spv_frac:
630 case Intrinsic::spv_gep:
631 case Intrinsic::spv_global_offset:
632 case Intrinsic::spv_global_size:
633 case Intrinsic::spv_group_id:
634 case Intrinsic::spv_insertelt:
635 case Intrinsic::spv_insertv:
636 case Intrinsic::spv_isinf:
637 case Intrinsic::spv_isnan:
638 case Intrinsic::spv_lerp:
639 case Intrinsic::spv_length:
640 case Intrinsic::spv_normalize:
641 case Intrinsic::spv_num_subgroups:
642 case Intrinsic::spv_num_workgroups:
643 case Intrinsic::spv_ptrcast:
644 case Intrinsic::spv_radians:
645 case Intrinsic::spv_reflect:
646 case Intrinsic::spv_refract:
647 case Intrinsic::spv_resource_getpointer:
648 case Intrinsic::spv_resource_handlefrombinding:
649 case Intrinsic::spv_resource_handlefromimplicitbinding:
650 case Intrinsic::spv_resource_nonuniformindex:
651 case Intrinsic::spv_resource_sample:
652 case Intrinsic::spv_rsqrt:
653 case Intrinsic::spv_saturate:
654 case Intrinsic::spv_sdot:
655 case Intrinsic::spv_sign:
656 case Intrinsic::spv_smoothstep:
657 case Intrinsic::spv_step:
658 case Intrinsic::spv_subgroup_id:
659 case Intrinsic::spv_subgroup_local_invocation_id:
660 case Intrinsic::spv_subgroup_max_size:
661 case Intrinsic::spv_subgroup_size:
662 case Intrinsic::spv_thread_id:
663 case Intrinsic::spv_thread_id_in_group:
664 case Intrinsic::spv_udot:
665 case Intrinsic::spv_undef:
666 case Intrinsic::spv_value_md:
667 case Intrinsic::spv_workgroup_size:
668 return false;
669 default:
670 return true;
671 }
672}
673
674// TODO(168736): We should make this either a flag in tabelgen
675// or reduce our dependence on the global registry, so we can remove this
676// function. It can easily be missed when new intrinsics are added.
677static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
678 switch (Opcode) {
679 case SPIRV::OpTypeVoid:
680 case SPIRV::OpTypeBool:
681 case SPIRV::OpTypeInt:
682 case SPIRV::OpTypeFloat:
683 case SPIRV::OpTypeVector:
684 case SPIRV::OpTypeMatrix:
685 case SPIRV::OpTypeImage:
686 case SPIRV::OpTypeSampler:
687 case SPIRV::OpTypeSampledImage:
688 case SPIRV::OpTypeArray:
689 case SPIRV::OpTypeRuntimeArray:
690 case SPIRV::OpTypeStruct:
691 case SPIRV::OpTypeOpaque:
692 case SPIRV::OpTypePointer:
693 case SPIRV::OpTypeFunction:
694 case SPIRV::OpTypeEvent:
695 case SPIRV::OpTypeDeviceEvent:
696 case SPIRV::OpTypeReserveId:
697 case SPIRV::OpTypeQueue:
698 case SPIRV::OpTypePipe:
699 case SPIRV::OpTypeForwardPointer:
700 case SPIRV::OpTypePipeStorage:
701 case SPIRV::OpTypeNamedBarrier:
702 case SPIRV::OpTypeAccelerationStructureNV:
703 case SPIRV::OpTypeCooperativeMatrixNV:
704 case SPIRV::OpTypeCooperativeMatrixKHR:
705 return true;
706 default:
707 return false;
708 }
709}
710
711bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
712 // If there are no definitions, then assume there is some other
713 // side-effect that makes this instruction live.
714 if (MI.getNumDefs() == 0)
715 return false;
716
717 for (const auto &MO : MI.all_defs()) {
718 Register Reg = MO.getReg();
719 if (Reg.isPhysical()) {
720 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
721 return false;
722 }
723 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
724 if (UseMI.getOpcode() != SPIRV::OpName) {
725 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
726 return false;
727 }
728 }
729 }
730
731 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
732 MI.isLifetimeMarker()) {
733 LLVM_DEBUG(
734 dbgs()
735 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
736 return false;
737 }
738 if (MI.isPHI()) {
739 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
740 return true;
741 }
742
743 // It is possible that the only side effect is that the instruction is
744 // referenced in the global registry. If that is the only side effect, the
745 // intrinsic is dead.
746 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
747 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
748 const auto &Intr = cast<GIntrinsic>(Val: MI);
749 if (!intrinsicHasSideEffects(ID: Intr.getIntrinsicID())) {
750 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
751 return true;
752 }
753 }
754
755 if (MI.mayStore() || MI.isCall() ||
756 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
757 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
758 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
759 return false;
760 }
761
762 if (isPreISelGenericOpcode(Opcode: MI.getOpcode())) {
763 // TODO: Is there a generic way to check if the opcode has side effects?
764 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
765 return true;
766 }
767
768 if (isOpcodeWithNoSideEffects(Opcode: MI.getOpcode())) {
769 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
770 return true;
771 }
772
773 return false;
774}
775
776void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
777 // Delete the OpName that uses the result if there is one.
778 for (const auto &MO : MI.all_defs()) {
779 Register Reg = MO.getReg();
780 if (Reg.isPhysical())
781 continue;
782 SmallVector<MachineInstr *, 4> UselessOpNames;
783 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
784 assert(UseMI.getOpcode() == SPIRV::OpName &&
785 "There is still a use of the dead function.");
786 UselessOpNames.push_back(Elt: &UseMI);
787 }
788 for (MachineInstr *OpNameMI : UselessOpNames) {
789 GR.invalidateMachineInstr(MI: OpNameMI);
790 OpNameMI->eraseFromParent();
791 }
792 }
793}
794
795void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
796 salvageDebugInfo(MRI: *MRI, MI);
797 GR.invalidateMachineInstr(MI: &MI);
798 removeOpNamesForDeadMI(MI);
799 MI.eraseFromParent();
800}
801
802bool SPIRVInstructionSelector::select(MachineInstr &I) {
803 resetVRegsType(MF&: *I.getParent()->getParent());
804
805 assert(I.getParent() && "Instruction should be in a basic block!");
806 assert(I.getParent()->getParent() && "Instruction should be in a function!");
807
808 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
809 if (isDead(MI: I, MRI: *MRI)) {
810 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
811 removeDeadInstruction(MI&: I);
812 return true;
813 }
814
815 Register Opcode = I.getOpcode();
816 // If it's not a GMIR instruction, we've selected it already.
817 if (!isPreISelGenericOpcode(Opcode)) {
818 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
819 Register DstReg = I.getOperand(i: 0).getReg();
820 Register SrcReg = I.getOperand(i: 1).getReg();
821 auto *Def = MRI->getVRegDef(Reg: SrcReg);
822 if (isTypeFoldingSupported(Opcode: Def->getOpcode()) &&
823 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
824 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
825 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
826 Register SelectDstReg = Def->getOperand(i: 0).getReg();
827 bool SuccessToSelectSelect [[maybe_unused]] = selectSelect(
828 ResVReg: SelectDstReg, ResType: GR.getSPIRVTypeForVReg(VReg: SelectDstReg), I&: *Def);
829 assert(SuccessToSelectSelect);
830 GR.invalidateMachineInstr(MI: Def);
831 Def->eraseFromParent();
832 MRI->replaceRegWith(FromReg: DstReg, ToReg: SelectDstReg);
833 GR.invalidateMachineInstr(MI: &I);
834 I.eraseFromParent();
835 return true;
836 }
837
838 bool Res = selectImpl(I, CoverageInfo&: *CoverageInfo);
839 LLVM_DEBUG({
840 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
841 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
842 I.print(dbgs());
843 }
844 });
845 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
846 if (Res) {
847 if (!isTriviallyDead(MI: *Def, MRI: *MRI) && isDead(MI: *Def, MRI: *MRI))
848 DeadMIs.insert(Ptr: Def);
849 return Res;
850 }
851 }
852 MRI->setRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg));
853 MRI->replaceRegWith(FromReg: SrcReg, ToReg: DstReg);
854 GR.invalidateMachineInstr(MI: &I);
855 I.eraseFromParent();
856 return true;
857 } else if (I.getNumDefs() == 1) {
858 // Make all vregs 64 bits (for SPIR-V IDs).
859 MRI->setType(VReg: I.getOperand(i: 0).getReg(), Ty: LLT::scalar(SizeInBits: 64));
860 }
861 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
862 return true;
863 }
864
865 if (DeadMIs.contains(Ptr: &I)) {
866 // if the instruction has been already made dead by folding it away
867 // erase it
868 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
869 removeDeadInstruction(MI&: I);
870 DeadMIs.erase(Ptr: &I);
871 return true;
872 }
873
874 if (I.getNumOperands() != I.getNumExplicitOperands()) {
875 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
876 return false;
877 }
878
879 // Common code for getting return reg+type, and removing selected instr
880 // from parent occurs here. Instr-specific selection happens in spvSelect().
881 bool HasDefs = I.getNumDefs() > 0;
882 Register ResVReg = HasDefs ? I.getOperand(i: 0).getReg() : Register(0);
883 SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(VReg: ResVReg) : nullptr;
884 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
885 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
886 if (spvSelect(ResVReg, ResType, I)) {
887 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
888 for (unsigned i = 0; i < I.getNumDefs(); ++i)
889 MRI->setType(VReg: I.getOperand(i).getReg(), Ty: LLT::scalar(SizeInBits: 64));
890 GR.invalidateMachineInstr(MI: &I);
891 I.eraseFromParent();
892 return true;
893 }
894 return false;
895}
896
897static bool mayApplyGenericSelection(unsigned Opcode) {
898 switch (Opcode) {
899 case TargetOpcode::G_CONSTANT:
900 case TargetOpcode::G_FCONSTANT:
901 return false;
902 case TargetOpcode::G_SADDO:
903 case TargetOpcode::G_SSUBO:
904 return true;
905 }
906 return isTypeFoldingSupported(Opcode);
907}
908
909bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
910 MachineInstr &I) const {
911 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: DestReg);
912 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(Reg: SrcReg);
913 if (DstRC != SrcRC && SrcRC)
914 MRI->setRegClass(Reg: DestReg, RC: SrcRC);
915 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY))
916 .addDef(RegNo: DestReg)
917 .addUse(RegNo: SrcReg)
918 .constrainAllUses(TII, TRI, RBI);
919 return true;
920}
921
922bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
923 SPIRVTypeInst ResType,
924 MachineInstr &I) const {
925 const unsigned Opcode = I.getOpcode();
926 if (mayApplyGenericSelection(Opcode))
927 return selectImpl(I, CoverageInfo&: *CoverageInfo);
928 switch (Opcode) {
929 case TargetOpcode::G_CONSTANT:
930 case TargetOpcode::G_FCONSTANT:
931 return selectConst(ResVReg, ResType, I);
932 case TargetOpcode::G_GLOBAL_VALUE:
933 return selectGlobalValue(ResVReg, I);
934 case TargetOpcode::G_IMPLICIT_DEF:
935 return selectOpUndef(ResVReg, ResType, I);
936 case TargetOpcode::G_FREEZE:
937 return selectFreeze(ResVReg, ResType, I);
938
939 case TargetOpcode::G_INTRINSIC:
940 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
941 case TargetOpcode::G_INTRINSIC_CONVERGENT:
942 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
943 return selectIntrinsic(ResVReg, ResType, I);
944 case TargetOpcode::G_BITREVERSE:
945 return selectBitreverse(ResVReg, ResType, I);
946
947 case TargetOpcode::G_BUILD_VECTOR:
948 return selectBuildVector(ResVReg, ResType, I);
949 case TargetOpcode::G_SPLAT_VECTOR:
950 return selectSplatVector(ResVReg, ResType, I);
951
952 case TargetOpcode::G_SHUFFLE_VECTOR: {
953 MachineBasicBlock &BB = *I.getParent();
954 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
955 .addDef(RegNo: ResVReg)
956 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
957 .addUse(RegNo: I.getOperand(i: 1).getReg())
958 .addUse(RegNo: I.getOperand(i: 2).getReg());
959 for (auto V : I.getOperand(i: 3).getShuffleMask())
960 MIB.addImm(Val: V);
961 MIB.constrainAllUses(TII, TRI, RBI);
962 return true;
963 }
964 case TargetOpcode::G_MEMMOVE:
965 case TargetOpcode::G_MEMCPY:
966 case TargetOpcode::G_MEMSET:
967 return selectMemOperation(ResVReg, I);
968
969 case TargetOpcode::G_ICMP:
970 return selectICmp(ResVReg, ResType, I);
971 case TargetOpcode::G_FCMP:
972 return selectFCmp(ResVReg, ResType, I);
973
974 case TargetOpcode::G_FRAME_INDEX:
975 return selectFrameIndex(ResVReg, ResType, I);
976
977 case TargetOpcode::G_LOAD:
978 return selectLoad(ResVReg, ResType, I);
979 case TargetOpcode::G_STORE:
980 return selectStore(I);
981
982 case TargetOpcode::G_BR:
983 return selectBranch(I);
984 case TargetOpcode::G_BRCOND:
985 return selectBranchCond(I);
986
987 case TargetOpcode::G_PHI:
988 return selectPhi(ResVReg, I);
989
990 case TargetOpcode::G_FPTOSI:
991 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
992 case TargetOpcode::G_FPTOUI:
993 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
994
995 case TargetOpcode::G_FPTOSI_SAT:
996 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToS);
997 case TargetOpcode::G_FPTOUI_SAT:
998 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertFToU);
999
1000 case TargetOpcode::G_SITOFP:
1001 return selectIToF(ResVReg, ResType, I, IsSigned: true, Opcode: SPIRV::OpConvertSToF);
1002 case TargetOpcode::G_UITOFP:
1003 return selectIToF(ResVReg, ResType, I, IsSigned: false, Opcode: SPIRV::OpConvertUToF);
1004
1005 case TargetOpcode::G_CTPOP:
1006 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitCount);
1007 case TargetOpcode::G_SMIN:
1008 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_min, GLInst: GL::SMin);
1009 case TargetOpcode::G_UMIN:
1010 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_min, GLInst: GL::UMin);
1011
1012 case TargetOpcode::G_SMAX:
1013 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_max, GLInst: GL::SMax);
1014 case TargetOpcode::G_UMAX:
1015 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_max, GLInst: GL::UMax);
1016
1017 case TargetOpcode::G_SCMP:
1018 return selectSUCmp(ResVReg, ResType, I, IsSigned: true);
1019 case TargetOpcode::G_UCMP:
1020 return selectSUCmp(ResVReg, ResType, I, IsSigned: false);
1021 case TargetOpcode::G_LROUND:
1022 case TargetOpcode::G_LLROUND: {
1023 Register regForLround =
1024 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ResVReg), Name: "lround");
1025 MRI->setRegClass(Reg: regForLround, RC: &SPIRV::iIDRegClass);
1026 GR.assignSPIRVTypeToVReg(Type: GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg()),
1027 VReg: regForLround, MF: *(I.getParent()->getParent()));
1028 selectExtInst(ResVReg: regForLround, ResType: GR.getSPIRVTypeForVReg(VReg: regForLround), I,
1029 CLInst: CL::round, GLInst: GL::Round, /* setMIFlags */ false);
1030 MachineBasicBlock &BB = *I.getParent();
1031 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConvertFToS))
1032 .addDef(RegNo: ResVReg)
1033 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1034 .addUse(RegNo: regForLround);
1035 MIB.constrainAllUses(TII, TRI, RBI);
1036 return true;
1037 }
1038 case TargetOpcode::G_STRICT_FMA:
1039 case TargetOpcode::G_FMA: {
1040 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
1041 MachineBasicBlock &BB = *I.getParent();
1042 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFmaKHR))
1043 .addDef(RegNo: ResVReg)
1044 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1045 .addUse(RegNo: I.getOperand(i: 1).getReg())
1046 .addUse(RegNo: I.getOperand(i: 2).getReg())
1047 .addUse(RegNo: I.getOperand(i: 3).getReg())
1048 .setMIFlags(I.getFlags());
1049 MIB.constrainAllUses(TII, TRI, RBI);
1050 return true;
1051 }
1052 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fma, GLInst: GL::Fma);
1053 }
1054
1055 case TargetOpcode::G_STRICT_FLDEXP:
1056 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ldexp);
1057
1058 case TargetOpcode::G_FPOW:
1059 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pow, GLInst: GL::Pow);
1060 case TargetOpcode::G_FPOWI:
1061 return selectFpowi(ResVReg, ResType, I);
1062
1063 case TargetOpcode::G_FEXP:
1064 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp, GLInst: GL::Exp);
1065 case TargetOpcode::G_FEXP2:
1066 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp2, GLInst: GL::Exp2);
1067 case TargetOpcode::G_FEXP10:
1068 return selectExp10(ResVReg, ResType, I);
1069
1070 case TargetOpcode::G_FMODF:
1071 return selectModf(ResVReg, ResType, I);
1072 case TargetOpcode::G_FSINCOS:
1073 return selectSincos(ResVReg, ResType, I);
1074
1075 case TargetOpcode::G_FLOG:
1076 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log, GLInst: GL::Log);
1077 case TargetOpcode::G_FLOG2:
1078 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log2, GLInst: GL::Log2);
1079 case TargetOpcode::G_FLOG10:
1080 return selectLog10(ResVReg, ResType, I);
1081
1082 case TargetOpcode::G_FABS:
1083 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fabs, GLInst: GL::FAbs);
1084 case TargetOpcode::G_ABS:
1085 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_abs, GLInst: GL::SAbs);
1086
1087 case TargetOpcode::G_FMINNUM:
1088 case TargetOpcode::G_FMINIMUM:
1089 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmin, GLInst: GL::NMin);
1090 case TargetOpcode::G_FMAXNUM:
1091 case TargetOpcode::G_FMAXIMUM:
1092 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fmax, GLInst: GL::NMax);
1093
1094 case TargetOpcode::G_FCOPYSIGN:
1095 return selectExtInst(ResVReg, ResType, I, CLInst: CL::copysign);
1096
1097 case TargetOpcode::G_FCEIL:
1098 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ceil, GLInst: GL::Ceil);
1099 case TargetOpcode::G_FFLOOR:
1100 return selectExtInst(ResVReg, ResType, I, CLInst: CL::floor, GLInst: GL::Floor);
1101
1102 case TargetOpcode::G_FCOS:
1103 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cos, GLInst: GL::Cos);
1104 case TargetOpcode::G_FSIN:
1105 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sin, GLInst: GL::Sin);
1106 case TargetOpcode::G_FTAN:
1107 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tan, GLInst: GL::Tan);
1108 case TargetOpcode::G_FACOS:
1109 return selectExtInst(ResVReg, ResType, I, CLInst: CL::acos, GLInst: GL::Acos);
1110 case TargetOpcode::G_FASIN:
1111 return selectExtInst(ResVReg, ResType, I, CLInst: CL::asin, GLInst: GL::Asin);
1112 case TargetOpcode::G_FATAN:
1113 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan, GLInst: GL::Atan);
1114 case TargetOpcode::G_FATAN2:
1115 return selectExtInst(ResVReg, ResType, I, CLInst: CL::atan2, GLInst: GL::Atan2);
1116 case TargetOpcode::G_FCOSH:
1117 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cosh, GLInst: GL::Cosh);
1118 case TargetOpcode::G_FSINH:
1119 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sinh, GLInst: GL::Sinh);
1120 case TargetOpcode::G_FTANH:
1121 return selectExtInst(ResVReg, ResType, I, CLInst: CL::tanh, GLInst: GL::Tanh);
1122
1123 case TargetOpcode::G_STRICT_FSQRT:
1124 case TargetOpcode::G_FSQRT:
1125 return selectExtInst(ResVReg, ResType, I, CLInst: CL::sqrt, GLInst: GL::Sqrt);
1126
1127 case TargetOpcode::G_CTTZ:
1128 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1129 return selectExtInst(ResVReg, ResType, I, CLInst: CL::ctz);
1130 case TargetOpcode::G_CTLZ:
1131 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1132 return selectExtInst(ResVReg, ResType, I, CLInst: CL::clz);
1133
1134 case TargetOpcode::G_INTRINSIC_ROUND:
1135 return selectExtInst(ResVReg, ResType, I, CLInst: CL::round, GLInst: GL::Round);
1136 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1137 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1138 case TargetOpcode::G_INTRINSIC_TRUNC:
1139 return selectExtInst(ResVReg, ResType, I, CLInst: CL::trunc, GLInst: GL::Trunc);
1140 case TargetOpcode::G_FRINT:
1141 case TargetOpcode::G_FNEARBYINT:
1142 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rint, GLInst: GL::RoundEven);
1143
1144 case TargetOpcode::G_SMULH:
1145 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_mul_hi);
1146 case TargetOpcode::G_UMULH:
1147 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_mul_hi);
1148
1149 case TargetOpcode::G_SADDSAT:
1150 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_add_sat);
1151 case TargetOpcode::G_UADDSAT:
1152 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_add_sat);
1153 case TargetOpcode::G_SSUBSAT:
1154 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_sub_sat);
1155 case TargetOpcode::G_USUBSAT:
1156 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_sub_sat);
1157
1158 case TargetOpcode::G_FFREXP:
1159 return selectFrexp(ResVReg, ResType, I);
1160
1161 case TargetOpcode::G_UADDO:
1162 return selectOverflowArith(ResVReg, ResType, I,
1163 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1164 ? SPIRV::OpIAddCarryV
1165 : SPIRV::OpIAddCarryS);
1166 case TargetOpcode::G_USUBO:
1167 return selectOverflowArith(ResVReg, ResType, I,
1168 Opcode: ResType->getOpcode() == SPIRV::OpTypeVector
1169 ? SPIRV::OpISubBorrowV
1170 : SPIRV::OpISubBorrowS);
1171 case TargetOpcode::G_UMULO:
1172 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpUMulExtended);
1173 case TargetOpcode::G_SMULO:
1174 return selectOverflowArith(ResVReg, ResType, I, Opcode: SPIRV::OpSMulExtended);
1175
1176 case TargetOpcode::G_SEXT:
1177 return selectExt(ResVReg, ResType, I, IsSigned: true);
1178 case TargetOpcode::G_ANYEXT:
1179 case TargetOpcode::G_ZEXT:
1180 return selectExt(ResVReg, ResType, I, IsSigned: false);
1181 case TargetOpcode::G_TRUNC:
1182 return selectTrunc(ResVReg, ResType, I);
1183 case TargetOpcode::G_FPTRUNC:
1184 case TargetOpcode::G_FPEXT:
1185 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpFConvert);
1186
1187 case TargetOpcode::G_PTRTOINT:
1188 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertPtrToU);
1189 case TargetOpcode::G_INTTOPTR:
1190 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpConvertUToPtr);
1191 case TargetOpcode::G_BITCAST:
1192 return selectBitcast(ResVReg, ResType, I);
1193 case TargetOpcode::G_ADDRSPACE_CAST:
1194 return selectAddrSpaceCast(ResVReg, ResType, I);
1195 case TargetOpcode::G_PTR_ADD: {
1196 // Currently, we get G_PTR_ADD only applied to global variables.
1197 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1198 Register GV = I.getOperand(i: 1).getReg();
1199 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(RegNo: GV);
1200 (void)II;
1201 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1202 (*II).getOpcode() == TargetOpcode::COPY ||
1203 (*II).getOpcode() == SPIRV::OpVariable) &&
1204 getImm(I.getOperand(2), MRI));
1205 // It may be the initialization of a global variable.
1206 bool IsGVInit = false;
1207 for (MachineRegisterInfo::use_instr_iterator
1208 UseIt = MRI->use_instr_begin(RegNo: I.getOperand(i: 0).getReg()),
1209 UseEnd = MRI->use_instr_end();
1210 UseIt != UseEnd; UseIt = std::next(x: UseIt)) {
1211 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1212 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1213 (*UseIt).getOpcode() == SPIRV::OpVariable) {
1214 IsGVInit = true;
1215 break;
1216 }
1217 }
1218 MachineBasicBlock &BB = *I.getParent();
1219 if (!IsGVInit) {
1220 SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(VReg: GV);
1221 SPIRVTypeInst GVPointeeType = GR.getPointeeType(PtrType: GVType);
1222 SPIRVTypeInst ResPointeeType = GR.getPointeeType(PtrType: ResType);
1223 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1224 // Build a new virtual register that is associated with the required
1225 // data type.
1226 Register NewVReg = MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: GV));
1227 MRI->setRegClass(Reg: NewVReg, RC: MRI->getRegClass(Reg: GV));
1228 // Having a correctly typed base we are ready to build the actually
1229 // required GEP. It may not be a constant though, because all Operands
1230 // of OpSpecConstantOp is to originate from other const instructions,
1231 // and only the AccessChain named opcodes accept a global OpVariable
1232 // instruction. We can't use an AccessChain opcode because of the type
1233 // mismatch between result and base types.
1234 if (!GR.isBitcastCompatible(Type1: ResType, Type2: GVType))
1235 report_fatal_error(
1236 reason: "incompatible result and operand types in a bitcast");
1237 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
1238 MachineInstrBuilder MIB =
1239 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitcast))
1240 .addDef(RegNo: NewVReg)
1241 .addUse(RegNo: ResTypeReg)
1242 .addUse(RegNo: GV);
1243 MIB.constrainAllUses(TII, TRI, RBI);
1244 BuildMI(BB, I, MIMD: I.getDebugLoc(),
1245 MCID: TII.get(Opcode: STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain
1246 : SPIRV::OpInBoundsPtrAccessChain))
1247 .addDef(RegNo: ResVReg)
1248 .addUse(RegNo: ResTypeReg)
1249 .addUse(RegNo: NewVReg)
1250 .addUse(RegNo: I.getOperand(i: 2).getReg())
1251 .constrainAllUses(TII, TRI, RBI);
1252 } else {
1253 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1254 .addDef(RegNo: ResVReg)
1255 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1256 .addImm(
1257 Val: static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1258 .addUse(RegNo: GV)
1259 .addUse(RegNo: I.getOperand(i: 2).getReg())
1260 .constrainAllUses(TII, TRI, RBI);
1261 }
1262 return true;
1263 }
1264 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1265 // initialize a global variable with a constant expression (e.g., the test
1266 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1267 Register Idx = buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
1268 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1269 .addDef(RegNo: ResVReg)
1270 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1271 .addImm(Val: static_cast<uint32_t>(
1272 SPIRV::Opcode::InBoundsPtrAccessChain))
1273 .addUse(RegNo: GV)
1274 .addUse(RegNo: Idx)
1275 .addUse(RegNo: I.getOperand(i: 2).getReg());
1276 MIB.constrainAllUses(TII, TRI, RBI);
1277 return true;
1278 }
1279
1280 case TargetOpcode::G_ATOMICRMW_OR:
1281 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicOr);
1282 case TargetOpcode::G_ATOMICRMW_ADD:
1283 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicIAdd);
1284 case TargetOpcode::G_ATOMICRMW_AND:
1285 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicAnd);
1286 case TargetOpcode::G_ATOMICRMW_MAX:
1287 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMax);
1288 case TargetOpcode::G_ATOMICRMW_MIN:
1289 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicSMin);
1290 case TargetOpcode::G_ATOMICRMW_SUB:
1291 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicISub);
1292 case TargetOpcode::G_ATOMICRMW_XOR:
1293 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicXor);
1294 case TargetOpcode::G_ATOMICRMW_UMAX:
1295 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMax);
1296 case TargetOpcode::G_ATOMICRMW_UMIN:
1297 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicUMin);
1298 case TargetOpcode::G_ATOMICRMW_XCHG:
1299 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicExchange);
1300 case TargetOpcode::G_ATOMIC_CMPXCHG:
1301 return selectAtomicCmpXchg(ResVReg, ResType, I);
1302
1303 case TargetOpcode::G_ATOMICRMW_FADD:
1304 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT);
1305 case TargetOpcode::G_ATOMICRMW_FSUB:
1306 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1307 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFAddEXT,
1308 NegateOpcode: ResType->getOpcode() == SPIRV::OpTypeVector
1309 ? SPIRV::OpFNegateV
1310 : SPIRV::OpFNegate);
1311 case TargetOpcode::G_ATOMICRMW_FMIN:
1312 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMinEXT);
1313 case TargetOpcode::G_ATOMICRMW_FMAX:
1314 return selectAtomicRMW(ResVReg, ResType, I, NewOpcode: SPIRV::OpAtomicFMaxEXT);
1315
1316 case TargetOpcode::G_FENCE:
1317 return selectFence(I);
1318
1319 case TargetOpcode::G_STACKSAVE:
1320 return selectStackSave(ResVReg, ResType, I);
1321 case TargetOpcode::G_STACKRESTORE:
1322 return selectStackRestore(I);
1323
1324 case TargetOpcode::G_UNMERGE_VALUES:
1325 return selectUnmergeValues(I);
1326
1327 // Discard gen opcodes for intrinsics which we do not expect to actually
1328 // represent code after lowering or intrinsics which are not implemented but
1329 // should not crash when found in a customer's LLVM IR input.
1330 case TargetOpcode::G_TRAP:
1331 case TargetOpcode::G_UBSANTRAP:
1332 case TargetOpcode::DBG_LABEL:
1333 return true;
1334 case TargetOpcode::G_DEBUGTRAP:
1335 return selectDebugTrap(ResVReg, ResType, I);
1336
1337 default:
1338 return false;
1339 }
1340}
1341
1342bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1343 SPIRVTypeInst ResType,
1344 MachineInstr &I) const {
1345 unsigned Opcode = SPIRV::OpNop;
1346 MachineBasicBlock &BB = *I.getParent();
1347 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1348 .constrainAllUses(TII, TRI, RBI);
1349 return true;
1350}
1351
1352bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1353 SPIRVTypeInst ResType,
1354 MachineInstr &I,
1355 GL::GLSLExtInst GLInst,
1356 bool setMIFlags, bool useMISrc,
1357 ArrayRef<Register> SrcRegs) const {
1358 if (!STI.canUseExtInstSet(
1359 E: SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) {
1360 std::string DiagMsg;
1361 raw_string_ostream OS(DiagMsg);
1362 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
1363 DiagMsg += " is only supported with the GLSL extended instruction set.\n";
1364 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1365 }
1366 return selectExtInst(ResVReg, ResType, I,
1367 ExtInsts: {{SPIRV::InstructionSet::GLSL_std_450, GLInst}},
1368 setMIFlags, useMISrc, SrcRegs);
1369}
1370
1371bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1372 SPIRVTypeInst ResType,
1373 MachineInstr &I,
1374 CL::OpenCLExtInst CLInst,
1375 bool setMIFlags, bool useMISrc,
1376 ArrayRef<Register> SrcRegs) const {
1377 return selectExtInst(ResVReg, ResType, I,
1378 ExtInsts: {{SPIRV::InstructionSet::OpenCL_std, CLInst}},
1379 setMIFlags, useMISrc, SrcRegs);
1380}
1381
1382bool SPIRVInstructionSelector::selectExtInst(
1383 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1384 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, bool setMIFlags,
1385 bool useMISrc, ArrayRef<Register> SrcRegs) const {
1386 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1387 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1388 return selectExtInst(ResVReg, ResType, I, ExtInsts, setMIFlags, useMISrc,
1389 SrcRegs);
1390}
1391
1392bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1393 SPIRVTypeInst ResType,
1394 MachineInstr &I,
1395 const ExtInstList &Insts,
1396 bool setMIFlags, bool useMISrc,
1397 ArrayRef<Register> SrcRegs) const {
1398
1399 for (const auto &[InstructionSet, Opcode] : Insts) {
1400 if (!STI.canUseExtInstSet(E: InstructionSet))
1401 continue;
1402 MachineBasicBlock &BB = *I.getParent();
1403 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1404 .addDef(RegNo: ResVReg)
1405 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1406 .addImm(Val: static_cast<uint32_t>(InstructionSet))
1407 .addImm(Val: Opcode);
1408 if (setMIFlags)
1409 MIB.setMIFlags(I.getFlags());
1410 if (useMISrc) {
1411 const unsigned NumOps = I.getNumOperands();
1412 unsigned Index = 1;
1413 if (Index < NumOps &&
1414 I.getOperand(i: Index).getType() ==
1415 MachineOperand::MachineOperandType::MO_IntrinsicID)
1416 Index = 2;
1417 for (; Index < NumOps; ++Index)
1418 MIB.add(MO: I.getOperand(i: Index));
1419 } else {
1420 for (Register SReg : SrcRegs) {
1421 MIB.addUse(RegNo: SReg);
1422 }
1423 }
1424 MIB.constrainAllUses(TII, TRI, RBI);
1425 return true;
1426 }
1427 return false;
1428}
1429
1430bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1431 SPIRVTypeInst ResType,
1432 MachineInstr &I) const {
1433 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1434 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1435 for (const auto &Ex : ExtInsts) {
1436 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1437 uint32_t Opcode = Ex.second;
1438 if (!STI.canUseExtInstSet(E: Set))
1439 continue;
1440
1441 MachineIRBuilder MIRBuilder(I);
1442 SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 1).getReg());
1443 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1444 BaseType: PointeeTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
1445 Register PointerVReg =
1446 createVirtualRegister(SpvType: PointerType, GR: &GR, MRI, MF: MRI->getMF());
1447
1448 auto It = getOpVariableMBBIt(I);
1449 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1450 .addDef(RegNo: PointerVReg)
1451 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointerType))
1452 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
1453 .constrainAllUses(TII, TRI, RBI);
1454
1455 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1456 .addDef(RegNo: ResVReg)
1457 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1458 .addImm(Val: static_cast<uint32_t>(Ex.first))
1459 .addImm(Val: Opcode)
1460 .add(MO: I.getOperand(i: 2))
1461 .addUse(RegNo: PointerVReg)
1462 .constrainAllUses(TII, TRI, RBI);
1463
1464 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1465 .addDef(RegNo: I.getOperand(i: 1).getReg())
1466 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointeeTy))
1467 .addUse(RegNo: PointerVReg)
1468 .constrainAllUses(TII, TRI, RBI);
1469 return true;
1470 }
1471 return false;
1472}
1473
1474bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
1475 SPIRVTypeInst ResType,
1476 MachineInstr &I) const {
1477 Register CosResVReg = I.getOperand(i: 1).getReg();
1478 unsigned SrcIdx = I.getNumExplicitDefs();
1479 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
1480
1481 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
1482 // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr.
1483 MachineIRBuilder MIRBuilder(I);
1484 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1485 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Function);
1486 Register PointerVReg =
1487 createVirtualRegister(SpvType: PointerType, GR: &GR, MRI, MF: MRI->getMF());
1488
1489 auto It = getOpVariableMBBIt(I);
1490 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1491 .addDef(RegNo: PointerVReg)
1492 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PointerType))
1493 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
1494 .constrainAllUses(TII, TRI, RBI);
1495 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1496 .addDef(RegNo: ResVReg)
1497 .addUse(RegNo: ResTypeReg)
1498 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1499 .addImm(Val: CL::sincos)
1500 .add(MO: I.getOperand(i: SrcIdx))
1501 .addUse(RegNo: PointerVReg)
1502 .constrainAllUses(TII, TRI, RBI);
1503 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1504 .addDef(RegNo: CosResVReg)
1505 .addUse(RegNo: ResTypeReg)
1506 .addUse(RegNo: PointerVReg)
1507 .constrainAllUses(TII, TRI, RBI);
1508 return true;
1509 } else if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
1510 // GLSL.std.450 has no combined sincos; emit separate Sin and Cos.
1511 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1512 .addDef(RegNo: ResVReg)
1513 .addUse(RegNo: ResTypeReg)
1514 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1515 .addImm(Val: GL::Sin)
1516 .add(MO: I.getOperand(i: SrcIdx))
1517 .constrainAllUses(TII, TRI, RBI);
1518 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
1519 .addDef(RegNo: CosResVReg)
1520 .addUse(RegNo: ResTypeReg)
1521 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1522 .addImm(Val: GL::Cos)
1523 .add(MO: I.getOperand(i: SrcIdx))
1524 .constrainAllUses(TII, TRI, RBI);
1525 return true;
1526 }
1527 return false;
1528}
1529
1530bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1531 SPIRVTypeInst ResType,
1532 MachineInstr &I,
1533 std::vector<Register> Srcs,
1534 unsigned Opcode) const {
1535 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
1536 .addDef(RegNo: ResVReg)
1537 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
1538 for (Register SReg : Srcs) {
1539 MIB.addUse(RegNo: SReg);
1540 }
1541 MIB.constrainAllUses(TII, TRI, RBI);
1542 return true;
1543}
1544
1545bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1546 SPIRVTypeInst ResType,
1547 MachineInstr &I,
1548 unsigned Opcode) const {
1549 if (STI.isPhysicalSPIRV() && I.getOperand(i: 1).isReg()) {
1550 Register SrcReg = I.getOperand(i: 1).getReg();
1551 bool IsGV = false;
1552 for (MachineRegisterInfo::def_instr_iterator DefIt =
1553 MRI->def_instr_begin(RegNo: SrcReg);
1554 DefIt != MRI->def_instr_end(); DefIt = std::next(x: DefIt)) {
1555 unsigned DefOpCode = DefIt->getOpcode();
1556 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1557 // We need special handling to look through the type assignment or the
1558 // COPY pseudo-op and see if this is a constant or a global.
1559 if (auto *VRD = getVRegDef(MRI&: *MRI, Reg: DefIt->getOperand(i: 1).getReg()))
1560 DefOpCode = VRD->getOpcode();
1561 }
1562 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1563 DefOpCode == TargetOpcode::G_CONSTANT ||
1564 DefOpCode == SPIRV::OpVariable || DefOpCode == SPIRV::OpConstantI) {
1565 IsGV = true;
1566 break;
1567 }
1568 }
1569 if (IsGV) {
1570 uint32_t SpecOpcode = 0;
1571 switch (Opcode) {
1572 case SPIRV::OpConvertPtrToU:
1573 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1574 break;
1575 case SPIRV::OpConvertUToPtr:
1576 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1577 break;
1578 }
1579 if (SpecOpcode) {
1580 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1581 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
1582 .addDef(RegNo: ResVReg)
1583 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1584 .addImm(Val: SpecOpcode)
1585 .addUse(RegNo: SrcReg)
1586 .constrainAllUses(TII, TRI, RBI);
1587 return true;
1588 }
1589 }
1590 }
1591 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {I.getOperand(i: 1).getReg()},
1592 Opcode);
1593}
1594
1595bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1596 SPIRVTypeInst ResType,
1597 MachineInstr &I) const {
1598 Register OpReg = I.getOperand(i: 1).getReg();
1599 SPIRVTypeInst OpType =
1600 OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
1601 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
1602 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
1603 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpBitcast);
1604}
1605
1606static void addMemoryOperands(MachineMemOperand *MemOp,
1607 MachineInstrBuilder &MIB,
1608 MachineIRBuilder &MIRBuilder,
1609 SPIRVGlobalRegistry &GR) {
1610 const SPIRVSubtarget *ST =
1611 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1612 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1613 if (MemOp->isVolatile())
1614 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1615 if (MemOp->isNonTemporal())
1616 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1617 // Aligned memory operand requires the Kernel capability.
1618 if (!ST->isShader() && MemOp->getAlign().value())
1619 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1620
1621 [[maybe_unused]] MachineInstr *AliasList = nullptr;
1622 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
1623 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
1624 if (auto *MD = MemOp->getAAInfo().Scope) {
1625 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1626 if (AliasList)
1627 SpvMemOp |=
1628 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
1629 }
1630 if (auto *MD = MemOp->getAAInfo().NoAlias) {
1631 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD: MD);
1632 if (NoAliasList)
1633 SpvMemOp |=
1634 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
1635 }
1636 }
1637
1638 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1639 MIB.addImm(Val: SpvMemOp);
1640 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1641 MIB.addImm(Val: MemOp->getAlign().value());
1642 if (AliasList)
1643 MIB.addUse(RegNo: AliasList->getOperand(i: 0).getReg());
1644 if (NoAliasList)
1645 MIB.addUse(RegNo: NoAliasList->getOperand(i: 0).getReg());
1646 }
1647}
1648
1649static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
1650 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1651 if (Flags & MachineMemOperand::Flags::MOVolatile)
1652 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1653 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1654 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1655
1656 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1657 MIB.addImm(Val: SpvMemOp);
1658}
1659
1660bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1661 SPIRVTypeInst ResType,
1662 MachineInstr &I) const {
1663 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1664 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1665
1666 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1667 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1668 if (IntPtrDef &&
1669 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1670 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1671 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1672 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1673 Register NewHandleReg =
1674 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1675 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1676 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1677 return false;
1678 }
1679
1680 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1681 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewHandleReg, IdxReg,
1682 Loc: I.getDebugLoc(), Pos&: I);
1683 }
1684 }
1685
1686 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
1687 .addDef(RegNo: ResVReg)
1688 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1689 .addUse(RegNo: Ptr);
1690 if (!I.getNumMemOperands()) {
1691 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1692 I.getOpcode() ==
1693 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1694 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1695 } else {
1696 MachineIRBuilder MIRBuilder(I);
1697 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1698 }
1699 MIB.constrainAllUses(TII, TRI, RBI);
1700 return true;
1701}
1702
1703bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1704 unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0;
1705 Register StoreVal = I.getOperand(i: 0 + OpOffset).getReg();
1706 Register Ptr = I.getOperand(i: 1 + OpOffset).getReg();
1707
1708 auto *PtrDef = getVRegDef(MRI&: *MRI, Reg: Ptr);
1709 auto *IntPtrDef = dyn_cast<GIntrinsic>(Val: PtrDef);
1710 if (IntPtrDef &&
1711 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1712 Register HandleReg = IntPtrDef->getOperand(i: 2).getReg();
1713 Register NewHandleReg =
1714 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: HandleReg));
1715 auto *HandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: HandleReg));
1716 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(VReg: HandleReg);
1717 if (!loadHandleBeforePosition(HandleReg&: NewHandleReg, ResType: HandleType, HandleDef&: *HandleDef, Pos&: I)) {
1718 return false;
1719 }
1720
1721 Register IdxReg = IntPtrDef->getOperand(i: 3).getReg();
1722 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
1723 auto BMI = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1724 MCID: TII.get(Opcode: SPIRV::OpImageWrite))
1725 .addUse(RegNo: NewHandleReg)
1726 .addUse(RegNo: IdxReg)
1727 .addUse(RegNo: StoreVal);
1728
1729 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(Ty: HandleType);
1730 if (sampledTypeIsSignedInteger(HandleType: LLVMHandleType))
1731 BMI.addImm(Val: 0x1000); // SignExtend
1732
1733 BMI.constrainAllUses(TII, TRI, RBI);
1734 return true;
1735 }
1736 }
1737
1738 MachineBasicBlock &BB = *I.getParent();
1739 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpStore))
1740 .addUse(RegNo: Ptr)
1741 .addUse(RegNo: StoreVal);
1742 if (!I.getNumMemOperands()) {
1743 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1744 I.getOpcode() ==
1745 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1746 addMemoryOperands(Flags: I.getOperand(i: 2 + OpOffset).getImm(), MIB);
1747 } else {
1748 MachineIRBuilder MIRBuilder(I);
1749 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1750 }
1751 MIB.constrainAllUses(TII, TRI, RBI);
1752 return true;
1753}
1754
1755bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
1756 SPIRVTypeInst ResType,
1757 MachineInstr &I) const {
1758 assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
1759 // Operand indices:
1760 // 0: result (def)
1761 // 1: intrinsic ID
1762 // 2: vector of pointers
1763 // 3: alignment (i32 immediate)
1764 // 4: mask (vector of i1)
1765 // 5: passthru/fill value
1766 const Register PtrsReg = I.getOperand(i: 2).getReg();
1767 const uint32_t Alignment = I.getOperand(i: 3).getImm();
1768 const Register MaskReg = I.getOperand(i: 4).getReg();
1769 const Register PassthruReg = I.getOperand(i: 5).getReg();
1770 const Register AlignmentReg = buildI32Constant(Val: Alignment, I);
1771
1772 MachineBasicBlock &BB = *I.getParent();
1773 auto MIB =
1774 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMaskedGatherINTEL))
1775 .addDef(RegNo: ResVReg)
1776 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1777 .addUse(RegNo: PtrsReg)
1778 .addUse(RegNo: AlignmentReg)
1779 .addUse(RegNo: MaskReg)
1780 .addUse(RegNo: PassthruReg);
1781 MIB.constrainAllUses(TII, TRI, RBI);
1782 return true;
1783}
1784
1785bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
1786 assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
1787 // Operand indices (no explicit defs):
1788 // 0: intrinsic ID
1789 // 1: value vector
1790 // 2: vector of pointers
1791 // 3: alignment (i32 immediate)
1792 // 4: mask (vector of i1)
1793 const Register ValuesReg = I.getOperand(i: 1).getReg();
1794 const Register PtrsReg = I.getOperand(i: 2).getReg();
1795 const uint32_t Alignment = I.getOperand(i: 3).getImm();
1796 const Register MaskReg = I.getOperand(i: 4).getReg();
1797 const Register AlignmentReg = buildI32Constant(Val: Alignment, I);
1798 MachineBasicBlock &BB = *I.getParent();
1799
1800 auto MIB =
1801 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMaskedScatterINTEL))
1802 .addUse(RegNo: PtrsReg)
1803 .addUse(RegNo: AlignmentReg)
1804 .addUse(RegNo: MaskReg)
1805 .addUse(RegNo: ValuesReg);
1806 MIB.constrainAllUses(TII, TRI, RBI);
1807 return true;
1808}
1809
1810bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I,
1811 const Twine &Msg) const {
1812 const Function &F = I.getMF()->getFunction();
1813 F.getContext().diagnose(
1814 DI: DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error));
1815 return false;
1816}
1817
1818bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
1819 SPIRVTypeInst ResType,
1820 MachineInstr &I) const {
1821 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1822 report_fatal_error(
1823 reason: "llvm.stacksave intrinsic: this instruction requires the following "
1824 "SPIR-V extension: SPV_INTEL_variable_length_array",
1825 gen_crash_diag: false);
1826 MachineBasicBlock &BB = *I.getParent();
1827 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSaveMemoryINTEL))
1828 .addDef(RegNo: ResVReg)
1829 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1830 .constrainAllUses(TII, TRI, RBI);
1831 return true;
1832}
1833
1834bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
1835 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1836 report_fatal_error(
1837 reason: "llvm.stackrestore intrinsic: this instruction requires the following "
1838 "SPIR-V extension: SPV_INTEL_variable_length_array",
1839 gen_crash_diag: false);
1840 if (!I.getOperand(i: 0).isReg())
1841 return false;
1842 MachineBasicBlock &BB = *I.getParent();
1843 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpRestoreMemoryINTEL))
1844 .addUse(RegNo: I.getOperand(i: 0).getReg())
1845 .constrainAllUses(TII, TRI, RBI);
1846 return true;
1847}
1848
1849Register
1850SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
1851 MachineIRBuilder MIRBuilder(I);
1852 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1853
1854 // TODO: check if we have such GV, add init, use buildGlobalVariable.
1855 unsigned Num = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1856 Function &CurFunction = GR.CurMF->getFunction();
1857 Type *LLVMArrTy =
1858 ArrayType::get(ElementType: IntegerType::get(C&: CurFunction.getContext(), NumBits: 8), NumElements: Num);
1859 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
1860 true, GlobalValue::InternalLinkage,
1861 Constant::getNullValue(Ty: LLVMArrTy));
1862
1863 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1864 Type *ArrTy = ArrayType::get(ElementType: ValTy, NumElements: Num);
1865 SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType(
1866 BaseType: ArrTy, MIRBuilder, SC: SPIRV::StorageClass::UniformConstant);
1867
1868 SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType(
1869 Type: ArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::None, EmitIR: false);
1870
1871 unsigned Val = getIConstVal(ConstReg: I.getOperand(i: 1).getReg(), MRI);
1872 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvType: SpvArrTy, TII);
1873
1874 Register VarReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1875 auto MIBVar =
1876 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
1877 .addDef(RegNo: VarReg)
1878 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VarTy))
1879 .addImm(Val: SPIRV::StorageClass::UniformConstant)
1880 .addUse(RegNo: Const);
1881 MIBVar.constrainAllUses(TII, TRI, RBI);
1882
1883 GR.add(V: GV, MI: MIBVar);
1884 GR.addGlobalObject(V: GV, MF: GR.CurMF, R: VarReg);
1885
1886 buildOpDecorate(Reg: VarReg, I, TII, Dec: SPIRV::Decoration::Constant, DecArgs: {});
1887 return VarReg;
1888}
1889
1890bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
1891 Register SrcReg) const {
1892 MachineBasicBlock &BB = *I.getParent();
1893 Register DstReg = I.getOperand(i: 0).getReg();
1894 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(VReg: DstReg);
1895 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(VReg: SrcReg);
1896 if (GR.getPointeeType(PtrType: DstTy) != GR.getPointeeType(PtrType: SrcTy))
1897 report_fatal_error(reason: "OpCopyMemory requires operands to have the same type");
1898 uint64_t CopySize = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI);
1899 SPIRVTypeInst PointeeTy = GR.getPointeeType(PtrType: DstTy);
1900 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(Ty: PointeeTy);
1901 if (!LLVMPointeeTy)
1902 report_fatal_error(
1903 reason: "Unable to determine pointee type size for OpCopyMemory");
1904 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
1905 if (CopySize != DL.getTypeStoreSize(Ty: const_cast<Type *>(LLVMPointeeTy)))
1906 report_fatal_error(
1907 reason: "OpCopyMemory requires the size to match the pointee type size");
1908 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemory))
1909 .addUse(RegNo: DstReg)
1910 .addUse(RegNo: SrcReg);
1911 if (I.getNumMemOperands()) {
1912 MachineIRBuilder MIRBuilder(I);
1913 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1914 }
1915 MIB.constrainAllUses(TII, TRI, RBI);
1916 return true;
1917}
1918
1919bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
1920 Register SrcReg) const {
1921 MachineBasicBlock &BB = *I.getParent();
1922 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCopyMemorySized))
1923 .addUse(RegNo: I.getOperand(i: 0).getReg())
1924 .addUse(RegNo: SrcReg)
1925 .addUse(RegNo: I.getOperand(i: 2).getReg());
1926 if (I.getNumMemOperands()) {
1927 MachineIRBuilder MIRBuilder(I);
1928 addMemoryOperands(MemOp: *I.memoperands_begin(), MIB, MIRBuilder, GR);
1929 }
1930 MIB.constrainAllUses(TII, TRI, RBI);
1931 return true;
1932}
1933
1934bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
1935 MachineInstr &I) const {
1936 Register SrcReg = I.getOperand(i: 1).getReg();
1937 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
1938 Register VarReg = getOrCreateMemSetGlobal(I);
1939 if (!VarReg.isValid())
1940 return false;
1941 Type *ValTy = Type::getInt8Ty(C&: I.getMF()->getFunction().getContext());
1942 SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType(
1943 BaseType: ValTy, I, SC: SPIRV::StorageClass::UniformConstant);
1944 SrcReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
1945 if (!selectOpWithSrcs(ResVReg: SrcReg, ResType: SourceTy, I, Srcs: {VarReg}, Opcode: SPIRV::OpBitcast))
1946 return false;
1947 }
1948 if (STI.isLogicalSPIRV()) {
1949 if (!selectCopyMemory(I, SrcReg))
1950 return false;
1951 } else {
1952 if (!selectCopyMemorySized(I, SrcReg))
1953 return false;
1954 }
1955 if (ResVReg.isValid() && ResVReg != I.getOperand(i: 0).getReg())
1956 if (!BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 0).getReg(), I))
1957 return false;
1958 return true;
1959}
1960
1961bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
1962 SPIRVTypeInst ResType,
1963 MachineInstr &I,
1964 unsigned NewOpcode,
1965 unsigned NegateOpcode) const {
1966 assert(I.hasOneMemOperand());
1967 const MachineMemOperand *MemOp = *I.memoperands_begin();
1968 uint32_t Scope = static_cast<uint32_t>(getMemScope(
1969 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
1970 Register ScopeReg = buildI32Constant(Val: Scope, I);
1971
1972 Register Ptr = I.getOperand(i: 1).getReg();
1973 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
1974 // auto ScSem =
1975 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
1976 AtomicOrdering AO = MemOp->getSuccessOrdering();
1977 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
1978 Register MemSemReg = buildI32Constant(Val: MemSem /*| ScSem*/, I);
1979
1980 Register ValueReg = I.getOperand(i: 2).getReg();
1981 if (NegateOpcode != 0) {
1982 // Translation with negative value operand is requested
1983 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: MRI->getMF());
1984 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {ValueReg}, Opcode: NegateOpcode))
1985 return false;
1986 ValueReg = TmpReg;
1987 }
1988
1989 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: NewOpcode))
1990 .addDef(RegNo: ResVReg)
1991 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
1992 .addUse(RegNo: Ptr)
1993 .addUse(RegNo: ScopeReg)
1994 .addUse(RegNo: MemSemReg)
1995 .addUse(RegNo: ValueReg)
1996 .constrainAllUses(TII, TRI, RBI);
1997 return true;
1998}
1999
2000bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
2001 unsigned ArgI = I.getNumOperands() - 1;
2002 Register SrcReg =
2003 I.getOperand(i: ArgI).isReg() ? I.getOperand(i: ArgI).getReg() : Register(0);
2004 SPIRVTypeInst SrcType =
2005 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: SrcReg) : nullptr;
2006 if (!SrcType || SrcType->getOpcode() != SPIRV::OpTypeVector)
2007 report_fatal_error(
2008 reason: "cannot select G_UNMERGE_VALUES with a non-vector argument");
2009
2010 SPIRVTypeInst ScalarType =
2011 GR.getSPIRVTypeForVReg(VReg: SrcType->getOperand(i: 1).getReg());
2012 MachineBasicBlock &BB = *I.getParent();
2013 unsigned CurrentIndex = 0;
2014 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2015 Register ResVReg = I.getOperand(i).getReg();
2016 SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(VReg: ResVReg);
2017 if (!ResType) {
2018 LLT ResLLT = MRI->getType(Reg: ResVReg);
2019 assert(ResLLT.isValid());
2020 if (ResLLT.isVector()) {
2021 ResType = GR.getOrCreateSPIRVVectorType(
2022 BaseType: ScalarType, NumElements: ResLLT.getNumElements(), I, TII);
2023 } else {
2024 ResType = ScalarType;
2025 }
2026 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
2027 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *GR.CurMF);
2028 }
2029
2030 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
2031 Register UndefReg = GR.getOrCreateUndef(I, SpvType: SrcType, TII);
2032 auto MIB =
2033 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
2034 .addDef(RegNo: ResVReg)
2035 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2036 .addUse(RegNo: SrcReg)
2037 .addUse(RegNo: UndefReg);
2038 unsigned NumElements = GR.getScalarOrVectorComponentCount(Type: ResType);
2039 for (unsigned j = 0; j < NumElements; ++j) {
2040 MIB.addImm(Val: CurrentIndex + j);
2041 }
2042 CurrentIndex += NumElements;
2043 MIB.constrainAllUses(TII, TRI, RBI);
2044 } else {
2045 auto MIB =
2046 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2047 .addDef(RegNo: ResVReg)
2048 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2049 .addUse(RegNo: SrcReg)
2050 .addImm(Val: CurrentIndex);
2051 CurrentIndex++;
2052 MIB.constrainAllUses(TII, TRI, RBI);
2053 }
2054 }
2055 return true;
2056}
2057
2058bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
2059 AtomicOrdering AO = AtomicOrdering(I.getOperand(i: 0).getImm());
2060 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(Ord: AO));
2061 Register MemSemReg = buildI32Constant(Val: MemSem, I);
2062 SyncScope::ID Ord = SyncScope::ID(I.getOperand(i: 1).getImm());
2063 uint32_t Scope = static_cast<uint32_t>(
2064 getMemScope(Ctx&: GR.CurMF->getFunction().getContext(), Id: Ord));
2065 Register ScopeReg = buildI32Constant(Val: Scope, I);
2066 MachineBasicBlock &BB = *I.getParent();
2067 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMemoryBarrier))
2068 .addUse(RegNo: ScopeReg)
2069 .addUse(RegNo: MemSemReg)
2070 .constrainAllUses(TII, TRI, RBI);
2071 return true;
2072}
2073
2074bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
2075 SPIRVTypeInst ResType,
2076 MachineInstr &I,
2077 unsigned Opcode) const {
2078 Type *ResTy = nullptr;
2079 StringRef ResName;
2080 if (!GR.findValueAttrs(Key: &I, Ty&: ResTy, Name&: ResName))
2081 report_fatal_error(
2082 reason: "Not enough info to select the arithmetic with overflow instruction");
2083 if (!ResTy || !ResTy->isStructTy())
2084 report_fatal_error(reason: "Expect struct type result for the arithmetic "
2085 "with overflow instruction");
2086 // "Result Type must be from OpTypeStruct. The struct must have two members,
2087 // and the two members must be the same type."
2088 Type *ResElemTy = cast<StructType>(Val: ResTy)->getElementType(N: 0);
2089 ResTy = StructType::get(elt1: ResElemTy, elts: ResElemTy);
2090 // Build SPIR-V types and constant(s) if needed.
2091 MachineIRBuilder MIRBuilder(I);
2092 SPIRVTypeInst StructType = GR.getOrCreateSPIRVType(
2093 Type: ResTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
2094 assert(I.getNumDefs() > 1 && "Not enought operands");
2095 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2096 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
2097 if (N > 1)
2098 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
2099 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
2100 Register ZeroReg = buildZerosVal(ResType, I);
2101 // A new virtual register to store the result struct.
2102 Register StructVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2103 MRI->setRegClass(Reg: StructVReg, RC: &SPIRV::IDRegClass);
2104 // Build the result name if needed.
2105 if (ResName.size() > 0)
2106 buildOpName(Target: StructVReg, Name: ResName, MIRBuilder);
2107 // Build the arithmetic with overflow instruction.
2108 MachineBasicBlock &BB = *I.getParent();
2109 auto MIB =
2110 BuildMI(BB, I: MIRBuilder.getInsertPt(), MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2111 .addDef(RegNo: StructVReg)
2112 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: StructType));
2113 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
2114 MIB.addUse(RegNo: I.getOperand(i).getReg());
2115 MIB.constrainAllUses(TII, TRI, RBI);
2116 // Build instructions to extract fields of the instruction's result.
2117 // A new virtual register to store the higher part of the result struct.
2118 Register HigherVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2119 MRI->setRegClass(Reg: HigherVReg, RC: &SPIRV::iIDRegClass);
2120 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2121 auto MIB =
2122 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2123 .addDef(RegNo: i == 1 ? HigherVReg : I.getOperand(i).getReg())
2124 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2125 .addUse(RegNo: StructVReg)
2126 .addImm(Val: i);
2127 MIB.constrainAllUses(TII, TRI, RBI);
2128 }
2129 // Build boolean value from the higher part.
2130 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
2131 .addDef(RegNo: I.getOperand(i: 1).getReg())
2132 .addUse(RegNo: BoolTypeReg)
2133 .addUse(RegNo: HigherVReg)
2134 .addUse(RegNo: ZeroReg)
2135 .constrainAllUses(TII, TRI, RBI);
2136 return true;
2137}
2138
2139bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
2140 SPIRVTypeInst ResType,
2141 MachineInstr &I) const {
2142 Register ScopeReg;
2143 Register MemSemEqReg;
2144 Register MemSemNeqReg;
2145 Register Ptr = I.getOperand(i: 2).getReg();
2146 if (!isa<GIntrinsic>(Val: I)) {
2147 assert(I.hasOneMemOperand());
2148 const MachineMemOperand *MemOp = *I.memoperands_begin();
2149 unsigned Scope = static_cast<uint32_t>(getMemScope(
2150 Ctx&: GR.CurMF->getFunction().getContext(), Id: MemOp->getSyncScopeID()));
2151 ScopeReg = buildI32Constant(Val: Scope, I);
2152
2153 unsigned ScSem = static_cast<uint32_t>(
2154 getMemSemanticsForStorageClass(SC: GR.getPointerStorageClass(VReg: Ptr)));
2155 AtomicOrdering AO = MemOp->getSuccessOrdering();
2156 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(Ord: AO)) | ScSem;
2157 Register MemSemEqReg = buildI32Constant(Val: MemSemEq, I);
2158 AtomicOrdering FO = MemOp->getFailureOrdering();
2159 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(Ord: FO)) | ScSem;
2160 if (MemSemEq == MemSemNeq)
2161 MemSemNeqReg = MemSemEqReg;
2162 else {
2163 MemSemNeqReg = buildI32Constant(Val: MemSemEq, I);
2164 }
2165 } else {
2166 ScopeReg = I.getOperand(i: 5).getReg();
2167 MemSemEqReg = I.getOperand(i: 6).getReg();
2168 MemSemNeqReg = I.getOperand(i: 7).getReg();
2169 }
2170
2171 Register Cmp = I.getOperand(i: 3).getReg();
2172 Register Val = I.getOperand(i: 4).getReg();
2173 SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(VReg: Val);
2174 Register ACmpRes = createVirtualRegister(SpvType: SpvValTy, GR: &GR, MRI, MF: *I.getMF());
2175 const DebugLoc &DL = I.getDebugLoc();
2176 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpAtomicCompareExchange))
2177 .addDef(RegNo: ACmpRes)
2178 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvValTy))
2179 .addUse(RegNo: Ptr)
2180 .addUse(RegNo: ScopeReg)
2181 .addUse(RegNo: MemSemEqReg)
2182 .addUse(RegNo: MemSemNeqReg)
2183 .addUse(RegNo: Val)
2184 .addUse(RegNo: Cmp)
2185 .constrainAllUses(TII, TRI, RBI);
2186 SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2187 Register CmpSuccReg = createVirtualRegister(SpvType: BoolTy, GR: &GR, MRI, MF: *I.getMF());
2188 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpIEqual))
2189 .addDef(RegNo: CmpSuccReg)
2190 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
2191 .addUse(RegNo: ACmpRes)
2192 .addUse(RegNo: Cmp)
2193 .constrainAllUses(TII, TRI, RBI);
2194 Register TmpReg = createVirtualRegister(SpvType: ResType, GR: &GR, MRI, MF: *I.getMF());
2195 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2196 .addDef(RegNo: TmpReg)
2197 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2198 .addUse(RegNo: ACmpRes)
2199 .addUse(RegNo: GR.getOrCreateUndef(I, SpvType: ResType, TII))
2200 .addImm(Val: 0)
2201 .constrainAllUses(TII, TRI, RBI);
2202 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
2203 .addDef(RegNo: ResVReg)
2204 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2205 .addUse(RegNo: CmpSuccReg)
2206 .addUse(RegNo: TmpReg)
2207 .addImm(Val: 1)
2208 .constrainAllUses(TII, TRI, RBI);
2209 return true;
2210}
2211
2212static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2213 switch (SC) {
2214 case SPIRV::StorageClass::DeviceOnlyINTEL:
2215 case SPIRV::StorageClass::HostOnlyINTEL:
2216 return true;
2217 default:
2218 return false;
2219 }
2220}
2221
2222// Returns true ResVReg is referred only from global vars and OpName's.
2223static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
2224 bool IsGRef = false;
2225 bool IsAllowedRefs =
2226 llvm::all_of(Range: MRI->use_instructions(Reg: ResVReg), P: [&IsGRef](auto const &It) {
2227 unsigned Opcode = It.getOpcode();
2228 if (Opcode == SPIRV::OpConstantComposite ||
2229 Opcode == SPIRV::OpVariable ||
2230 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2231 return IsGRef = true;
2232 return Opcode == SPIRV::OpName;
2233 });
2234 return IsAllowedRefs && IsGRef;
2235}
2236
2237Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2238 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2239 return GR.getSPIRVTypeID(SpirvType: GR.getOrCreateSPIRVPointerType(
2240 BaseType: Type::getInt8Ty(C&: I.getMF()->getFunction().getContext()), I, SC));
2241}
2242
2243MachineInstrBuilder
2244SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2245 Register Src, Register DestType,
2246 uint32_t Opcode) const {
2247 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
2248 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
2249 .addDef(RegNo: Dest)
2250 .addUse(RegNo: DestType)
2251 .addImm(Val: Opcode)
2252 .addUse(RegNo: Src);
2253}
2254
2255MachineInstrBuilder
2256SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2257 SPIRVTypeInst SrcPtrTy) const {
2258 SPIRVTypeInst GenericPtrTy =
2259 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2260 Register Tmp = MRI->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
2261 MRI->setType(VReg: Tmp, Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(
2262 SC: SPIRV::StorageClass::Generic),
2263 SizeInBits: GR.getPointerSize()));
2264 MachineFunction *MF = I.getParent()->getParent();
2265 GR.assignSPIRVTypeToVReg(Type: GenericPtrTy, VReg: Tmp, MF: *MF);
2266 MachineInstrBuilder MIB = buildSpecConstantOp(
2267 I, Dest: Tmp, Src: SrcPtr, DestType: GR.getSPIRVTypeID(SpirvType: GenericPtrTy),
2268 Opcode: static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
2269 GR.add(Obj: MIB.getInstr(), MI: MIB);
2270 return MIB;
2271}
2272
2273// In SPIR-V address space casting can only happen to and from the Generic
2274// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
2275// pointers to and from Generic pointers. As such, we can convert e.g. from
2276// Workgroup to Function by going via a Generic pointer as an intermediary. All
2277// other combinations can only be done by a bitcast, and are probably not safe.
2278bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
2279 SPIRVTypeInst ResType,
2280 MachineInstr &I) const {
2281 MachineBasicBlock &BB = *I.getParent();
2282 const DebugLoc &DL = I.getDebugLoc();
2283
2284 Register SrcPtr = I.getOperand(i: 1).getReg();
2285 SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(VReg: SrcPtr);
2286
2287 // don't generate a cast for a null that may be represented by OpTypeInt
2288 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
2289 ResType->getOpcode() != SPIRV::OpTypePointer)
2290 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2291
2292 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(Type: SrcPtrTy);
2293 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(Type: ResType);
2294
2295 if (isASCastInGVar(MRI, ResVReg)) {
2296 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
2297 // are expressed by OpSpecConstantOp with an Opcode.
2298 // TODO: maybe insert a check whether the Kernel capability was declared and
2299 // so PtrCastToGeneric/GenericCastToPtr are available.
2300 unsigned SpecOpcode =
2301 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC)
2302 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
2303 : (SrcSC == SPIRV::StorageClass::Generic &&
2304 isGenericCastablePtr(SC: DstSC)
2305 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
2306 : 0);
2307 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
2308 // correct value of ResType and use general i8* instead. Maybe this should
2309 // be addressed in the emit-intrinsic step to infer a correct
2310 // OpConstantComposite type.
2311 if (SpecOpcode) {
2312 buildSpecConstantOp(I, Dest: ResVReg, Src: SrcPtr, DestType: getUcharPtrTypeReg(I, SC: DstSC),
2313 Opcode: SpecOpcode)
2314 .constrainAllUses(TII, TRI, RBI);
2315 } else if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2316 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
2317 MIB.constrainAllUses(TII, TRI, RBI);
2318 buildSpecConstantOp(
2319 I, Dest: ResVReg, Src: MIB->getOperand(i: 0).getReg(), DestType: getUcharPtrTypeReg(I, SC: DstSC),
2320 Opcode: static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
2321 .constrainAllUses(TII, TRI, RBI);
2322 }
2323 return true;
2324 }
2325
2326 // don't generate a cast between identical storage classes
2327 if (SrcSC == DstSC)
2328 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2329
2330 if ((SrcSC == SPIRV::StorageClass::Function &&
2331 DstSC == SPIRV::StorageClass::Private) ||
2332 (DstSC == SPIRV::StorageClass::Function &&
2333 SrcSC == SPIRV::StorageClass::Private))
2334 return BuildCOPY(DestReg: ResVReg, SrcReg: SrcPtr, I);
2335
2336 // Casting from an eligible pointer to Generic.
2337 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: SrcSC))
2338 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2339 // Casting from Generic to an eligible pointer.
2340 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SC: DstSC))
2341 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2342 // Casting between 2 eligible pointers using Generic as an intermediary.
2343 if (isGenericCastablePtr(SC: SrcSC) && isGenericCastablePtr(SC: DstSC)) {
2344 SPIRVTypeInst GenericPtrTy =
2345 GR.changePointerStorageClass(PtrType: SrcPtrTy, SC: SPIRV::StorageClass::Generic, I);
2346 Register Tmp = createVirtualRegister(SpvType: GenericPtrTy, GR: &GR, MRI, MF: MRI->getMF());
2347 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpPtrCastToGeneric))
2348 .addDef(RegNo: Tmp)
2349 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: GenericPtrTy))
2350 .addUse(RegNo: SrcPtr)
2351 .constrainAllUses(TII, TRI, RBI);
2352 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtr))
2353 .addDef(RegNo: ResVReg)
2354 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2355 .addUse(RegNo: Tmp)
2356 .constrainAllUses(TII, TRI, RBI);
2357 return true;
2358 }
2359
2360 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
2361 // be applied
2362 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
2363 return selectUnOp(ResVReg, ResType, I,
2364 Opcode: SPIRV::OpPtrCastToCrossWorkgroupINTEL);
2365 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(SC: DstSC))
2366 return selectUnOp(ResVReg, ResType, I,
2367 Opcode: SPIRV::OpCrossWorkgroupCastToPtrINTEL);
2368 if (isUSMStorageClass(SC: SrcSC) && DstSC == SPIRV::StorageClass::Generic)
2369 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpPtrCastToGeneric);
2370 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(SC: DstSC))
2371 return selectUnOp(ResVReg, ResType, I, Opcode: SPIRV::OpGenericCastToPtr);
2372
2373 // Bitcast for pointers requires that the address spaces must match
2374 return false;
2375}
2376
2377static unsigned getFCmpOpcode(unsigned PredNum) {
2378 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2379 switch (Pred) {
2380 case CmpInst::FCMP_OEQ:
2381 return SPIRV::OpFOrdEqual;
2382 case CmpInst::FCMP_OGE:
2383 return SPIRV::OpFOrdGreaterThanEqual;
2384 case CmpInst::FCMP_OGT:
2385 return SPIRV::OpFOrdGreaterThan;
2386 case CmpInst::FCMP_OLE:
2387 return SPIRV::OpFOrdLessThanEqual;
2388 case CmpInst::FCMP_OLT:
2389 return SPIRV::OpFOrdLessThan;
2390 case CmpInst::FCMP_ONE:
2391 return SPIRV::OpFOrdNotEqual;
2392 case CmpInst::FCMP_ORD:
2393 return SPIRV::OpOrdered;
2394 case CmpInst::FCMP_UEQ:
2395 return SPIRV::OpFUnordEqual;
2396 case CmpInst::FCMP_UGE:
2397 return SPIRV::OpFUnordGreaterThanEqual;
2398 case CmpInst::FCMP_UGT:
2399 return SPIRV::OpFUnordGreaterThan;
2400 case CmpInst::FCMP_ULE:
2401 return SPIRV::OpFUnordLessThanEqual;
2402 case CmpInst::FCMP_ULT:
2403 return SPIRV::OpFUnordLessThan;
2404 case CmpInst::FCMP_UNE:
2405 return SPIRV::OpFUnordNotEqual;
2406 case CmpInst::FCMP_UNO:
2407 return SPIRV::OpUnordered;
2408 default:
2409 llvm_unreachable("Unknown predicate type for FCmp");
2410 }
2411}
2412
2413static unsigned getICmpOpcode(unsigned PredNum) {
2414 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2415 switch (Pred) {
2416 case CmpInst::ICMP_EQ:
2417 return SPIRV::OpIEqual;
2418 case CmpInst::ICMP_NE:
2419 return SPIRV::OpINotEqual;
2420 case CmpInst::ICMP_SGE:
2421 return SPIRV::OpSGreaterThanEqual;
2422 case CmpInst::ICMP_SGT:
2423 return SPIRV::OpSGreaterThan;
2424 case CmpInst::ICMP_SLE:
2425 return SPIRV::OpSLessThanEqual;
2426 case CmpInst::ICMP_SLT:
2427 return SPIRV::OpSLessThan;
2428 case CmpInst::ICMP_UGE:
2429 return SPIRV::OpUGreaterThanEqual;
2430 case CmpInst::ICMP_UGT:
2431 return SPIRV::OpUGreaterThan;
2432 case CmpInst::ICMP_ULE:
2433 return SPIRV::OpULessThanEqual;
2434 case CmpInst::ICMP_ULT:
2435 return SPIRV::OpULessThan;
2436 default:
2437 llvm_unreachable("Unknown predicate type for ICmp");
2438 }
2439}
2440
2441static unsigned getPtrCmpOpcode(unsigned Pred) {
2442 switch (static_cast<CmpInst::Predicate>(Pred)) {
2443 case CmpInst::ICMP_EQ:
2444 return SPIRV::OpPtrEqual;
2445 case CmpInst::ICMP_NE:
2446 return SPIRV::OpPtrNotEqual;
2447 default:
2448 llvm_unreachable("Unknown predicate type for pointer comparison");
2449 }
2450}
2451
2452// Return the logical operation, or abort if none exists.
2453static unsigned getBoolCmpOpcode(unsigned PredNum) {
2454 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
2455 switch (Pred) {
2456 case CmpInst::ICMP_EQ:
2457 return SPIRV::OpLogicalEqual;
2458 case CmpInst::ICMP_NE:
2459 return SPIRV::OpLogicalNotEqual;
2460 default:
2461 llvm_unreachable("Unknown predicate type for Bool comparison");
2462 }
2463}
2464
2465static APFloat getZeroFP(const Type *LLVMFloatTy) {
2466 if (!LLVMFloatTy)
2467 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2468 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2469 case Type::HalfTyID:
2470 return APFloat::getZero(Sem: APFloat::IEEEhalf());
2471 default:
2472 case Type::FloatTyID:
2473 return APFloat::getZero(Sem: APFloat::IEEEsingle());
2474 case Type::DoubleTyID:
2475 return APFloat::getZero(Sem: APFloat::IEEEdouble());
2476 }
2477}
2478
2479static APFloat getOneFP(const Type *LLVMFloatTy) {
2480 if (!LLVMFloatTy)
2481 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2482 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
2483 case Type::HalfTyID:
2484 return APFloat::getOne(Sem: APFloat::IEEEhalf());
2485 default:
2486 case Type::FloatTyID:
2487 return APFloat::getOne(Sem: APFloat::IEEEsingle());
2488 case Type::DoubleTyID:
2489 return APFloat::getOne(Sem: APFloat::IEEEdouble());
2490 }
2491}
2492
2493bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
2494 SPIRVTypeInst ResType,
2495 MachineInstr &I,
2496 unsigned OpAnyOrAll) const {
2497 assert(I.getNumOperands() == 3);
2498 assert(I.getOperand(2).isReg());
2499 MachineBasicBlock &BB = *I.getParent();
2500 Register InputRegister = I.getOperand(i: 2).getReg();
2501 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2502
2503 assert(InputType && "VReg has no type assigned");
2504
2505 bool IsBoolTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeBool);
2506 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
2507 if (IsBoolTy && !IsVectorTy) {
2508 assert(ResVReg == I.getOperand(0).getReg());
2509 return BuildCOPY(DestReg: ResVReg, SrcReg: InputRegister, I);
2510 }
2511
2512 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2513 unsigned SpirvNotEqualId =
2514 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
2515 SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
2516 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
2517 Register NotEqualReg = ResVReg;
2518
2519 if (IsVectorTy) {
2520 NotEqualReg =
2521 IsBoolTy ? InputRegister
2522 : createVirtualRegister(SpvType: SpvBoolTy, GR: &GR, MRI, MF: MRI->getMF());
2523 const unsigned NumElts = InputType->getOperand(i: 2).getImm();
2524 SpvBoolTy = GR.getOrCreateSPIRVVectorType(BaseType: SpvBoolTy, NumElements: NumElts, I, TII);
2525 }
2526
2527 if (!IsBoolTy) {
2528 Register ConstZeroReg =
2529 IsFloatTy ? buildZerosValF(ResType: InputType, I) : buildZerosVal(ResType: InputType, I);
2530
2531 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SpirvNotEqualId))
2532 .addDef(RegNo: NotEqualReg)
2533 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolTy))
2534 .addUse(RegNo: InputRegister)
2535 .addUse(RegNo: ConstZeroReg)
2536 .constrainAllUses(TII, TRI, RBI);
2537 }
2538
2539 if (IsVectorTy)
2540 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpAnyOrAll))
2541 .addDef(RegNo: ResVReg)
2542 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvBoolScalarTy))
2543 .addUse(RegNo: NotEqualReg)
2544 .constrainAllUses(TII, TRI, RBI);
2545 return true;
2546}
2547
2548bool SPIRVInstructionSelector::selectAll(Register ResVReg,
2549 SPIRVTypeInst ResType,
2550 MachineInstr &I) const {
2551 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAll);
2552}
2553
2554bool SPIRVInstructionSelector::selectAny(Register ResVReg,
2555 SPIRVTypeInst ResType,
2556 MachineInstr &I) const {
2557 return selectAnyOrAll(ResVReg, ResType, I, OpAnyOrAll: SPIRV::OpAny);
2558}
2559
2560// Select the OpDot instruction for the given float dot
2561bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
2562 SPIRVTypeInst ResType,
2563 MachineInstr &I) const {
2564 assert(I.getNumOperands() == 4);
2565 assert(I.getOperand(2).isReg());
2566 assert(I.getOperand(3).isReg());
2567
2568 [[maybe_unused]] SPIRVTypeInst VecType =
2569 GR.getSPIRVTypeForVReg(VReg: I.getOperand(i: 2).getReg());
2570
2571 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2572 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2573 "dot product requires a vector of at least 2 components");
2574
2575 [[maybe_unused]] SPIRVTypeInst EltType =
2576 GR.getSPIRVTypeForVReg(VReg: VecType->getOperand(i: 1).getReg());
2577
2578 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
2579
2580 MachineBasicBlock &BB = *I.getParent();
2581 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDot))
2582 .addDef(RegNo: ResVReg)
2583 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2584 .addUse(RegNo: I.getOperand(i: 2).getReg())
2585 .addUse(RegNo: I.getOperand(i: 3).getReg())
2586 .constrainAllUses(TII, TRI, RBI);
2587 return true;
2588}
2589
2590bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
2591 SPIRVTypeInst ResType,
2592 MachineInstr &I,
2593 bool Signed) const {
2594 assert(I.getNumOperands() == 4);
2595 assert(I.getOperand(2).isReg());
2596 assert(I.getOperand(3).isReg());
2597 MachineBasicBlock &BB = *I.getParent();
2598
2599 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2600 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2601 .addDef(RegNo: ResVReg)
2602 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2603 .addUse(RegNo: I.getOperand(i: 2).getReg())
2604 .addUse(RegNo: I.getOperand(i: 3).getReg())
2605 .constrainAllUses(TII, TRI, RBI);
2606 return true;
2607}
2608
2609// Since pre-1.6 SPIRV has no integer dot implementation,
2610// expand by piecewise multiplying and adding the results
2611bool SPIRVInstructionSelector::selectIntegerDotExpansion(
2612 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2613 assert(I.getNumOperands() == 4);
2614 assert(I.getOperand(2).isReg());
2615 assert(I.getOperand(3).isReg());
2616 MachineBasicBlock &BB = *I.getParent();
2617
2618 // Multiply the vectors, then sum the results
2619 Register Vec0 = I.getOperand(i: 2).getReg();
2620 Register Vec1 = I.getOperand(i: 3).getReg();
2621 Register TmpVec = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2622 SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(VReg: Vec0);
2623
2624 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulV))
2625 .addDef(RegNo: TmpVec)
2626 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: VecType))
2627 .addUse(RegNo: Vec0)
2628 .addUse(RegNo: Vec1)
2629 .constrainAllUses(TII, TRI, RBI);
2630
2631 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
2632 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
2633 "dot product requires a vector of at least 2 components");
2634
2635 Register Res = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2636 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2637 .addDef(RegNo: Res)
2638 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2639 .addUse(RegNo: TmpVec)
2640 .addImm(Val: 0)
2641 .constrainAllUses(TII, TRI, RBI);
2642
2643 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(Type: VecType); i++) {
2644 Register Elt = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2645
2646 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2647 .addDef(RegNo: Elt)
2648 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2649 .addUse(RegNo: TmpVec)
2650 .addImm(Val: i)
2651 .constrainAllUses(TII, TRI, RBI);
2652
2653 Register Sum = i < GR.getScalarOrVectorComponentCount(Type: VecType) - 1
2654 ? MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType))
2655 : ResVReg;
2656
2657 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2658 .addDef(RegNo: Sum)
2659 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2660 .addUse(RegNo: Res)
2661 .addUse(RegNo: Elt)
2662 .constrainAllUses(TII, TRI, RBI);
2663 Res = Sum;
2664 }
2665
2666 return true;
2667}
2668
2669bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
2670 SPIRVTypeInst ResType,
2671 MachineInstr &I) const {
2672 MachineBasicBlock &BB = *I.getParent();
2673 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsInf))
2674 .addDef(RegNo: ResVReg)
2675 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2676 .addUse(RegNo: I.getOperand(i: 2).getReg())
2677 .constrainAllUses(TII, TRI, RBI);
2678 return true;
2679}
2680
2681bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
2682 SPIRVTypeInst ResType,
2683 MachineInstr &I) const {
2684 MachineBasicBlock &BB = *I.getParent();
2685 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIsNan))
2686 .addDef(RegNo: ResVReg)
2687 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2688 .addUse(RegNo: I.getOperand(i: 2).getReg())
2689 .constrainAllUses(TII, TRI, RBI);
2690 return true;
2691}
2692
2693template <bool Signed>
2694bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
2695 SPIRVTypeInst ResType,
2696 MachineInstr &I) const {
2697 assert(I.getNumOperands() == 5);
2698 assert(I.getOperand(2).isReg());
2699 assert(I.getOperand(3).isReg());
2700 assert(I.getOperand(4).isReg());
2701 MachineBasicBlock &BB = *I.getParent();
2702
2703 Register Acc = I.getOperand(i: 2).getReg();
2704 Register X = I.getOperand(i: 3).getReg();
2705 Register Y = I.getOperand(i: 4).getReg();
2706
2707 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
2708 Register Dot = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
2709 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DotOp))
2710 .addDef(RegNo: Dot)
2711 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2712 .addUse(RegNo: X)
2713 .addUse(RegNo: Y);
2714 MIB.addImm(Val: SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
2715 MIB.constrainAllUses(TII, TRI, RBI);
2716
2717 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2718 .addDef(RegNo: ResVReg)
2719 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2720 .addUse(RegNo: Dot)
2721 .addUse(RegNo: Acc)
2722 .constrainAllUses(TII, TRI, RBI);
2723 return true;
2724}
2725
2726// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
2727// extract the elements of the packed inputs, multiply them and add the result
2728// to the accumulator.
2729template <bool Signed>
2730bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
2731 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2732 assert(I.getNumOperands() == 5);
2733 assert(I.getOperand(2).isReg());
2734 assert(I.getOperand(3).isReg());
2735 assert(I.getOperand(4).isReg());
2736 MachineBasicBlock &BB = *I.getParent();
2737
2738 Register Acc = I.getOperand(i: 2).getReg();
2739 Register X = I.getOperand(i: 3).getReg();
2740 Register Y = I.getOperand(i: 4).getReg();
2741
2742 SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII);
2743 auto ExtractOp =
2744 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
2745
2746 bool ZeroAsNull = !STI.isShader();
2747 // Extract the i8 element, multiply and add it to the accumulator
2748 for (unsigned i = 0; i < 4; i++) {
2749 // A[i]
2750 Register AElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2751 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2752 .addDef(RegNo: AElt)
2753 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2754 .addUse(RegNo: X)
2755 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2756 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2757 .constrainAllUses(TII, TRI, RBI);
2758
2759 // B[i]
2760 Register BElt = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2761 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2762 .addDef(RegNo: BElt)
2763 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2764 .addUse(RegNo: Y)
2765 .addUse(RegNo: GR.getOrCreateConstInt(Val: i * 8, I, SpvType: EltType, TII, ZeroAsNull))
2766 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2767 .constrainAllUses(TII, TRI, RBI);
2768
2769 // A[i] * B[i]
2770 Register Mul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2771 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIMulS))
2772 .addDef(RegNo: Mul)
2773 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2774 .addUse(RegNo: AElt)
2775 .addUse(RegNo: BElt)
2776 .constrainAllUses(TII, TRI, RBI);
2777
2778 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
2779 Register MaskMul = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2780 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: ExtractOp))
2781 .addDef(RegNo: MaskMul)
2782 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2783 .addUse(RegNo: Mul)
2784 .addUse(RegNo: GR.getOrCreateConstInt(Val: 0, I, SpvType: EltType, TII, ZeroAsNull))
2785 .addUse(RegNo: GR.getOrCreateConstInt(Val: 8, I, SpvType: EltType, TII, ZeroAsNull))
2786 .constrainAllUses(TII, TRI, RBI);
2787
2788 // Acc = Acc + A[i] * B[i]
2789 Register Sum =
2790 i < 3 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass) : ResVReg;
2791 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
2792 .addDef(RegNo: Sum)
2793 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2794 .addUse(RegNo: Acc)
2795 .addUse(RegNo: MaskMul)
2796 .constrainAllUses(TII, TRI, RBI);
2797
2798 Acc = Sum;
2799 }
2800
2801 return true;
2802}
2803
2804/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
2805/// does not have a saturate builtin.
2806bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
2807 SPIRVTypeInst ResType,
2808 MachineInstr &I) const {
2809 assert(I.getNumOperands() == 3);
2810 assert(I.getOperand(2).isReg());
2811 MachineBasicBlock &BB = *I.getParent();
2812 Register VZero = buildZerosValF(ResType, I);
2813 Register VOne = buildOnesValF(ResType, I);
2814
2815 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
2816 .addDef(RegNo: ResVReg)
2817 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2818 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2819 .addImm(Val: GL::FClamp)
2820 .addUse(RegNo: I.getOperand(i: 2).getReg())
2821 .addUse(RegNo: VZero)
2822 .addUse(RegNo: VOne)
2823 .constrainAllUses(TII, TRI, RBI);
2824 return true;
2825}
2826
2827bool SPIRVInstructionSelector::selectSign(Register ResVReg,
2828 SPIRVTypeInst ResType,
2829 MachineInstr &I) const {
2830 assert(I.getNumOperands() == 3);
2831 assert(I.getOperand(2).isReg());
2832 MachineBasicBlock &BB = *I.getParent();
2833 Register InputRegister = I.getOperand(i: 2).getReg();
2834 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
2835 auto &DL = I.getDebugLoc();
2836
2837 if (!InputType)
2838 report_fatal_error(reason: "Input Type could not be determined.");
2839
2840 bool IsFloatTy = GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
2841
2842 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(Type: InputType);
2843 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
2844
2845 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
2846
2847 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
2848 Register SignReg = NeedsConversion
2849 ? MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass)
2850 : ResVReg;
2851
2852 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpExtInst))
2853 .addDef(RegNo: SignReg)
2854 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: InputType))
2855 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2856 .addImm(Val: SignOpcode)
2857 .addUse(RegNo: InputRegister)
2858 .constrainAllUses(TII, TRI, RBI);
2859
2860 if (NeedsConversion) {
2861 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
2862 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: ConvertOpcode))
2863 .addDef(RegNo: ResVReg)
2864 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2865 .addUse(RegNo: SignReg)
2866 .constrainAllUses(TII, TRI, RBI);
2867 }
2868
2869 return true;
2870}
2871
2872bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
2873 SPIRVTypeInst ResType,
2874 MachineInstr &I,
2875 unsigned Opcode) const {
2876 MachineBasicBlock &BB = *I.getParent();
2877 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2878
2879 auto BMI = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
2880 .addDef(RegNo: ResVReg)
2881 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2882 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I,
2883 SpvType: IntTy, TII, ZeroAsNull: !STI.isShader()));
2884
2885 for (unsigned J = 2; J < I.getNumOperands(); J++) {
2886 BMI.addUse(RegNo: I.getOperand(i: J).getReg());
2887 }
2888
2889 BMI.constrainAllUses(TII, TRI, RBI);
2890 return true;
2891}
2892
2893bool SPIRVInstructionSelector::selectWaveActiveCountBits(
2894 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
2895
2896 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2897 SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(BaseType: IntTy, NumElements: 4, I, TII);
2898 Register BallotReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BallotType));
2899 if (!selectWaveOpInst(ResVReg: BallotReg, ResType: BallotType, I,
2900 Opcode: SPIRV::OpGroupNonUniformBallot))
2901 return false;
2902
2903 MachineBasicBlock &BB = *I.getParent();
2904 BuildMI(BB, I, MIMD: I.getDebugLoc(),
2905 MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
2906 .addDef(RegNo: ResVReg)
2907 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
2908 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
2909 ZeroAsNull: !STI.isShader()))
2910 .addImm(Val: SPIRV::GroupOperation::Reduce)
2911 .addUse(RegNo: BallotReg)
2912 .constrainAllUses(TII, TRI, RBI);
2913
2914 return true;
2915}
2916
2917unsigned getVectorSizeOrOne(SPIRVTypeInst Type) {
2918
2919 if (Type->getOpcode() != SPIRV::OpTypeVector)
2920 return 1;
2921
2922 // Operand(2) is the vector size
2923 return Type->getOperand(i: 2).getImm();
2924}
2925
2926bool SPIRVInstructionSelector::selectWaveActiveAllEqual(Register ResVReg,
2927 SPIRVTypeInst ResType,
2928 MachineInstr &I) const {
2929 MachineBasicBlock &BB = *I.getParent();
2930 const DebugLoc &DL = I.getDebugLoc();
2931
2932 // Input to the intrinsic
2933 Register InputReg = I.getOperand(i: 2).getReg();
2934 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputReg);
2935
2936 // Determine if input is vector
2937 unsigned NumElems = getVectorSizeOrOne(Type: InputType);
2938 bool IsVector = NumElems > 1;
2939
2940 // Determine element types
2941 SPIRVTypeInst ElemInputType = InputType;
2942 SPIRVTypeInst ElemBoolType = ResType;
2943 if (IsVector) {
2944 ElemInputType = GR.getSPIRVTypeForVReg(VReg: InputType->getOperand(i: 1).getReg());
2945 ElemBoolType = GR.getSPIRVTypeForVReg(VReg: ResType->getOperand(i: 1).getReg());
2946 }
2947
2948 // Subgroup scope constant
2949 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
2950 Register ScopeConst = GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy,
2951 TII, ZeroAsNull: !STI.isShader());
2952
2953 // Scalar case
2954 if (!IsVector) {
2955 return selectWaveOpInst(ResVReg, ResType: ElemBoolType, I,
2956 Opcode: SPIRV::OpGroupNonUniformAllEqual);
2957 }
2958
2959 // Vector case
2960 SmallVector<Register, 4> ElementResults;
2961 ElementResults.reserve(N: NumElems);
2962
2963 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2964 // Extract element
2965 Register ElemInput = InputReg;
2966 Register Extracted =
2967 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ElemInputType));
2968
2969 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
2970 .addDef(RegNo: Extracted)
2971 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ElemInputType))
2972 .addUse(RegNo: InputReg)
2973 .addImm(Val: Idx)
2974 .constrainAllUses(TII, TRI, RBI);
2975
2976 ElemInput = Extracted;
2977
2978 // Emit per-element AllEqual
2979 Register ElemResult =
2980 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ElemBoolType));
2981
2982 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformAllEqual))
2983 .addDef(RegNo: ElemResult)
2984 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ElemBoolType))
2985 .addUse(RegNo: ScopeConst)
2986 .addUse(RegNo: ElemInput)
2987 .constrainAllUses(TII, TRI, RBI);
2988
2989 ElementResults.push_back(Elt: ElemResult);
2990 }
2991
2992 // Reconstruct vector<bool>
2993 auto MIB = BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpCompositeConstruct))
2994 .addDef(RegNo: ResVReg)
2995 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
2996 for (Register R : ElementResults)
2997 MIB.addUse(RegNo: R);
2998
2999 MIB.constrainAllUses(TII, TRI, RBI);
3000
3001 return true;
3002}
3003
3004bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg,
3005 SPIRVTypeInst ResType,
3006 MachineInstr &I) const {
3007
3008 assert(I.getNumOperands() == 3);
3009
3010 auto Op = I.getOperand(i: 2);
3011 assert(Op.isReg());
3012
3013 MachineBasicBlock &BB = *I.getParent();
3014 DebugLoc DL = I.getDebugLoc();
3015
3016 Register InputRegister = Op.getReg();
3017 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
3018
3019 if (!InputType)
3020 report_fatal_error(reason: "Input Type could not be determined.");
3021
3022 if (InputType->getOpcode() != SPIRV::OpTypeBool)
3023 report_fatal_error(reason: "WavePrefixBitCount requires boolean input");
3024
3025 // Types
3026 SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3027
3028 // Ballot result type: vector<uint32>
3029 // Match DXC: %v4uint for Subgroup size
3030 SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(BaseType: Int32Ty, NumElements: 4, I, TII);
3031
3032 // Create a vreg for the ballot result
3033 Register BallotVReg = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
3034
3035 // 1. OpGroupNonUniformBallot
3036 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallot))
3037 .addDef(RegNo: BallotVReg)
3038 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BallotTy))
3039 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
3040 .addUse(RegNo: InputRegister)
3041 .constrainAllUses(TII, TRI, RBI);
3042
3043 // 2. OpGroupNonUniformBallotBitCount
3044 BuildMI(BB, I, MIMD: DL, MCID: TII.get(Opcode: SPIRV::OpGroupNonUniformBallotBitCount))
3045 .addDef(RegNo: ResVReg)
3046 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3047 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: Int32Ty, TII))
3048 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
3049 .addUse(RegNo: BallotVReg)
3050 .constrainAllUses(TII, TRI, RBI);
3051
3052 return true;
3053}
3054
3055bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
3056 SPIRVTypeInst ResType,
3057 MachineInstr &I,
3058 bool IsUnsigned) const {
3059 return selectWaveReduce(
3060 ResVReg, ResType, I, IsUnsigned,
3061 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3062 const bool IsFloatTy =
3063 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3064 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
3065 : SPIRV::OpGroupNonUniformSMax;
3066 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
3067 });
3068}
3069
3070bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
3071 SPIRVTypeInst ResType,
3072 MachineInstr &I,
3073 bool IsUnsigned) const {
3074 return selectWaveReduce(
3075 ResVReg, ResType, I, IsUnsigned,
3076 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3077 const bool IsFloatTy =
3078 GR.isScalarOrVectorOfType(VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3079 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
3080 : SPIRV::OpGroupNonUniformSMin;
3081 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
3082 });
3083}
3084
3085bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
3086 SPIRVTypeInst ResType,
3087 MachineInstr &I) const {
3088 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3089 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3090 bool IsFloatTy = GR.isScalarOrVectorOfType(
3091 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3092 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
3093 : SPIRV::OpGroupNonUniformIAdd;
3094 });
3095}
3096
3097bool SPIRVInstructionSelector::selectWaveReduceProduct(Register ResVReg,
3098 SPIRVTypeInst ResType,
3099 MachineInstr &I) const {
3100 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3101 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3102 bool IsFloatTy = GR.isScalarOrVectorOfType(
3103 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3104 return IsFloatTy ? SPIRV::OpGroupNonUniformFMul
3105 : SPIRV::OpGroupNonUniformIMul;
3106 });
3107}
3108
3109template <typename PickOpcodeFn>
3110bool SPIRVInstructionSelector::selectWaveReduce(
3111 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
3112 PickOpcodeFn &&PickOpcode) const {
3113 assert(I.getNumOperands() == 3);
3114 assert(I.getOperand(2).isReg());
3115 MachineBasicBlock &BB = *I.getParent();
3116 Register InputRegister = I.getOperand(i: 2).getReg();
3117 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
3118
3119 if (!InputType)
3120 report_fatal_error(reason: "Input Type could not be determined.");
3121
3122 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3123 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
3124 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3125 .addDef(RegNo: ResVReg)
3126 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3127 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
3128 ZeroAsNull: !STI.isShader()))
3129 .addImm(Val: SPIRV::GroupOperation::Reduce)
3130 .addUse(RegNo: I.getOperand(i: 2).getReg())
3131 .constrainAllUses(TII, TRI, RBI);
3132 return true;
3133}
3134
3135bool SPIRVInstructionSelector::selectWaveReduceOp(Register ResVReg,
3136 SPIRVTypeInst ResType,
3137 MachineInstr &I,
3138 unsigned Opcode) const {
3139 return selectWaveReduce(
3140 ResVReg, ResType, I, IsUnsigned: false,
3141 PickOpcode: [&](Register InputRegister, bool IsUnsigned) { return Opcode; });
3142}
3143
3144bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
3145 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3146 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3147 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3148 bool IsFloatTy = GR.isScalarOrVectorOfType(
3149 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3150 return IsFloatTy
3151 ? SPIRV::OpGroupNonUniformFAdd
3152 : SPIRV::OpGroupNonUniformIAdd;
3153 });
3154}
3155
3156bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
3157 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3158 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3159 PickOpcode: [&](Register InputRegister, bool IsUnsigned) {
3160 bool IsFloatTy = GR.isScalarOrVectorOfType(
3161 VReg: InputRegister, TypeOpcode: SPIRV::OpTypeFloat);
3162 return IsFloatTy
3163 ? SPIRV::OpGroupNonUniformFMul
3164 : SPIRV::OpGroupNonUniformIMul;
3165 });
3166}
3167
3168template <typename PickOpcodeFn>
3169bool SPIRVInstructionSelector::selectWaveExclusiveScan(
3170 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
3171 PickOpcodeFn &&PickOpcode) const {
3172 assert(I.getNumOperands() == 3);
3173 assert(I.getOperand(2).isReg());
3174 MachineBasicBlock &BB = *I.getParent();
3175 Register InputRegister = I.getOperand(i: 2).getReg();
3176 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister);
3177
3178 if (!InputType)
3179 report_fatal_error(reason: "Input Type could not be determined.");
3180
3181 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3182 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
3183 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3184 .addDef(RegNo: ResVReg)
3185 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3186 .addUse(RegNo: GR.getOrCreateConstInt(Val: SPIRV::Scope::Subgroup, I, SpvType: IntTy, TII,
3187 ZeroAsNull: !STI.isShader()))
3188 .addImm(Val: SPIRV::GroupOperation::ExclusiveScan)
3189 .addUse(RegNo: I.getOperand(i: 2).getReg())
3190 .constrainAllUses(TII, TRI, RBI);
3191 return true;
3192}
3193
3194bool SPIRVInstructionSelector::selectBitreverse16(Register ResVReg,
3195 SPIRVTypeInst ResType,
3196 MachineInstr &I,
3197 Register Op) const {
3198 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3199 Register ShiftConst = GR.getOrCreateConstInt(Val: 16, I, SpvType: Int32Type, TII);
3200 unsigned ShiftOp = SPIRV::OpShiftRightLogicalS;
3201
3202 const unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
3203 const unsigned ExtendOpcode = GR.isScalarOrVectorSigned(Type: ResType)
3204 ? SPIRV::OpSConvert
3205 : SPIRV::OpUConvert;
3206
3207 if (N > 1) {
3208 Int32Type = GR.getOrCreateSPIRVVectorType(BaseType: Int32Type, NumElements: N, I, TII);
3209 ShiftOp = SPIRV::OpShiftRightLogicalV;
3210
3211 // Vector shifts require a composite constant
3212 const Register CompositeReg =
3213 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Int32Type));
3214 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3215 MCID: TII.get(Opcode: SPIRV::OpConstantComposite))
3216 .addDef(RegNo: CompositeReg)
3217 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: Int32Type));
3218 for (unsigned It = 0; It < N; ++It)
3219 MIB.addUse(RegNo: ShiftConst);
3220 MIB.constrainAllUses(TII, TRI, RBI);
3221
3222 ShiftConst = CompositeReg;
3223 }
3224
3225 // Converts the i16 input to i32 (or vector of i32)
3226 Register ExtReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Int32Type));
3227 if (!selectOpWithSrcs(ResVReg: ExtReg, ResType: Int32Type, I, Srcs: {Op}, Opcode: ExtendOpcode))
3228 return false;
3229
3230 // Perform bitreverse on the i32 value
3231 Register BitrevReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Int32Type));
3232 if (!selectBitreverse32(ResVReg: BitrevReg, ResType: Int32Type, I, Op: ExtReg))
3233 return false;
3234
3235 // Shift the bit-reversed value to get the final result.
3236 Register ShiftReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Int32Type));
3237 if (!selectOpWithSrcs(ResVReg: ShiftReg, ResType: Int32Type, I, Srcs: {BitrevReg, ShiftConst},
3238 Opcode: ShiftOp))
3239 return false;
3240
3241 // Finally, convert the result back to i16 (or vector of i16).
3242 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {ShiftReg}, Opcode: ExtendOpcode);
3243}
3244
3245bool SPIRVInstructionSelector::selectBitreverse32(Register ResVReg,
3246 SPIRVTypeInst ResType,
3247 MachineInstr &I,
3248 Register Op) const {
3249 MachineBasicBlock &BB = *I.getParent();
3250 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBitReverse))
3251 .addDef(RegNo: ResVReg)
3252 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3253 .addUse(RegNo: Op)
3254 .constrainAllUses(TII, TRI, RBI);
3255 return true;
3256}
3257
3258bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
3259 SPIRVTypeInst ResType,
3260 MachineInstr &I) const {
3261 Register OpReg = I.getOperand(i: 1).getReg();
3262 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3263 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
3264 case 16:
3265 return selectBitreverse16(ResVReg, ResType, I, Op: OpReg);
3266 default:
3267 return selectBitreverse32(ResVReg, ResType, I, Op: OpReg);
3268 }
3269}
3270
3271bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
3272 SPIRVTypeInst ResType,
3273 MachineInstr &I) const {
3274 // There is no way to implement `freeze` correctly without support on SPIR-V
3275 // standard side, but we may at least address a simple (static) case when
3276 // undef/poison value presence is obvious. The main benefit of even
3277 // incomplete `freeze` support is preventing of translation from crashing due
3278 // to lack of support on legalization and instruction selection steps.
3279 if (!I.getOperand(i: 0).isReg() || !I.getOperand(i: 1).isReg())
3280 return false;
3281 Register OpReg = I.getOperand(i: 1).getReg();
3282 if (MachineInstr *Def = MRI->getVRegDef(Reg: OpReg)) {
3283 if (Def->getOpcode() == TargetOpcode::COPY)
3284 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
3285 Register Reg;
3286 switch (Def->getOpcode()) {
3287 case SPIRV::ASSIGN_TYPE:
3288 if (MachineInstr *AssignToDef =
3289 MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())) {
3290 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
3291 Reg = Def->getOperand(i: 2).getReg();
3292 }
3293 break;
3294 case SPIRV::OpUndef:
3295 Reg = Def->getOperand(i: 1).getReg();
3296 break;
3297 }
3298 unsigned DestOpCode;
3299 if (Reg.isValid()) {
3300 DestOpCode = SPIRV::OpConstantNull;
3301 } else {
3302 DestOpCode = TargetOpcode::COPY;
3303 Reg = OpReg;
3304 }
3305 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DestOpCode))
3306 .addDef(RegNo: I.getOperand(i: 0).getReg())
3307 .addUse(RegNo: Reg)
3308 .constrainAllUses(TII, TRI, RBI);
3309 return true;
3310 }
3311 return false;
3312}
3313
3314bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
3315 SPIRVTypeInst ResType,
3316 MachineInstr &I) const {
3317 unsigned N = 0;
3318 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3319 N = GR.getScalarOrVectorComponentCount(Type: ResType);
3320 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3321 N = getArrayComponentCount(MRI, ResType);
3322 else
3323 report_fatal_error(reason: "Cannot select G_BUILD_VECTOR with a non-vector result");
3324 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
3325 report_fatal_error(reason: "G_BUILD_VECTOR and the result type are inconsistent");
3326
3327 // check if we may construct a constant vector
3328 bool IsConst = true;
3329 for (unsigned i = I.getNumExplicitDefs();
3330 i < I.getNumExplicitOperands() && IsConst; ++i)
3331 if (!isConstReg(MRI, OpReg: I.getOperand(i).getReg()))
3332 IsConst = false;
3333
3334 if (!IsConst && N < 2)
3335 report_fatal_error(
3336 reason: "There must be at least two constituent operands in a vector");
3337
3338 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3339 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3340 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
3341 : SPIRV::OpCompositeConstruct))
3342 .addDef(RegNo: ResVReg)
3343 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3344 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
3345 MIB.addUse(RegNo: I.getOperand(i).getReg());
3346 MIB.constrainAllUses(TII, TRI, RBI);
3347 return true;
3348}
3349
3350bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
3351 SPIRVTypeInst ResType,
3352 MachineInstr &I) const {
3353 unsigned N = 0;
3354 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3355 N = GR.getScalarOrVectorComponentCount(Type: ResType);
3356 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
3357 N = getArrayComponentCount(MRI, ResType);
3358 else
3359 report_fatal_error(reason: "Cannot select G_SPLAT_VECTOR with a non-vector result");
3360
3361 unsigned OpIdx = I.getNumExplicitDefs();
3362 if (!I.getOperand(i: OpIdx).isReg())
3363 report_fatal_error(reason: "Unexpected argument in G_SPLAT_VECTOR");
3364
3365 // check if we may construct a constant vector
3366 Register OpReg = I.getOperand(i: OpIdx).getReg();
3367 bool IsConst = isConstReg(MRI, OpReg);
3368
3369 if (!IsConst && N < 2)
3370 report_fatal_error(
3371 reason: "There must be at least two constituent operands in a vector");
3372
3373 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
3374 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3375 MCID: TII.get(Opcode: IsConst ? SPIRV::OpConstantComposite
3376 : SPIRV::OpCompositeConstruct))
3377 .addDef(RegNo: ResVReg)
3378 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
3379 for (unsigned i = 0; i < N; ++i)
3380 MIB.addUse(RegNo: OpReg);
3381 MIB.constrainAllUses(TII, TRI, RBI);
3382 return true;
3383}
3384
3385bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
3386 SPIRVTypeInst ResType,
3387 MachineInstr &I) const {
3388
3389 unsigned Opcode;
3390
3391 if (STI.canUseExtension(
3392 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
3393 STI.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6))) {
3394 Opcode = SPIRV::OpDemoteToHelperInvocation;
3395 } else {
3396 Opcode = SPIRV::OpKill;
3397 // OpKill must be the last operation of any basic block.
3398 if (MachineInstr *NextI = I.getNextNode()) {
3399 GR.invalidateMachineInstr(MI: NextI);
3400 NextI->eraseFromParent();
3401 }
3402 }
3403
3404 MachineBasicBlock &BB = *I.getParent();
3405 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3406 .constrainAllUses(TII, TRI, RBI);
3407 return true;
3408}
3409
3410bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
3411 SPIRVTypeInst ResType, unsigned CmpOpc,
3412 MachineInstr &I) const {
3413 Register Cmp0 = I.getOperand(i: 2).getReg();
3414 Register Cmp1 = I.getOperand(i: 3).getReg();
3415 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
3416 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
3417 "CMP operands should have the same type");
3418 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CmpOpc))
3419 .addDef(RegNo: ResVReg)
3420 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3421 .addUse(RegNo: Cmp0)
3422 .addUse(RegNo: Cmp1)
3423 .setMIFlags(I.getFlags())
3424 .constrainAllUses(TII, TRI, RBI);
3425 return true;
3426}
3427
3428bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
3429 SPIRVTypeInst ResType,
3430 MachineInstr &I) const {
3431 auto Pred = I.getOperand(i: 1).getPredicate();
3432 unsigned CmpOpc;
3433
3434 Register CmpOperand = I.getOperand(i: 2).getReg();
3435 if (GR.isScalarOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypePointer))
3436 CmpOpc = getPtrCmpOpcode(Pred);
3437 else if (GR.isScalarOrVectorOfType(VReg: CmpOperand, TypeOpcode: SPIRV::OpTypeBool))
3438 CmpOpc = getBoolCmpOpcode(PredNum: Pred);
3439 else
3440 CmpOpc = getICmpOpcode(PredNum: Pred);
3441 return selectCmp(ResVReg, ResType, CmpOpc, I);
3442}
3443
3444Register
3445SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
3446 SPIRVTypeInst ResType) const {
3447 Type *LLVMTy = IntegerType::get(C&: GR.CurMF->getFunction().getContext(), NumBits: 32);
3448 SPIRVTypeInst SpvI32Ty =
3449 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII);
3450 // Find a constant in DT or build a new one.
3451 auto ConstInt = ConstantInt::get(Ty: LLVMTy, V: Val);
3452 Register NewReg = GR.find(V: ConstInt, MF: GR.CurMF);
3453 if (!NewReg.isValid()) {
3454 NewReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
3455 MachineBasicBlock &BB = *I.getParent();
3456 MachineInstr *MI =
3457 Val == 0
3458 ? BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
3459 .addDef(RegNo: NewReg)
3460 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3461 : BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantI))
3462 .addDef(RegNo: NewReg)
3463 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SpvI32Ty))
3464 .addImm(Val: APInt(32, Val).getZExtValue());
3465 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
3466 GR.add(V: ConstInt, MI);
3467 }
3468 return NewReg;
3469}
3470
3471bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
3472 SPIRVTypeInst ResType,
3473 MachineInstr &I) const {
3474 unsigned CmpOp = getFCmpOpcode(PredNum: I.getOperand(i: 1).getPredicate());
3475 return selectCmp(ResVReg, ResType, CmpOpc: CmpOp, I);
3476}
3477
3478bool SPIRVInstructionSelector::selectExp10(Register ResVReg,
3479 SPIRVTypeInst ResType,
3480 MachineInstr &I) const {
3481 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
3482 return selectExtInst(ResVReg, ResType, I, CLInst: CL::exp10);
3483 }
3484
3485 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
3486 /// There is no exp10 in GLSL. Use exp10(x) = exp2(x * log2(10)) instead
3487 /// log2(10) ~= 3.3219280948874l
3488
3489 if (ResType->getOpcode() != SPIRV::OpTypeVector &&
3490 ResType->getOpcode() != SPIRV::OpTypeFloat)
3491 return false;
3492
3493 MachineIRBuilder MIRBuilder(I);
3494
3495 SPIRVTypeInst SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector
3496 ? SPIRVTypeInst(GR.getSPIRVTypeForVReg(
3497 VReg: ResType->getOperand(i: 1).getReg()))
3498 : ResType;
3499
3500 assert(SpirvScalarType->getOperand(1).getImm() == 32 &&
3501 "only float operands supported by GLSL extended math");
3502
3503 Register ConstReg = GR.buildConstantFP(Val: APFloat(3.3219280948874f),
3504 MIRBuilder, SpvType: SpirvScalarType);
3505 Register ArgReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3506 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
3507 ? SPIRV::OpVectorTimesScalar
3508 : SPIRV::OpFMulS;
3509
3510 if (!selectOpWithSrcs(ResVReg: ArgReg, ResType, I,
3511 Srcs: {I.getOperand(i: 1).getReg(), ConstReg}, Opcode))
3512 return false;
3513 if (!selectExtInst(ResVReg, ResType, I,
3514 Insts: {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}}, setMIFlags: false,
3515 useMISrc: false, SrcRegs: {ArgReg}))
3516 return false;
3517
3518 return true;
3519 }
3520
3521 return false;
3522}
3523
3524Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
3525 MachineInstr &I) const {
3526 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3527 bool ZeroAsNull = !STI.isShader();
3528 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3529 return GR.getOrCreateConstVector(Val: 0UL, I, SpvType: ResType, TII, ZeroAsNull);
3530 return GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
3531}
3532
3533bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
3534 Register Reg) const {
3535 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(VReg: Reg);
3536 if (!Type)
3537 return false;
3538 SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type);
3539 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
3540 return false;
3541
3542 auto IsZero = [this](Register Reg) {
3543 MachineInstr *Def = getDefInstrMaybeConstant(ConstReg&: Reg, MRI);
3544 if (!Def)
3545 return false;
3546
3547 if (Def->getOpcode() == SPIRV::OpConstantNull)
3548 return true;
3549
3550 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
3551 Def->getOpcode() == SPIRV::OpConstantI)
3552 return getIConstVal(ConstReg: Reg, MRI) == 0;
3553
3554 return false;
3555 };
3556
3557 if (IsZero(Reg))
3558 return true;
3559
3560 MachineInstr *Def = MRI->getVRegDef(Reg);
3561 if (!Def)
3562 return false;
3563
3564 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
3565 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
3566 cast<GIntrinsic>(Val: Def)->getIntrinsicID() ==
3567 Intrinsic::spv_const_composite)) {
3568 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
3569 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
3570 if (!IsZero(Def->getOperand(i).getReg()))
3571 return false;
3572 }
3573 return true;
3574 }
3575
3576 return false;
3577}
3578
3579Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
3580 MachineInstr &I) const {
3581 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3582 bool ZeroAsNull = !STI.isShader();
3583 APFloat VZero = getZeroFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3584 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3585 return GR.getOrCreateConstVector(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3586 return GR.getOrCreateConstFP(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull);
3587}
3588
3589Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
3590 MachineInstr &I) const {
3591 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
3592 bool ZeroAsNull = !STI.isShader();
3593 APFloat VOne = getOneFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType));
3594 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3595 return GR.getOrCreateConstVector(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3596 return GR.getOrCreateConstFP(Val: VOne, I, SpvType: ResType, TII, ZeroAsNull);
3597}
3598
3599Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
3600 SPIRVTypeInst ResType,
3601 MachineInstr &I) const {
3602 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3603 APInt One =
3604 AllOnes ? APInt::getAllOnes(numBits: BitWidth) : APInt::getOneBitSet(numBits: BitWidth, BitNo: 0);
3605 if (ResType->getOpcode() == SPIRV::OpTypeVector)
3606 return GR.getOrCreateConstVector(Val: One, I, SpvType: ResType, TII);
3607 return GR.getOrCreateConstInt(Val: One, I, SpvType: ResType, TII);
3608}
3609
3610bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
3611 SPIRVTypeInst ResType,
3612 MachineInstr &I) const {
3613 Register SelectFirstArg = I.getOperand(i: 2).getReg();
3614 Register SelectSecondArg = I.getOperand(i: 3).getReg();
3615 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
3616 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
3617
3618 bool IsFloatTy =
3619 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypeFloat);
3620 bool IsPtrTy =
3621 GR.isScalarOrVectorOfType(VReg: SelectFirstArg, TypeOpcode: SPIRV::OpTypePointer);
3622 bool IsVectorTy = GR.getSPIRVTypeForVReg(VReg: SelectFirstArg)->getOpcode() ==
3623 SPIRV::OpTypeVector;
3624
3625 bool IsScalarBool =
3626 GR.isScalarOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool);
3627 unsigned Opcode;
3628 if (IsVectorTy) {
3629 if (IsFloatTy) {
3630 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
3631 } else if (IsPtrTy) {
3632 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
3633 } else {
3634 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
3635 }
3636 } else {
3637 if (IsFloatTy) {
3638 Opcode = IsScalarBool ? SPIRV::OpSelectSFSCond : SPIRV::OpSelectVFVCond;
3639 } else if (IsPtrTy) {
3640 Opcode = IsScalarBool ? SPIRV::OpSelectSPSCond : SPIRV::OpSelectVPVCond;
3641 } else {
3642 Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3643 }
3644 }
3645 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3646 .addDef(RegNo: ResVReg)
3647 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3648 .addUse(RegNo: I.getOperand(i: 1).getReg())
3649 .addUse(RegNo: SelectFirstArg)
3650 .addUse(RegNo: SelectSecondArg)
3651 .constrainAllUses(TII, TRI, RBI);
3652 return true;
3653}
3654
3655// This function is used to extend a bool or a vector of bools into an integer
3656// or vector of integers.
3657bool SPIRVInstructionSelector::selectBoolToInt(Register ResVReg,
3658 SPIRVTypeInst ResType,
3659 Register BooleanVReg,
3660 MachineInstr &InsertAt,
3661 bool IsSigned) const {
3662 // To extend a bool, we need to use OpSelect between constants.
3663 Register ZeroReg = buildZerosVal(ResType, I&: InsertAt);
3664 Register OneReg = buildOnesVal(AllOnes: IsSigned, ResType, I&: InsertAt);
3665 bool IsScalarBool = GR.isScalarOfType(VReg: BooleanVReg, TypeOpcode: SPIRV::OpTypeBool);
3666 unsigned Opcode =
3667 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
3668 BuildMI(BB&: *InsertAt.getParent(), I&: InsertAt, MIMD: InsertAt.getDebugLoc(),
3669 MCID: TII.get(Opcode))
3670 .addDef(RegNo: ResVReg)
3671 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3672 .addUse(RegNo: BooleanVReg)
3673 .addUse(RegNo: OneReg)
3674 .addUse(RegNo: ZeroReg)
3675 .constrainAllUses(TII, TRI, RBI);
3676 return true;
3677}
3678
3679bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
3680 SPIRVTypeInst ResType,
3681 MachineInstr &I, bool IsSigned,
3682 unsigned Opcode) const {
3683 Register SrcReg = I.getOperand(i: 1).getReg();
3684 // We can convert bool value directly to float type without OpConvert*ToF,
3685 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
3686 if (GR.isScalarOrVectorOfType(VReg: I.getOperand(i: 1).getReg(), TypeOpcode: SPIRV::OpTypeBool)) {
3687 unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType);
3688 SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
3689 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
3690 const unsigned NumElts = ResType->getOperand(i: 2).getImm();
3691 TmpType = GR.getOrCreateSPIRVVectorType(BaseType: TmpType, NumElements: NumElts, I, TII);
3692 }
3693 SrcReg = createVirtualRegister(SpvType: TmpType, GR: &GR, MRI, MF: MRI->getMF());
3694 selectBoolToInt(ResVReg: SrcReg, ResType: TmpType, BooleanVReg: I.getOperand(i: 1).getReg(), InsertAt&: I, IsSigned: false);
3695 }
3696 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {SrcReg}, Opcode);
3697}
3698
3699bool SPIRVInstructionSelector::selectExt(Register ResVReg,
3700 SPIRVTypeInst ResType, MachineInstr &I,
3701 bool IsSigned) const {
3702 Register SrcReg = I.getOperand(i: 1).getReg();
3703 if (GR.isScalarOrVectorOfType(VReg: SrcReg, TypeOpcode: SPIRV::OpTypeBool))
3704 return selectBoolToInt(ResVReg, ResType, BooleanVReg: I.getOperand(i: 1).getReg(), InsertAt&: I,
3705 IsSigned);
3706
3707 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
3708 if (ResType == SrcType)
3709 return BuildCOPY(DestReg: ResVReg, SrcReg, I);
3710
3711 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3712 return selectUnOp(ResVReg, ResType, I, Opcode);
3713}
3714
3715bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
3716 SPIRVTypeInst ResType,
3717 MachineInstr &I,
3718 bool IsSigned) const {
3719 MachineIRBuilder MIRBuilder(I);
3720 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3721 MachineBasicBlock &BB = *I.getParent();
3722 // Ensure we have bool.
3723 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
3724 unsigned N = GR.getScalarOrVectorComponentCount(Type: ResType);
3725 if (N > 1)
3726 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: N, I, TII);
3727 Register BoolTypeReg = GR.getSPIRVTypeID(SpirvType: BoolType);
3728 // Build less-than-equal and less-than.
3729 // TODO: replace with one-liner createVirtualRegister() from
3730 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
3731 Register IsLessEqReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3732 MRI->setType(VReg: IsLessEqReg, Ty: LLT::scalar(SizeInBits: 64));
3733 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessEqReg, MF: MIRBuilder.getMF());
3734 BuildMI(BB, I, MIMD: I.getDebugLoc(),
3735 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
3736 .addDef(RegNo: IsLessEqReg)
3737 .addUse(RegNo: BoolTypeReg)
3738 .addUse(RegNo: I.getOperand(i: 1).getReg())
3739 .addUse(RegNo: I.getOperand(i: 2).getReg())
3740 .constrainAllUses(TII, TRI, RBI);
3741 Register IsLessReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3742 MRI->setType(VReg: IsLessReg, Ty: LLT::scalar(SizeInBits: 64));
3743 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: IsLessReg, MF: MIRBuilder.getMF());
3744 BuildMI(BB, I, MIMD: I.getDebugLoc(),
3745 MCID: TII.get(Opcode: IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
3746 .addDef(RegNo: IsLessReg)
3747 .addUse(RegNo: BoolTypeReg)
3748 .addUse(RegNo: I.getOperand(i: 1).getReg())
3749 .addUse(RegNo: I.getOperand(i: 2).getReg())
3750 .constrainAllUses(TII, TRI, RBI);
3751 // Build selects.
3752 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
3753 Register NegOneOrZeroReg =
3754 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
3755 MRI->setType(VReg: NegOneOrZeroReg, Ty: LLT::scalar(SizeInBits: 64));
3756 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: NegOneOrZeroReg, MF: MIRBuilder.getMF());
3757 unsigned SelectOpcode =
3758 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
3759 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3760 .addDef(RegNo: NegOneOrZeroReg)
3761 .addUse(RegNo: ResTypeReg)
3762 .addUse(RegNo: IsLessReg)
3763 .addUse(RegNo: buildOnesVal(AllOnes: true, ResType, I)) // -1
3764 .addUse(RegNo: buildZerosVal(ResType, I))
3765 .constrainAllUses(TII, TRI, RBI);
3766 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SelectOpcode))
3767 .addDef(RegNo: ResVReg)
3768 .addUse(RegNo: ResTypeReg)
3769 .addUse(RegNo: IsLessEqReg)
3770 .addUse(RegNo: NegOneOrZeroReg) // -1 or 0
3771 .addUse(RegNo: buildOnesVal(AllOnes: false, ResType, I))
3772 .constrainAllUses(TII, TRI, RBI);
3773 return true;
3774}
3775
3776bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
3777 Register ResVReg,
3778 MachineInstr &I,
3779 SPIRVTypeInst IntTy,
3780 SPIRVTypeInst BoolTy) const {
3781 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
3782 Register BitIntReg = createVirtualRegister(SpvType: IntTy, GR: &GR, MRI, MF: MRI->getMF());
3783 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
3784 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
3785 Register Zero = buildZerosVal(ResType: IntTy, I);
3786 Register One = buildOnesVal(AllOnes: false, ResType: IntTy, I);
3787 MachineBasicBlock &BB = *I.getParent();
3788 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3789 .addDef(RegNo: BitIntReg)
3790 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntTy))
3791 .addUse(RegNo: IntReg)
3792 .addUse(RegNo: One)
3793 .constrainAllUses(TII, TRI, RBI);
3794 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpINotEqual))
3795 .addDef(RegNo: ResVReg)
3796 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: BoolTy))
3797 .addUse(RegNo: BitIntReg)
3798 .addUse(RegNo: Zero)
3799 .constrainAllUses(TII, TRI, RBI);
3800 return true;
3801}
3802
3803bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
3804 SPIRVTypeInst ResType,
3805 MachineInstr &I) const {
3806 Register IntReg = I.getOperand(i: 1).getReg();
3807 const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(VReg: IntReg);
3808 if (GR.isScalarOrVectorOfType(VReg: ResVReg, TypeOpcode: SPIRV::OpTypeBool))
3809 return selectIntToBool(IntReg, ResVReg, I, IntTy: ArgType, BoolTy: ResType);
3810 if (ArgType == ResType)
3811 return BuildCOPY(DestReg: ResVReg, SrcReg: IntReg, I);
3812 bool IsSigned = GR.isScalarOrVectorSigned(Type: ResType);
3813 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3814 return selectUnOp(ResVReg, ResType, I, Opcode);
3815}
3816
3817bool SPIRVInstructionSelector::selectConst(Register ResVReg,
3818 SPIRVTypeInst ResType,
3819 MachineInstr &I) const {
3820 unsigned Opcode = I.getOpcode();
3821 unsigned TpOpcode = ResType->getOpcode();
3822 Register Reg;
3823 if (TpOpcode == SPIRV::OpTypePointer || TpOpcode == SPIRV::OpTypeEvent) {
3824 assert(Opcode == TargetOpcode::G_CONSTANT &&
3825 I.getOperand(1).getCImm()->isZero());
3826 MachineBasicBlock &DepMBB = I.getMF()->front();
3827 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
3828 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, SpvType: ResType);
3829 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
3830 Reg = GR.getOrCreateConstFP(Val: I.getOperand(i: 1).getFPImm()->getValue(), I,
3831 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3832 } else {
3833 Reg = GR.getOrCreateConstInt(Val: I.getOperand(i: 1).getCImm()->getValue(), I,
3834 SpvType: ResType, TII, ZeroAsNull: !STI.isShader());
3835 }
3836 return Reg == ResVReg ? true : BuildCOPY(DestReg: ResVReg, SrcReg: Reg, I);
3837}
3838
3839bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
3840 SPIRVTypeInst ResType,
3841 MachineInstr &I) const {
3842 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
3843 .addDef(RegNo: ResVReg)
3844 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3845 .constrainAllUses(TII, TRI, RBI);
3846 return true;
3847}
3848
3849bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
3850 SPIRVTypeInst ResType,
3851 MachineInstr &I) const {
3852 MachineBasicBlock &BB = *I.getParent();
3853 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeInsert))
3854 .addDef(RegNo: ResVReg)
3855 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3856 // object to insert
3857 .addUse(RegNo: I.getOperand(i: 3).getReg())
3858 // composite to insert into
3859 .addUse(RegNo: I.getOperand(i: 2).getReg());
3860 for (unsigned i = 4; i < I.getNumOperands(); i++)
3861 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3862 MIB.constrainAllUses(TII, TRI, RBI);
3863 return true;
3864}
3865
3866bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
3867 SPIRVTypeInst ResType,
3868 MachineInstr &I) const {
3869 Type *MaybeResTy = nullptr;
3870 StringRef ResName;
3871 if (GR.findValueAttrs(Key: &I, Ty&: MaybeResTy, Name&: ResName) &&
3872 MaybeResTy != GR.getTypeForSPIRVType(Ty: ResType)) {
3873 assert((!MaybeResTy || MaybeResTy->isAggregateType()) &&
3874 "Expected aggregate type for extractv instruction");
3875 ResType = GR.getOrCreateSPIRVType(Type: MaybeResTy, I,
3876 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
3877 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *I.getMF());
3878 }
3879 MachineBasicBlock &BB = *I.getParent();
3880 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
3881 .addDef(RegNo: ResVReg)
3882 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3883 .addUse(RegNo: I.getOperand(i: 2).getReg());
3884 for (unsigned i = 3; i < I.getNumOperands(); i++)
3885 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
3886 MIB.constrainAllUses(TII, TRI, RBI);
3887 return true;
3888}
3889
3890bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
3891 SPIRVTypeInst ResType,
3892 MachineInstr &I) const {
3893 if (getImm(MO: I.getOperand(i: 4), MRI))
3894 return selectInsertVal(ResVReg, ResType, I);
3895 MachineBasicBlock &BB = *I.getParent();
3896 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorInsertDynamic))
3897 .addDef(RegNo: ResVReg)
3898 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3899 .addUse(RegNo: I.getOperand(i: 2).getReg())
3900 .addUse(RegNo: I.getOperand(i: 3).getReg())
3901 .addUse(RegNo: I.getOperand(i: 4).getReg())
3902 .constrainAllUses(TII, TRI, RBI);
3903 return true;
3904}
3905
3906bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
3907 SPIRVTypeInst ResType,
3908 MachineInstr &I) const {
3909 if (getImm(MO: I.getOperand(i: 3), MRI))
3910 return selectExtractVal(ResVReg, ResType, I);
3911 MachineBasicBlock &BB = *I.getParent();
3912 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVectorExtractDynamic))
3913 .addDef(RegNo: ResVReg)
3914 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3915 .addUse(RegNo: I.getOperand(i: 2).getReg())
3916 .addUse(RegNo: I.getOperand(i: 3).getReg())
3917 .constrainAllUses(TII, TRI, RBI);
3918 return true;
3919}
3920
3921bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
3922 SPIRVTypeInst ResType,
3923 MachineInstr &I) const {
3924 const bool IsGEPInBounds = I.getOperand(i: 2).getImm();
3925
3926 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
3927 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
3928 // we have to use Op[InBounds]AccessChain.
3929 const unsigned Opcode = STI.isLogicalSPIRV()
3930 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
3931 : SPIRV::OpAccessChain)
3932 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
3933 : SPIRV::OpPtrAccessChain);
3934
3935 auto Res = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
3936 .addDef(RegNo: ResVReg)
3937 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
3938 // Object to get a pointer to.
3939 .addUse(RegNo: I.getOperand(i: 3).getReg());
3940 assert(
3941 (Opcode == SPIRV::OpPtrAccessChain ||
3942 Opcode == SPIRV::OpInBoundsPtrAccessChain ||
3943 (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0)) &&
3944 "Cannot translate GEP to OpAccessChain. First index must be 0.");
3945
3946 // Adding indices.
3947 const unsigned StartingIndex =
3948 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
3949 ? 5
3950 : 4;
3951 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
3952 Res.addUse(RegNo: I.getOperand(i).getReg());
3953 Res.constrainAllUses(TII, TRI, RBI);
3954 return true;
3955}
3956
3957// Maybe wrap a value into OpSpecConstantOp
3958bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
3959 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
3960 unsigned Lim = I.getNumExplicitOperands();
3961 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
3962 Register OpReg = I.getOperand(i).getReg();
3963 MachineInstr *OpDefine = MRI->getVRegDef(Reg: OpReg);
3964 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
3965 if (!OpDefine || !OpType || isConstReg(MRI, OpDef: OpDefine) ||
3966 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
3967 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
3968 GR.isAggregateType(Type: OpType)) {
3969 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
3970 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
3971 CompositeArgs.push_back(Elt: OpReg);
3972 continue;
3973 }
3974 MachineFunction *MF = I.getMF();
3975 Register WrapReg = GR.find(MI: OpDefine, MF);
3976 if (WrapReg.isValid()) {
3977 CompositeArgs.push_back(Elt: WrapReg);
3978 continue;
3979 }
3980 // Create a new register for the wrapper
3981 WrapReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: OpType));
3982 CompositeArgs.push_back(Elt: WrapReg);
3983 // Decorate the wrapper register and generate a new instruction
3984 MRI->setType(VReg: WrapReg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3985 GR.assignSPIRVTypeToVReg(Type: OpType, VReg: WrapReg, MF: *MF);
3986 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3987 MCID: TII.get(Opcode: SPIRV::OpSpecConstantOp))
3988 .addDef(RegNo: WrapReg)
3989 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: OpType))
3990 .addImm(Val: static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
3991 .addUse(RegNo: OpReg);
3992 GR.add(Obj: OpDefine, MI: MIB);
3993 MIB.constrainAllUses(TII, TRI, RBI);
3994 }
3995 return true;
3996}
3997
3998bool SPIRVInstructionSelector::selectDerivativeInst(
3999 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
4000 const unsigned DPdOpCode) const {
4001 // TODO: This should check specifically for Fragment Execution Model, but STI
4002 // doesn't provide that information yet. See #167562
4003 errorIfInstrOutsideShader(I);
4004
4005 // If the arg/result types are half then we need to wrap the instr in
4006 // conversions to float
4007 // This case occurs because a half arg/result is legal in HLSL but not spirv.
4008 Register SrcReg = I.getOperand(i: 2).getReg();
4009 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg);
4010 unsigned BitWidth = std::min(a: GR.getScalarOrVectorBitWidth(Type: SrcType),
4011 b: GR.getScalarOrVectorBitWidth(Type: ResType));
4012 if (BitWidth == 32)
4013 return BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
4014 .addDef(RegNo: ResVReg)
4015 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4016 .addUse(RegNo: I.getOperand(i: 2).getReg());
4017
4018 MachineIRBuilder MIRBuilder(I);
4019 unsigned componentCount = GR.getScalarOrVectorComponentCount(Type: SrcType);
4020 SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(BitWidth: 32, I, TII);
4021 if (componentCount != 1)
4022 F32ConvertTy = GR.getOrCreateSPIRVVectorType(BaseType: F32ConvertTy, NumElements: componentCount,
4023 MIRBuilder, EmitIR: false);
4024
4025 const TargetRegisterClass *RegClass = GR.getRegClass(SpvType: SrcType);
4026 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
4027 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
4028
4029 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
4030 .addDef(RegNo: ConvertToVReg)
4031 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
4032 .addUse(RegNo: SrcReg)
4033 .constrainAllUses(TII, TRI, RBI);
4034 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: DPdOpCode))
4035 .addDef(RegNo: DpdOpVReg)
4036 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: F32ConvertTy))
4037 .addUse(RegNo: ConvertToVReg)
4038 .constrainAllUses(TII, TRI, RBI);
4039 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpFConvert))
4040 .addDef(RegNo: ResVReg)
4041 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4042 .addUse(RegNo: DpdOpVReg)
4043 .constrainAllUses(TII, TRI, RBI);
4044 return true;
4045}
4046
4047bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
4048 SPIRVTypeInst ResType,
4049 MachineInstr &I) const {
4050 MachineBasicBlock &BB = *I.getParent();
4051 Intrinsic::ID IID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
4052 switch (IID) {
4053 case Intrinsic::spv_load:
4054 return selectLoad(ResVReg, ResType, I);
4055 case Intrinsic::spv_store:
4056 return selectStore(I);
4057 case Intrinsic::spv_extractv:
4058 return selectExtractVal(ResVReg, ResType, I);
4059 case Intrinsic::spv_insertv:
4060 return selectInsertVal(ResVReg, ResType, I);
4061 case Intrinsic::spv_extractelt:
4062 return selectExtractElt(ResVReg, ResType, I);
4063 case Intrinsic::spv_insertelt:
4064 return selectInsertElt(ResVReg, ResType, I);
4065 case Intrinsic::spv_gep:
4066 return selectGEP(ResVReg, ResType, I);
4067 case Intrinsic::spv_bitcast: {
4068 Register OpReg = I.getOperand(i: 2).getReg();
4069 SPIRVTypeInst OpType =
4070 OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr;
4071 if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType))
4072 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
4073 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {OpReg}, Opcode: SPIRV::OpBitcast);
4074 }
4075 case Intrinsic::spv_unref_global:
4076 case Intrinsic::spv_init_global: {
4077 MachineInstr *MI = MRI->getVRegDef(Reg: I.getOperand(i: 1).getReg());
4078 MachineInstr *Init = I.getNumExplicitOperands() > 2
4079 ? MRI->getVRegDef(Reg: I.getOperand(i: 2).getReg())
4080 : nullptr;
4081 assert(MI);
4082 Register GVarVReg = MI->getOperand(i: 0).getReg();
4083 if (!selectGlobalValue(ResVReg: GVarVReg, I&: *MI, Init))
4084 return false;
4085 // We violate SSA form by inserting OpVariable and still having a gMIR
4086 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
4087 // the duplicated definition.
4088 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
4089 GR.invalidateMachineInstr(MI);
4090 MI->eraseFromParent();
4091 }
4092 return true;
4093 }
4094 case Intrinsic::spv_undef: {
4095 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
4096 .addDef(RegNo: ResVReg)
4097 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
4098 MIB.constrainAllUses(TII, TRI, RBI);
4099 return true;
4100 }
4101 case Intrinsic::spv_const_composite: {
4102 // If no values are attached, the composite is null constant.
4103 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
4104 SmallVector<Register> CompositeArgs;
4105 MRI->setRegClass(Reg: ResVReg, RC: GR.getRegClass(SpvType: ResType));
4106
4107 // skip type MD node we already used when generated assign.type for this
4108 if (!IsNull) {
4109 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
4110 return false;
4111 MachineIRBuilder MIR(I);
4112 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
4113 MIRBuilder&: MIR, Opcode: SPIRV::OpConstantComposite, MinWC: 3,
4114 ContinuedOpcode: SPIRV::OpConstantCompositeContinuedINTEL, Args: CompositeArgs, ReturnRegister: ResVReg,
4115 TypeID: GR.getSPIRVTypeID(SpirvType: ResType));
4116 for (auto *Instr : Instructions) {
4117 Instr->setDebugLoc(I.getDebugLoc());
4118 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
4119 }
4120 return true;
4121 } else {
4122 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
4123 .addDef(RegNo: ResVReg)
4124 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
4125 MIB.constrainAllUses(TII, TRI, RBI);
4126 return true;
4127 }
4128 }
4129 case Intrinsic::spv_assign_name: {
4130 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName));
4131 MIB.addUse(RegNo: I.getOperand(i: I.getNumExplicitDefs() + 1).getReg());
4132 for (unsigned i = I.getNumExplicitDefs() + 2;
4133 i < I.getNumExplicitOperands(); ++i) {
4134 MIB.addImm(Val: I.getOperand(i).getImm());
4135 }
4136 MIB.constrainAllUses(TII, TRI, RBI);
4137 return true;
4138 }
4139 case Intrinsic::spv_switch: {
4140 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSwitch));
4141 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
4142 if (I.getOperand(i).isReg())
4143 MIB.addReg(RegNo: I.getOperand(i).getReg());
4144 else if (I.getOperand(i).isCImm())
4145 addNumImm(Imm: I.getOperand(i).getCImm()->getValue(), MIB);
4146 else if (I.getOperand(i).isMBB())
4147 MIB.addMBB(MBB: I.getOperand(i).getMBB());
4148 else
4149 llvm_unreachable("Unexpected OpSwitch operand");
4150 }
4151 MIB.constrainAllUses(TII, TRI, RBI);
4152 return true;
4153 }
4154 case Intrinsic::spv_loop_merge: {
4155 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopMerge));
4156 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
4157 if (I.getOperand(i).isMBB())
4158 MIB.addMBB(MBB: I.getOperand(i).getMBB());
4159 else
4160 MIB.addImm(Val: foldImm(MO: I.getOperand(i), MRI));
4161 }
4162 MIB.constrainAllUses(TII, TRI, RBI);
4163 return true;
4164 }
4165 case Intrinsic::spv_loop_control_intel: {
4166 auto MIB =
4167 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoopControlINTEL));
4168 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
4169 MIB.addImm(Val: foldImm(MO: I.getOperand(i: J), MRI));
4170 MIB.constrainAllUses(TII, TRI, RBI);
4171 return true;
4172 }
4173 case Intrinsic::spv_selection_merge: {
4174 auto MIB =
4175 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSelectionMerge));
4176 assert(I.getOperand(1).isMBB() &&
4177 "operand 1 to spv_selection_merge must be a basic block");
4178 MIB.addMBB(MBB: I.getOperand(i: 1).getMBB());
4179 MIB.addImm(Val: getSelectionOperandForImm(Imm: I.getOperand(i: 2).getImm()));
4180 MIB.constrainAllUses(TII, TRI, RBI);
4181 return true;
4182 }
4183 case Intrinsic::spv_cmpxchg:
4184 return selectAtomicCmpXchg(ResVReg, ResType, I);
4185 case Intrinsic::spv_unreachable:
4186 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUnreachable))
4187 .constrainAllUses(TII, TRI, RBI);
4188 return true;
4189 case Intrinsic::spv_alloca:
4190 return selectFrameIndex(ResVReg, ResType, I);
4191 case Intrinsic::spv_alloca_array:
4192 return selectAllocaArray(ResVReg, ResType, I);
4193 case Intrinsic::spv_assume:
4194 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
4195 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAssumeTrueKHR))
4196 .addUse(RegNo: I.getOperand(i: 1).getReg())
4197 .constrainAllUses(TII, TRI, RBI);
4198 return true;
4199 }
4200 break;
4201 case Intrinsic::spv_expect:
4202 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
4203 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExpectKHR))
4204 .addDef(RegNo: ResVReg)
4205 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4206 .addUse(RegNo: I.getOperand(i: 2).getReg())
4207 .addUse(RegNo: I.getOperand(i: 3).getReg())
4208 .constrainAllUses(TII, TRI, RBI);
4209 return true;
4210 }
4211 break;
4212 case Intrinsic::arithmetic_fence:
4213 if (STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
4214 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpArithmeticFenceEXT))
4215 .addDef(RegNo: ResVReg)
4216 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4217 .addUse(RegNo: I.getOperand(i: 2).getReg())
4218 .constrainAllUses(TII, TRI, RBI);
4219 return true;
4220 } else
4221 return BuildCOPY(DestReg: ResVReg, SrcReg: I.getOperand(i: 2).getReg(), I);
4222 break;
4223 case Intrinsic::spv_thread_id:
4224 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
4225 // intrinsic in LLVM IR for SPIR-V backend.
4226 //
4227 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
4228 // `GlobalInvocationId` builtin variable
4229 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
4230 ResType, I);
4231 case Intrinsic::spv_thread_id_in_group:
4232 // The HLSL SV_GroupThreadId semantic is lowered to
4233 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
4234 //
4235 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
4236 // translated to a `LocalInvocationId` builtin variable
4237 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationId, ResVReg,
4238 ResType, I);
4239 case Intrinsic::spv_group_id:
4240 // The HLSL SV_GroupId semantic is lowered to
4241 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
4242 //
4243 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
4244 // builtin variable
4245 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
4246 I);
4247 case Intrinsic::spv_flattened_thread_id_in_group:
4248 // The HLSL SV_GroupIndex semantic is lowered to
4249 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
4250 // backend.
4251 //
4252 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
4253 // a `LocalInvocationIndex` builtin variable
4254 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
4255 ResType, I);
4256 case Intrinsic::spv_workgroup_size:
4257 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::WorkgroupSize, ResVReg,
4258 ResType, I);
4259 case Intrinsic::spv_global_size:
4260 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
4261 I);
4262 case Intrinsic::spv_global_offset:
4263 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::GlobalOffset, ResVReg,
4264 ResType, I);
4265 case Intrinsic::spv_num_workgroups:
4266 return loadVec3BuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumWorkgroups, ResVReg,
4267 ResType, I);
4268 case Intrinsic::spv_subgroup_size:
4269 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
4270 I);
4271 case Intrinsic::spv_num_subgroups:
4272 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
4273 I);
4274 case Intrinsic::spv_subgroup_id:
4275 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
4276 case Intrinsic::spv_subgroup_local_invocation_id:
4277 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupLocalInvocationId,
4278 ResVReg, ResType, I);
4279 case Intrinsic::spv_subgroup_max_size:
4280 return loadBuiltinInputID(BuiltInValue: SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
4281 I);
4282 case Intrinsic::spv_fdot:
4283 return selectFloatDot(ResVReg, ResType, I);
4284 case Intrinsic::spv_udot:
4285 case Intrinsic::spv_sdot:
4286 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4287 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
4288 return selectIntegerDot(ResVReg, ResType, I,
4289 /*Signed=*/IID == Intrinsic::spv_sdot);
4290 return selectIntegerDotExpansion(ResVReg, ResType, I);
4291 case Intrinsic::spv_dot4add_i8packed:
4292 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4293 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
4294 return selectDot4AddPacked<true>(ResVReg, ResType, I);
4295 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
4296 case Intrinsic::spv_dot4add_u8packed:
4297 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) ||
4298 STI.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
4299 return selectDot4AddPacked<false>(ResVReg, ResType, I);
4300 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
4301 case Intrinsic::spv_all:
4302 return selectAll(ResVReg, ResType, I);
4303 case Intrinsic::spv_any:
4304 return selectAny(ResVReg, ResType, I);
4305 case Intrinsic::spv_cross:
4306 return selectExtInst(ResVReg, ResType, I, CLInst: CL::cross, GLInst: GL::Cross);
4307 case Intrinsic::spv_distance:
4308 return selectExtInst(ResVReg, ResType, I, CLInst: CL::distance, GLInst: GL::Distance);
4309 case Intrinsic::spv_lerp:
4310 return selectExtInst(ResVReg, ResType, I, CLInst: CL::mix, GLInst: GL::FMix);
4311 case Intrinsic::spv_length:
4312 return selectExtInst(ResVReg, ResType, I, CLInst: CL::length, GLInst: GL::Length);
4313 case Intrinsic::spv_degrees:
4314 return selectExtInst(ResVReg, ResType, I, CLInst: CL::degrees, GLInst: GL::Degrees);
4315 case Intrinsic::spv_faceforward:
4316 return selectExtInst(ResVReg, ResType, I, GLInst: GL::FaceForward);
4317 case Intrinsic::spv_frac:
4318 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fract, GLInst: GL::Fract);
4319 case Intrinsic::spv_isinf:
4320 return selectOpIsInf(ResVReg, ResType, I);
4321 case Intrinsic::spv_isnan:
4322 return selectOpIsNan(ResVReg, ResType, I);
4323 case Intrinsic::spv_normalize:
4324 return selectExtInst(ResVReg, ResType, I, CLInst: CL::normalize, GLInst: GL::Normalize);
4325 case Intrinsic::spv_refract:
4326 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Refract);
4327 case Intrinsic::spv_reflect:
4328 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Reflect);
4329 case Intrinsic::spv_rsqrt:
4330 return selectExtInst(ResVReg, ResType, I, CLInst: CL::rsqrt, GLInst: GL::InverseSqrt);
4331 case Intrinsic::spv_sign:
4332 return selectSign(ResVReg, ResType, I);
4333 case Intrinsic::spv_smoothstep:
4334 return selectExtInst(ResVReg, ResType, I, CLInst: CL::smoothstep, GLInst: GL::SmoothStep);
4335 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
4336 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
4337 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
4338 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
4339 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
4340 return selectFirstBitLow(ResVReg, ResType, I);
4341 case Intrinsic::spv_group_memory_barrier_with_group_sync: {
4342 Register MemSemReg =
4343 buildI32Constant(Val: SPIRV::MemorySemantics::SequentiallyConsistent, I);
4344 Register ScopeReg = buildI32Constant(Val: SPIRV::Scope::Workgroup, I);
4345 MachineBasicBlock &BB = *I.getParent();
4346 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpControlBarrier))
4347 .addUse(RegNo: ScopeReg)
4348 .addUse(RegNo: ScopeReg)
4349 .addUse(RegNo: MemSemReg)
4350 .constrainAllUses(TII, TRI, RBI);
4351 return true;
4352 }
4353 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
4354 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 1).getReg();
4355 SPIRV::StorageClass::StorageClass ResSC =
4356 GR.getPointerStorageClass(Type: ResType);
4357 if (!isGenericCastablePtr(SC: ResSC))
4358 report_fatal_error(reason: "The target storage class is not castable from the "
4359 "Generic storage class");
4360 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpGenericCastToPtrExplicit))
4361 .addDef(RegNo: ResVReg)
4362 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4363 .addUse(RegNo: PtrReg)
4364 .addImm(Val: ResSC)
4365 .constrainAllUses(TII, TRI, RBI);
4366 return true;
4367 }
4368 case Intrinsic::spv_lifetime_start:
4369 case Intrinsic::spv_lifetime_end: {
4370 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
4371 : SPIRV::OpLifetimeStop;
4372 int64_t Size = I.getOperand(i: I.getNumExplicitDefs() + 1).getImm();
4373 Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 2).getReg();
4374 if (Size == -1)
4375 Size = 0;
4376 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Op))
4377 .addUse(RegNo: PtrReg)
4378 .addImm(Val: Size)
4379 .constrainAllUses(TII, TRI, RBI);
4380 return true;
4381 }
4382 case Intrinsic::spv_saturate:
4383 return selectSaturate(ResVReg, ResType, I);
4384 case Intrinsic::spv_nclamp:
4385 return selectExtInst(ResVReg, ResType, I, CLInst: CL::fclamp, GLInst: GL::NClamp);
4386 case Intrinsic::spv_uclamp:
4387 return selectExtInst(ResVReg, ResType, I, CLInst: CL::u_clamp, GLInst: GL::UClamp);
4388 case Intrinsic::spv_sclamp:
4389 return selectExtInst(ResVReg, ResType, I, CLInst: CL::s_clamp, GLInst: GL::SClamp);
4390 case Intrinsic::spv_subgroup_prefix_bit_count:
4391 return selectWavePrefixBitCount(ResVReg, ResType, I);
4392 case Intrinsic::spv_wave_active_countbits:
4393 return selectWaveActiveCountBits(ResVReg, ResType, I);
4394 case Intrinsic::spv_wave_all_equal:
4395 return selectWaveActiveAllEqual(ResVReg, ResType, I);
4396 case Intrinsic::spv_wave_all:
4397 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAll);
4398 case Intrinsic::spv_wave_any:
4399 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformAny);
4400 case Intrinsic::spv_subgroup_ballot:
4401 return selectWaveOpInst(ResVReg, ResType, I,
4402 Opcode: SPIRV::OpGroupNonUniformBallot);
4403 case Intrinsic::spv_wave_is_first_lane:
4404 return selectWaveOpInst(ResVReg, ResType, I, Opcode: SPIRV::OpGroupNonUniformElect);
4405 case Intrinsic::spv_wave_reduce_or:
4406 return selectWaveReduceOp(ResVReg, ResType, I,
4407 Opcode: SPIRV::OpGroupNonUniformBitwiseOr);
4408 case Intrinsic::spv_wave_reduce_xor:
4409 return selectWaveReduceOp(ResVReg, ResType, I,
4410 Opcode: SPIRV::OpGroupNonUniformBitwiseXor);
4411 case Intrinsic::spv_wave_reduce_umax:
4412 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
4413 case Intrinsic::spv_wave_reduce_max:
4414 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
4415 case Intrinsic::spv_wave_reduce_umin:
4416 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
4417 case Intrinsic::spv_wave_reduce_min:
4418 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
4419 case Intrinsic::spv_wave_reduce_sum:
4420 return selectWaveReduceSum(ResVReg, ResType, I);
4421 case Intrinsic::spv_wave_product:
4422 return selectWaveReduceProduct(ResVReg, ResType, I);
4423 case Intrinsic::spv_wave_readlane:
4424 return selectWaveOpInst(ResVReg, ResType, I,
4425 Opcode: SPIRV::OpGroupNonUniformShuffle);
4426 case Intrinsic::spv_wave_prefix_sum:
4427 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
4428 case Intrinsic::spv_wave_prefix_product:
4429 return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
4430 case Intrinsic::spv_step:
4431 return selectExtInst(ResVReg, ResType, I, CLInst: CL::step, GLInst: GL::Step);
4432 case Intrinsic::spv_radians:
4433 return selectExtInst(ResVReg, ResType, I, CLInst: CL::radians, GLInst: GL::Radians);
4434 // Discard intrinsics which we do not expect to actually represent code after
4435 // lowering or intrinsics which are not implemented but should not crash when
4436 // found in a customer's LLVM IR input.
4437 case Intrinsic::instrprof_increment:
4438 case Intrinsic::instrprof_increment_step:
4439 case Intrinsic::instrprof_value_profile:
4440 break;
4441 // Discard internal intrinsics.
4442 case Intrinsic::spv_value_md:
4443 break;
4444 case Intrinsic::spv_resource_handlefrombinding: {
4445 return selectHandleFromBinding(ResVReg, ResType, I);
4446 }
4447 case Intrinsic::spv_resource_counterhandlefrombinding:
4448 return selectCounterHandleFromBinding(ResVReg, ResType, I);
4449 case Intrinsic::spv_resource_updatecounter:
4450 return selectUpdateCounter(ResVReg, ResType, I);
4451 case Intrinsic::spv_resource_store_typedbuffer: {
4452 return selectImageWriteIntrinsic(I);
4453 }
4454 case Intrinsic::spv_resource_load_typedbuffer: {
4455 return selectReadImageIntrinsic(ResVReg, ResType, I);
4456 }
4457 case Intrinsic::spv_resource_load_level: {
4458 return selectLoadLevelIntrinsic(ResVReg, ResType, I);
4459 }
4460 case Intrinsic::spv_resource_sample:
4461 case Intrinsic::spv_resource_sample_clamp:
4462 return selectSampleBasicIntrinsic(ResVReg, ResType, I);
4463 case Intrinsic::spv_resource_samplebias:
4464 case Intrinsic::spv_resource_samplebias_clamp:
4465 return selectSampleBiasIntrinsic(ResVReg, ResType, I);
4466 case Intrinsic::spv_resource_samplegrad:
4467 case Intrinsic::spv_resource_samplegrad_clamp:
4468 return selectSampleGradIntrinsic(ResVReg, ResType, I);
4469 case Intrinsic::spv_resource_samplelevel:
4470 return selectSampleLevelIntrinsic(ResVReg, ResType, I);
4471 case Intrinsic::spv_resource_samplecmp:
4472 case Intrinsic::spv_resource_samplecmp_clamp:
4473 return selectSampleCmpIntrinsic(ResVReg, ResType, I);
4474 case Intrinsic::spv_resource_samplecmplevelzero:
4475 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I);
4476 case Intrinsic::spv_resource_gather:
4477 case Intrinsic::spv_resource_gather_cmp:
4478 return selectGatherIntrinsic(ResVReg, ResType, I);
4479 case Intrinsic::spv_resource_getpointer: {
4480 return selectResourceGetPointer(ResVReg, ResType, I);
4481 }
4482 case Intrinsic::spv_pushconstant_getpointer: {
4483 return selectPushConstantGetPointer(ResVReg, ResType, I);
4484 }
4485 case Intrinsic::spv_discard: {
4486 return selectDiscard(ResVReg, ResType, I);
4487 }
4488 case Intrinsic::spv_resource_nonuniformindex: {
4489 return selectResourceNonUniformIndex(ResVReg, ResType, I);
4490 }
4491 case Intrinsic::spv_unpackhalf2x16: {
4492 return selectExtInst(ResVReg, ResType, I, GLInst: GL::UnpackHalf2x16);
4493 }
4494 case Intrinsic::spv_packhalf2x16: {
4495 return selectExtInst(ResVReg, ResType, I, GLInst: GL::PackHalf2x16);
4496 }
4497 case Intrinsic::spv_ddx:
4498 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdx);
4499 case Intrinsic::spv_ddy:
4500 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdy);
4501 case Intrinsic::spv_ddx_coarse:
4502 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxCoarse);
4503 case Intrinsic::spv_ddy_coarse:
4504 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyCoarse);
4505 case Intrinsic::spv_ddx_fine:
4506 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdxFine);
4507 case Intrinsic::spv_ddy_fine:
4508 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpDPdyFine);
4509 case Intrinsic::spv_fwidth:
4510 return selectDerivativeInst(ResVReg, ResType, I, DPdOpCode: SPIRV::OpFwidth);
4511 case Intrinsic::spv_masked_gather:
4512 if (STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
4513 return selectMaskedGather(ResVReg, ResType, I);
4514 return diagnoseUnsupported(
4515 I, Msg: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
4516 case Intrinsic::spv_masked_scatter:
4517 if (STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
4518 return selectMaskedScatter(I);
4519 return diagnoseUnsupported(
4520 I, Msg: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
4521 default: {
4522 std::string DiagMsg;
4523 raw_string_ostream OS(DiagMsg);
4524 I.print(OS);
4525 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
4526 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
4527 }
4528 }
4529 return true;
4530}
4531
4532bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
4533 SPIRVTypeInst ResType,
4534 MachineInstr &I) const {
4535 // The images need to be loaded in the same basic block as their use. We defer
4536 // loading the image to the intrinsic that uses it.
4537 if (ResType->getOpcode() == SPIRV::OpTypeImage)
4538 return true;
4539
4540 return loadHandleBeforePosition(HandleReg&: ResVReg, ResType: GR.getSPIRVTypeForVReg(VReg: ResVReg),
4541 HandleDef&: *cast<GIntrinsic>(Val: &I), Pos&: I);
4542}
4543
4544bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
4545 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4546 auto &Intr = cast<GIntrinsic>(Val&: I);
4547 assert(Intr.getIntrinsicID() ==
4548 Intrinsic::spv_resource_counterhandlefrombinding);
4549
4550 // Extract information from the intrinsic call.
4551 Register MainHandleReg = Intr.getOperand(i: 2).getReg();
4552 auto *MainHandleDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: MainHandleReg));
4553 assert(MainHandleDef->getIntrinsicID() ==
4554 Intrinsic::spv_resource_handlefrombinding);
4555
4556 uint32_t Set = getIConstVal(ConstReg: Intr.getOperand(i: 4).getReg(), MRI);
4557 uint32_t Binding = getIConstVal(ConstReg: Intr.getOperand(i: 3).getReg(), MRI);
4558 uint32_t ArraySize = getIConstVal(ConstReg: MainHandleDef->getOperand(i: 4).getReg(), MRI);
4559 Register IndexReg = MainHandleDef->getOperand(i: 5).getReg();
4560 std::string CounterName =
4561 getStringValueFromReg(Reg: MainHandleDef->getOperand(i: 6).getReg(), MRI&: *MRI) +
4562 ".counter";
4563
4564 // Create the counter variable.
4565 MachineIRBuilder MIRBuilder(I);
4566 Register CounterVarReg =
4567 buildPointerToResource(ResType: SPIRVTypeInst(GR.getPointeeType(PtrType: ResType)),
4568 SC: GR.getPointerStorageClass(Type: ResType), Set, Binding,
4569 ArraySize, IndexReg, Name: CounterName, MIRBuilder);
4570
4571 return BuildCOPY(DestReg: ResVReg, SrcReg: CounterVarReg, I);
4572}
4573
4574bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
4575 SPIRVTypeInst ResType,
4576 MachineInstr &I) const {
4577 auto &Intr = cast<GIntrinsic>(Val&: I);
4578 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
4579
4580 Register CounterHandleReg = Intr.getOperand(i: 2).getReg();
4581 Register IncrReg = Intr.getOperand(i: 3).getReg();
4582
4583 // The counter handle is a pointer to the counter variable (which is a struct
4584 // containing an i32). We need to get a pointer to that i32 member to do the
4585 // atomic operation.
4586#ifndef NDEBUG
4587 SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
4588 SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType);
4589 assert(CounterVarPointeeType &&
4590 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
4591 "Counter variable must be a struct");
4592 assert(GR.getPointerStorageClass(CounterVarType) ==
4593 SPIRV::StorageClass::StorageBuffer &&
4594 "Counter variable must be in the storage buffer storage class");
4595 assert(CounterVarPointeeType->getNumOperands() == 2 &&
4596 "Counter variable must have exactly 1 member in the struct");
4597 const SPIRVTypeInst MemberType =
4598 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
4599 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
4600 "Counter variable struct must have a single i32 member");
4601#endif
4602
4603 // The struct has a single i32 member.
4604 MachineIRBuilder MIRBuilder(I);
4605 const Type *LLVMIntType =
4606 Type::getInt32Ty(C&: I.getMF()->getFunction().getContext());
4607
4608 SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType(
4609 BaseType: LLVMIntType, MIRBuilder, SC: SPIRV::StorageClass::StorageBuffer);
4610
4611 Register Zero = buildI32Constant(Val: 0, I);
4612
4613 Register PtrToCounter =
4614 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: IntPtrType));
4615 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAccessChain))
4616 .addDef(RegNo: PtrToCounter)
4617 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: IntPtrType))
4618 .addUse(RegNo: CounterHandleReg)
4619 .addUse(RegNo: Zero)
4620 .constrainAllUses(TII, TRI, RBI);
4621
4622 // For UAV/SSBO counters, the scope is Device. The counter variable is not
4623 // used as a flag. So the memory semantics can be None.
4624 Register Scope = buildI32Constant(Val: SPIRV::Scope::Device, I);
4625 Register Semantics = buildI32Constant(Val: SPIRV::MemorySemantics::None, I);
4626
4627 int64_t IncrVal = getIConstValSext(ConstReg: IncrReg, MRI);
4628 Register Incr = buildI32Constant(Val: static_cast<uint32_t>(IncrVal), I);
4629
4630 Register AtomicRes = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
4631 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAtomicIAdd))
4632 .addDef(RegNo: AtomicRes)
4633 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4634 .addUse(RegNo: PtrToCounter)
4635 .addUse(RegNo: Scope)
4636 .addUse(RegNo: Semantics)
4637 .addUse(RegNo: Incr)
4638 .constrainAllUses(TII, TRI, RBI);
4639 if (IncrVal >= 0) {
4640 return BuildCOPY(DestReg: ResVReg, SrcReg: AtomicRes, I);
4641 }
4642
4643 // In HLSL, IncrementCounter returns the value *before* the increment, while
4644 // DecrementCounter returns the value *after* the decrement. Both are lowered
4645 // to the same atomic intrinsic which returns the value *before* the
4646 // operation. So for decrements (negative IncrVal), we must subtract the
4647 // increment value from the result to get the post-decrement value.
4648 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpIAddS))
4649 .addDef(RegNo: ResVReg)
4650 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4651 .addUse(RegNo: AtomicRes)
4652 .addUse(RegNo: Incr)
4653 .constrainAllUses(TII, TRI, RBI);
4654 return true;
4655}
4656bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg,
4657 SPIRVTypeInst ResType,
4658 MachineInstr &I) const {
4659
4660 // If the load of the image is in a different basic block, then
4661 // this will generate invalid code. A proper solution is to move
4662 // the OpLoad from selectHandleFromBinding here. However, to do
4663 // that we will need to change the return type of the intrinsic.
4664 // We will do that when we can, but for now trying to move forward with other
4665 // issues.
4666 Register ImageReg = I.getOperand(i: 2).getReg();
4667 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4668 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4669 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4670 HandleDef&: *ImageDef, Pos&: I)) {
4671 return false;
4672 }
4673
4674 Register IdxReg = I.getOperand(i: 3).getReg();
4675 DebugLoc Loc = I.getDebugLoc();
4676 MachineInstr &Pos = I;
4677
4678 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewImageReg, IdxReg, Loc,
4679 Pos);
4680}
4681
4682bool SPIRVInstructionSelector::generateSampleImage(
4683 Register ResVReg, SPIRVTypeInst ResType, Register ImageReg,
4684 Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps,
4685 DebugLoc Loc, MachineInstr &Pos) const {
4686 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4687 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4688 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4689 HandleDef&: *ImageDef, Pos)) {
4690 return false;
4691 }
4692
4693 auto *SamplerDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: SamplerReg));
4694 Register NewSamplerReg =
4695 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: SamplerReg));
4696 if (!loadHandleBeforePosition(HandleReg&: NewSamplerReg,
4697 ResType: GR.getSPIRVTypeForVReg(VReg: SamplerReg), HandleDef&: *SamplerDef,
4698 Pos)) {
4699 return false;
4700 }
4701
4702 MachineIRBuilder MIRBuilder(Pos);
4703 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
4704 ImageType: GR.getSPIRVTypeForVReg(VReg: ImageReg), MIRBuilder);
4705 Register SampledImageReg =
4706 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: SampledImageType));
4707
4708 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpSampledImage))
4709 .addDef(RegNo: SampledImageReg)
4710 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SampledImageType))
4711 .addUse(RegNo: NewImageReg)
4712 .addUse(RegNo: NewSamplerReg)
4713 .constrainAllUses(TII, TRI, RBI);
4714
4715 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
4716 ImOps.Lod.has_value();
4717 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
4718 : SPIRV::OpImageSampleImplicitLod;
4719 if (ImOps.Compare)
4720 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
4721 : SPIRV::OpImageSampleDrefImplicitLod;
4722
4723 auto MIB = BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode))
4724 .addDef(RegNo: ResVReg)
4725 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4726 .addUse(RegNo: SampledImageReg)
4727 .addUse(RegNo: CoordinateReg);
4728
4729 if (ImOps.Compare)
4730 MIB.addUse(RegNo: *ImOps.Compare);
4731
4732 uint32_t ImageOperands = 0;
4733 if (ImOps.Bias)
4734 ImageOperands |= SPIRV::ImageOperand::Bias;
4735 if (ImOps.Lod)
4736 ImageOperands |= SPIRV::ImageOperand::Lod;
4737 if (ImOps.GradX && ImOps.GradY)
4738 ImageOperands |= SPIRV::ImageOperand::Grad;
4739 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(Reg: *ImOps.Offset)) {
4740 if (isConstReg(MRI, OpReg: *ImOps.Offset))
4741 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
4742 else {
4743 Pos.emitGenericError(
4744 ErrMsg: "Non-constant offsets are not supported in sample instructions.");
4745 }
4746 }
4747 if (ImOps.MinLod)
4748 ImageOperands |= SPIRV::ImageOperand::MinLod;
4749
4750 if (ImageOperands != 0) {
4751 MIB.addImm(Val: ImageOperands);
4752 if (ImageOperands & SPIRV::ImageOperand::Bias)
4753 MIB.addUse(RegNo: *ImOps.Bias);
4754 if (ImageOperands & SPIRV::ImageOperand::Lod)
4755 MIB.addUse(RegNo: *ImOps.Lod);
4756 if (ImageOperands & SPIRV::ImageOperand::Grad) {
4757 MIB.addUse(RegNo: *ImOps.GradX);
4758 MIB.addUse(RegNo: *ImOps.GradY);
4759 }
4760 if (ImageOperands &
4761 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
4762 MIB.addUse(RegNo: *ImOps.Offset);
4763 if (ImageOperands & SPIRV::ImageOperand::MinLod)
4764 MIB.addUse(RegNo: *ImOps.MinLod);
4765 }
4766
4767 MIB.constrainAllUses(TII, TRI, RBI);
4768 return true;
4769}
4770
4771bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
4772 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4773 Register ImageReg = I.getOperand(i: 2).getReg();
4774 Register SamplerReg = I.getOperand(i: 3).getReg();
4775 Register CoordinateReg = I.getOperand(i: 4).getReg();
4776 ImageOperands ImOps;
4777 if (I.getNumOperands() > 5)
4778 ImOps.Offset = I.getOperand(i: 5).getReg();
4779 if (I.getNumOperands() > 6)
4780 ImOps.MinLod = I.getOperand(i: 6).getReg();
4781 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4782 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4783}
4784
4785bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
4786 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4787 Register ImageReg = I.getOperand(i: 2).getReg();
4788 Register SamplerReg = I.getOperand(i: 3).getReg();
4789 Register CoordinateReg = I.getOperand(i: 4).getReg();
4790 ImageOperands ImOps;
4791 ImOps.Bias = I.getOperand(i: 5).getReg();
4792 if (I.getNumOperands() > 6)
4793 ImOps.Offset = I.getOperand(i: 6).getReg();
4794 if (I.getNumOperands() > 7)
4795 ImOps.MinLod = I.getOperand(i: 7).getReg();
4796 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4797 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4798}
4799
4800bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
4801 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4802 Register ImageReg = I.getOperand(i: 2).getReg();
4803 Register SamplerReg = I.getOperand(i: 3).getReg();
4804 Register CoordinateReg = I.getOperand(i: 4).getReg();
4805 ImageOperands ImOps;
4806 ImOps.GradX = I.getOperand(i: 5).getReg();
4807 ImOps.GradY = I.getOperand(i: 6).getReg();
4808 if (I.getNumOperands() > 7)
4809 ImOps.Offset = I.getOperand(i: 7).getReg();
4810 if (I.getNumOperands() > 8)
4811 ImOps.MinLod = I.getOperand(i: 8).getReg();
4812 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4813 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4814}
4815
4816bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
4817 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4818 Register ImageReg = I.getOperand(i: 2).getReg();
4819 Register SamplerReg = I.getOperand(i: 3).getReg();
4820 Register CoordinateReg = I.getOperand(i: 4).getReg();
4821 ImageOperands ImOps;
4822 ImOps.Lod = I.getOperand(i: 5).getReg();
4823 if (I.getNumOperands() > 6)
4824 ImOps.Offset = I.getOperand(i: 6).getReg();
4825 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4826 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4827}
4828
4829bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg,
4830 SPIRVTypeInst ResType,
4831 MachineInstr &I) const {
4832 Register ImageReg = I.getOperand(i: 2).getReg();
4833 Register SamplerReg = I.getOperand(i: 3).getReg();
4834 Register CoordinateReg = I.getOperand(i: 4).getReg();
4835 ImageOperands ImOps;
4836 ImOps.Compare = I.getOperand(i: 5).getReg();
4837 if (I.getNumOperands() > 6)
4838 ImOps.Offset = I.getOperand(i: 6).getReg();
4839 if (I.getNumOperands() > 7)
4840 ImOps.MinLod = I.getOperand(i: 7).getReg();
4841 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4842 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4843}
4844
4845bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(Register &ResVReg,
4846 SPIRVTypeInst ResType,
4847 MachineInstr &I) const {
4848 Register ImageReg = I.getOperand(i: 2).getReg();
4849 Register CoordinateReg = I.getOperand(i: 3).getReg();
4850 Register LodReg = I.getOperand(i: 4).getReg();
4851
4852 ImageOperands ImOps;
4853 ImOps.Lod = LodReg;
4854 if (I.getNumOperands() > 5)
4855 ImOps.Offset = I.getOperand(i: 5).getReg();
4856
4857 auto *ImageDef = dyn_cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4858 if (!ImageDef)
4859 return false;
4860
4861 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4862 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
4863 HandleDef&: *ImageDef, Pos&: I)) {
4864 return false;
4865 }
4866
4867 return generateImageReadOrFetch(ResVReg, ResType, ImageReg: NewImageReg, IdxReg: CoordinateReg,
4868 Loc: I.getDebugLoc(), Pos&: I, ImOps: &ImOps);
4869}
4870
4871bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
4872 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4873 Register ImageReg = I.getOperand(i: 2).getReg();
4874 Register SamplerReg = I.getOperand(i: 3).getReg();
4875 Register CoordinateReg = I.getOperand(i: 4).getReg();
4876 ImageOperands ImOps;
4877 ImOps.Compare = I.getOperand(i: 5).getReg();
4878 if (I.getNumOperands() > 6)
4879 ImOps.Offset = I.getOperand(i: 6).getReg();
4880 SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(BitWidth: 32, I, TII);
4881 ImOps.Lod = GR.getOrCreateConstFP(Val: APFloat(0.0f), I, SpvType: FloatTy, TII);
4882 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
4883 CoordinateReg, ImOps, Loc: I.getDebugLoc(), Pos&: I);
4884}
4885
4886bool SPIRVInstructionSelector::selectGatherIntrinsic(Register &ResVReg,
4887 SPIRVTypeInst ResType,
4888 MachineInstr &I) const {
4889 Register ImageReg = I.getOperand(i: 2).getReg();
4890 Register SamplerReg = I.getOperand(i: 3).getReg();
4891 Register CoordinateReg = I.getOperand(i: 4).getReg();
4892 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(VReg: ImageReg);
4893 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
4894 "ImageReg is not an image type.");
4895
4896 Register ComponentOrCompareReg;
4897 Register OffsetReg;
4898
4899 ComponentOrCompareReg = I.getOperand(i: 5).getReg();
4900 OffsetReg = I.getOperand(i: 6).getReg();
4901 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
4902 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
4903 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: ImageType, HandleDef&: *ImageDef, Pos&: I)) {
4904 return false;
4905 }
4906
4907 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(i: 2).getImm());
4908 if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube &&
4909 Dim != SPIRV::Dim::DIM_Rect) {
4910 I.emitGenericError(
4911 ErrMsg: "Gather operations are only supported for 2D, Cube, and Rect images.");
4912 return false;
4913 }
4914
4915 auto *SamplerDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: SamplerReg));
4916 Register NewSamplerReg =
4917 MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: SamplerReg));
4918 if (!loadHandleBeforePosition(
4919 HandleReg&: NewSamplerReg, ResType: GR.getSPIRVTypeForVReg(VReg: SamplerReg), HandleDef&: *SamplerDef, Pos&: I)) {
4920 return false;
4921 }
4922
4923 MachineIRBuilder MIRBuilder(I);
4924 SPIRVTypeInst SampledImageType =
4925 GR.getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
4926 Register SampledImageReg =
4927 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: SampledImageType));
4928
4929 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpSampledImage))
4930 .addDef(RegNo: SampledImageReg)
4931 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: SampledImageType))
4932 .addUse(RegNo: NewImageReg)
4933 .addUse(RegNo: NewSamplerReg)
4934 .constrainAllUses(TII, TRI, RBI);
4935
4936 auto IntrId = cast<GIntrinsic>(Val&: I).getIntrinsicID();
4937 bool IsGatherCmp = IntrId == Intrinsic::spv_resource_gather_cmp;
4938 unsigned Opcode =
4939 IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather;
4940
4941 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
4942 .addDef(RegNo: ResVReg)
4943 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
4944 .addUse(RegNo: SampledImageReg)
4945 .addUse(RegNo: CoordinateReg)
4946 .addUse(RegNo: ComponentOrCompareReg);
4947
4948 uint32_t ImageOperands = 0;
4949 if (OffsetReg && !isScalarOrVectorIntConstantZero(Reg: OffsetReg)) {
4950 if (Dim == SPIRV::Dim::DIM_Cube) {
4951 I.emitGenericError(
4952 ErrMsg: "Gather operations with offset are not supported for Cube images.");
4953 return false;
4954 }
4955 if (isConstReg(MRI, OpReg: OffsetReg))
4956 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
4957 else {
4958 ImageOperands |= SPIRV::ImageOperand::Offset;
4959 }
4960 }
4961
4962 if (ImageOperands != 0) {
4963 MIB.addImm(Val: ImageOperands);
4964 if (ImageOperands &
4965 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
4966 MIB.addUse(RegNo: OffsetReg);
4967 }
4968
4969 MIB.constrainAllUses(TII, TRI, RBI);
4970 return true;
4971}
4972
4973bool SPIRVInstructionSelector::generateImageReadOrFetch(
4974 Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg,
4975 Register IdxReg, DebugLoc Loc, MachineInstr &Pos,
4976 const ImageOperands *ImOps) const {
4977 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(VReg: ImageReg);
4978 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
4979 "ImageReg is not an image type.");
4980
4981 bool IsSignedInteger =
4982 sampledTypeIsSignedInteger(HandleType: GR.getTypeForSPIRVType(Ty: ImageType));
4983 // Check if the "sampled" operand of the image type is 1.
4984 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
4985 auto SampledOp = ImageType->getOperand(i: 6);
4986 bool IsFetch = (SampledOp.getImm() == 1);
4987
4988 auto AddOperands = [&](MachineInstrBuilder &MIB) {
4989 uint32_t ImageOperandsMask = 0;
4990 if (IsSignedInteger)
4991 ImageOperandsMask |= 0x1000; // SignExtend
4992
4993 if (IsFetch && ImOps) {
4994 if (ImOps->Lod)
4995 ImageOperandsMask |= SPIRV::ImageOperand::Lod;
4996 if (ImOps->Offset && !isScalarOrVectorIntConstantZero(Reg: *ImOps->Offset)) {
4997 if (isConstReg(MRI, OpReg: *ImOps->Offset))
4998 ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset;
4999 else
5000 ImageOperandsMask |= SPIRV::ImageOperand::Offset;
5001 }
5002 }
5003
5004 if (ImageOperandsMask != 0) {
5005 MIB.addImm(Val: ImageOperandsMask);
5006 if (IsFetch && ImOps) {
5007 if (ImOps->Lod)
5008 MIB.addUse(RegNo: *ImOps->Lod);
5009 if (ImOps->Offset &&
5010 (ImageOperandsMask &
5011 (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset)))
5012 MIB.addUse(RegNo: *ImOps->Offset);
5013 }
5014 }
5015 };
5016
5017 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
5018 if (ResultSize == 4) {
5019 auto BMI =
5020 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
5021 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
5022 .addDef(RegNo: ResVReg)
5023 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5024 .addUse(RegNo: ImageReg)
5025 .addUse(RegNo: IdxReg);
5026
5027 AddOperands(BMI);
5028 BMI.constrainAllUses(TII, TRI, RBI);
5029 return true;
5030 }
5031
5032 SPIRVTypeInst ReadType = widenTypeToVec4(Type: ResType, I&: Pos);
5033 Register ReadReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ReadType));
5034 auto BMI =
5035 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc,
5036 MCID: TII.get(Opcode: IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
5037 .addDef(RegNo: ReadReg)
5038 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ReadType))
5039 .addUse(RegNo: ImageReg)
5040 .addUse(RegNo: IdxReg);
5041 AddOperands(BMI);
5042 BMI.constrainAllUses(TII, TRI, RBI);
5043
5044 if (ResultSize == 1) {
5045 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: Loc, MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
5046 .addDef(RegNo: ResVReg)
5047 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5048 .addUse(RegNo: ReadReg)
5049 .addImm(Val: 0)
5050 .constrainAllUses(TII, TRI, RBI);
5051 return true;
5052 }
5053 return extractSubvector(ResVReg, ResType, ReadReg, InsertionPoint&: Pos);
5054}
5055
5056bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg,
5057 SPIRVTypeInst ResType,
5058 MachineInstr &I) const {
5059 Register ResourcePtr = I.getOperand(i: 2).getReg();
5060 SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(VReg: ResourcePtr, MF: I.getMF());
5061 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
5062 // For texel buffers, the index into the image is part of the OpImageRead or
5063 // OpImageWrite instructions. So we will do nothing in this case. This
5064 // intrinsic will be combined with the load or store when selecting the load
5065 // or store.
5066 return true;
5067 }
5068
5069 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
5070 MachineIRBuilder MIRBuilder(I);
5071
5072 Register IndexReg = I.getOperand(i: 3).getReg();
5073 Register ZeroReg =
5074 buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I);
5075 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpAccessChain))
5076 .addDef(RegNo: ResVReg)
5077 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5078 .addUse(RegNo: ResourcePtr)
5079 .addUse(RegNo: ZeroReg)
5080 .addUse(RegNo: IndexReg)
5081 .constrainAllUses(TII, TRI, RBI);
5082 return true;
5083}
5084
5085bool SPIRVInstructionSelector::selectPushConstantGetPointer(
5086 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5087 MRI->replaceRegWith(FromReg: ResVReg, ToReg: I.getOperand(i: 2).getReg());
5088 return true;
5089}
5090
5091bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
5092 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5093 Register ObjReg = I.getOperand(i: 2).getReg();
5094 if (!BuildCOPY(DestReg: ResVReg, SrcReg: ObjReg, I))
5095 return false;
5096
5097 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
5098 // Check for the registers that use the index marked as non-uniform
5099 // and recursively mark them as non-uniform.
5100 // Per the spec, it's necessary that the final argument used for
5101 // load/store/sample/atomic must be decorated, so we need to propagate the
5102 // decoration through access chains and copies.
5103 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
5104 decorateUsesAsNonUniform(NonUniformReg&: ResVReg);
5105 return true;
5106}
5107
5108void SPIRVInstructionSelector::decorateUsesAsNonUniform(
5109 Register &NonUniformReg) const {
5110 llvm::SmallVector<Register> WorkList = {NonUniformReg};
5111 while (WorkList.size() > 0) {
5112 Register CurrentReg = WorkList.back();
5113 WorkList.pop_back();
5114
5115 bool IsDecorated = false;
5116 for (MachineInstr &Use : MRI->use_instructions(Reg: CurrentReg)) {
5117 if (Use.getOpcode() == SPIRV::OpDecorate &&
5118 Use.getOperand(i: 1).getImm() == SPIRV::Decoration::NonUniformEXT) {
5119 IsDecorated = true;
5120 continue;
5121 }
5122 // Check if the instruction has the result register and add it to the
5123 // worklist.
5124 if (Use.getOperand(i: 0).isReg() && Use.getOperand(i: 0).isDef()) {
5125 Register ResultReg = Use.getOperand(i: 0).getReg();
5126 if (ResultReg == CurrentReg)
5127 continue;
5128 WorkList.push_back(Elt: ResultReg);
5129 }
5130 }
5131
5132 if (!IsDecorated) {
5133 buildOpDecorate(Reg: CurrentReg, I&: *MRI->getVRegDef(Reg: CurrentReg), TII,
5134 Dec: SPIRV::Decoration::NonUniformEXT, DecArgs: {});
5135 }
5136 }
5137}
5138
5139bool SPIRVInstructionSelector::extractSubvector(
5140 Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg,
5141 MachineInstr &InsertionPoint) const {
5142 SPIRVTypeInst InputType = GR.getResultType(VReg: ReadReg);
5143 [[maybe_unused]] uint64_t InputSize =
5144 GR.getScalarOrVectorComponentCount(Type: InputType);
5145 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(Type: ResType);
5146 assert(InputSize > 1 && "The input must be a vector.");
5147 assert(ResultSize > 1 && "The result must be a vector.");
5148 assert(ResultSize < InputSize &&
5149 "Cannot extract more element than there are in the input.");
5150 SmallVector<Register> ComponentRegisters;
5151 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(Type: ResType);
5152 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(SpvType: ScalarType);
5153 for (uint64_t I = 0; I < ResultSize; I++) {
5154 Register ComponentReg = MRI->createVirtualRegister(RegClass: ScalarRegClass);
5155 BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
5156 MIMD: InsertionPoint.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
5157 .addDef(RegNo: ComponentReg)
5158 .addUse(RegNo: ScalarType->getOperand(i: 0).getReg())
5159 .addUse(RegNo: ReadReg)
5160 .addImm(Val: I)
5161 .constrainAllUses(TII, TRI, RBI);
5162 ComponentRegisters.emplace_back(Args&: ComponentReg);
5163 }
5164
5165 MachineInstrBuilder MIB = BuildMI(BB&: *InsertionPoint.getParent(), I&: InsertionPoint,
5166 MIMD: InsertionPoint.getDebugLoc(),
5167 MCID: TII.get(Opcode: SPIRV::OpCompositeConstruct))
5168 .addDef(RegNo: ResVReg)
5169 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
5170
5171 for (Register ComponentReg : ComponentRegisters)
5172 MIB.addUse(RegNo: ComponentReg);
5173 MIB.constrainAllUses(TII, TRI, RBI);
5174 return true;
5175}
5176
5177bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
5178 MachineInstr &I) const {
5179 // If the load of the image is in a different basic block, then
5180 // this will generate invalid code. A proper solution is to move
5181 // the OpLoad from selectHandleFromBinding here. However, to do
5182 // that we will need to change the return type of the intrinsic.
5183 // We will do that when we can, but for now trying to move forward with other
5184 // issues.
5185 Register ImageReg = I.getOperand(i: 1).getReg();
5186 auto *ImageDef = cast<GIntrinsic>(Val: getVRegDef(MRI&: *MRI, Reg: ImageReg));
5187 Register NewImageReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: ImageReg));
5188 if (!loadHandleBeforePosition(HandleReg&: NewImageReg, ResType: GR.getSPIRVTypeForVReg(VReg: ImageReg),
5189 HandleDef&: *ImageDef, Pos&: I)) {
5190 return false;
5191 }
5192
5193 Register CoordinateReg = I.getOperand(i: 2).getReg();
5194 Register DataReg = I.getOperand(i: 3).getReg();
5195 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
5196 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
5197 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpImageWrite))
5198 .addUse(RegNo: NewImageReg)
5199 .addUse(RegNo: CoordinateReg)
5200 .addUse(RegNo: DataReg)
5201 .constrainAllUses(TII, TRI, RBI);
5202 return true;
5203}
5204
5205Register SPIRVInstructionSelector::buildPointerToResource(
5206 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
5207 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
5208 StringRef Name, MachineIRBuilder MIRBuilder) const {
5209 const Type *ResType = GR.getTypeForSPIRVType(Ty: SpirvResType);
5210 if (ArraySize == 1) {
5211 SPIRVTypeInst PtrType =
5212 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
5213 assert(GR.getPointeeType(PtrType) == SpirvResType &&
5214 "SpirvResType did not have an explicit layout.");
5215 return GR.getOrCreateGlobalVariableWithBinding(VarType: PtrType, Set, Binding, Name,
5216 MIRBuilder);
5217 }
5218
5219 const Type *VarType = ArrayType::get(ElementType: const_cast<Type *>(ResType), NumElements: ArraySize);
5220 SPIRVTypeInst VarPointerType =
5221 GR.getOrCreateSPIRVPointerType(BaseType: VarType, MIRBuilder, SC);
5222 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
5223 VarType: VarPointerType, Set, Binding, Name, MIRBuilder);
5224
5225 SPIRVTypeInst ResPointerType =
5226 GR.getOrCreateSPIRVPointerType(BaseType: ResType, MIRBuilder, SC);
5227 Register AcReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResPointerType));
5228
5229 MIRBuilder.buildInstr(Opcode: SPIRV::OpAccessChain)
5230 .addDef(RegNo: AcReg)
5231 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResPointerType))
5232 .addUse(RegNo: VarReg)
5233 .addUse(RegNo: IndexReg);
5234
5235 return AcReg;
5236}
5237
5238bool SPIRVInstructionSelector::selectFirstBitSet16(
5239 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
5240 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
5241 Register ExtReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5242 if (!selectOpWithSrcs(ResVReg: ExtReg, ResType, I, Srcs: {I.getOperand(i: 2).getReg()},
5243 Opcode: ExtendOpcode))
5244 return false;
5245
5246 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: ExtReg, BitSetOpcode);
5247}
5248
5249bool SPIRVInstructionSelector::selectFirstBitSet32(
5250 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
5251 unsigned BitSetOpcode) const {
5252 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5253 .addDef(RegNo: ResVReg)
5254 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5255 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
5256 .addImm(Val: BitSetOpcode)
5257 .addUse(RegNo: SrcReg)
5258 .constrainAllUses(TII, TRI, RBI);
5259 return true;
5260}
5261
5262bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
5263 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
5264 unsigned BitSetOpcode, bool SwapPrimarySide) const {
5265
5266 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
5267 // requires creating a param register and return register with an invalid
5268 // vector size. If that is resolved, then this function can be used for
5269 // vectors of any component size.
5270 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
5271 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
5272
5273 MachineIRBuilder MIRBuilder(I);
5274 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
5275 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 64, MIRBuilder);
5276 SPIRVTypeInst I64x2Type =
5277 GR.getOrCreateSPIRVVectorType(BaseType: I64Type, NumElements: 2, MIRBuilder, EmitIR: false);
5278 SPIRVTypeInst Vec2ResType =
5279 GR.getOrCreateSPIRVVectorType(BaseType, NumElements: 2, MIRBuilder, EmitIR: false);
5280
5281 std::vector<Register> PartialRegs;
5282
5283 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
5284 unsigned CurrentComponent = 0;
5285 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
5286 // This register holds the firstbitX result for each of the i64x2 vectors
5287 // extracted from SrcReg
5288 Register BitSetResult =
5289 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64x2Type));
5290
5291 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
5292 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
5293 .addDef(RegNo: BitSetResult)
5294 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: I64x2Type))
5295 .addUse(RegNo: SrcReg)
5296 .addUse(RegNo: SrcReg)
5297 .addImm(Val: CurrentComponent)
5298 .addImm(Val: CurrentComponent + 1);
5299
5300 MIB.constrainAllUses(TII, TRI, RBI);
5301
5302 Register SubVecBitSetReg =
5303 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: Vec2ResType));
5304
5305 if (!selectFirstBitSet64(ResVReg: SubVecBitSetReg, ResType: Vec2ResType, I, SrcReg: BitSetResult,
5306 BitSetOpcode, SwapPrimarySide))
5307 return false;
5308
5309 PartialRegs.push_back(x: SubVecBitSetReg);
5310 }
5311
5312 // On odd component counts we need to handle one more component
5313 if (CurrentComponent != ComponentCount) {
5314 bool ZeroAsNull = !STI.isShader();
5315 Register FinalElemReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: I64Type));
5316 Register ConstIntLastIdx = GR.getOrCreateConstInt(
5317 Val: ComponentCount - 1, I, SpvType: BaseType, TII, ZeroAsNull);
5318
5319 if (!selectOpWithSrcs(ResVReg: FinalElemReg, ResType: I64Type, I, Srcs: {SrcReg, ConstIntLastIdx},
5320 Opcode: SPIRV::OpVectorExtractDynamic))
5321 return false;
5322
5323 Register FinalElemBitSetReg =
5324 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BaseType));
5325
5326 if (!selectFirstBitSet64(ResVReg: FinalElemBitSetReg, ResType: BaseType, I, SrcReg: FinalElemReg,
5327 BitSetOpcode, SwapPrimarySide))
5328 return false;
5329
5330 PartialRegs.push_back(x: FinalElemBitSetReg);
5331 }
5332
5333 // Join all the resulting registers back into the return type in order
5334 // (ie i32x2, i32x2, i32x1 -> i32x5)
5335 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: std::move(PartialRegs),
5336 Opcode: SPIRV::OpCompositeConstruct);
5337}
5338
5339bool SPIRVInstructionSelector::selectFirstBitSet64(
5340 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
5341 unsigned BitSetOpcode, bool SwapPrimarySide) const {
5342 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(Type: ResType);
5343 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(Type: ResType);
5344 bool ZeroAsNull = !STI.isShader();
5345 Register ConstIntZero =
5346 GR.getOrCreateConstInt(Val: 0, I, SpvType: BaseType, TII, ZeroAsNull);
5347 Register ConstIntOne =
5348 GR.getOrCreateConstInt(Val: 1, I, SpvType: BaseType, TII, ZeroAsNull);
5349
5350 // SPIRV doesn't support vectors with more than 4 components. Since the
5351 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
5352 // operate on vectors with 2 or less components. When largers vectors are
5353 // seen. Split them, recurse, then recombine them.
5354 if (ComponentCount > 2) {
5355 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
5356 BitSetOpcode, SwapPrimarySide);
5357 }
5358
5359 // 1. Split int64 into 2 pieces using a bitcast
5360 MachineIRBuilder MIRBuilder(I);
5361 SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType(
5362 BaseType, NumElements: 2 * ComponentCount, MIRBuilder, EmitIR: false);
5363 Register BitcastReg =
5364 MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
5365
5366 if (!selectOpWithSrcs(ResVReg: BitcastReg, ResType: PostCastType, I, Srcs: {SrcReg},
5367 Opcode: SPIRV::OpBitcast))
5368 return false;
5369
5370 // 2. Find the first set bit from the primary side for all the pieces in #1
5371 Register FBSReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PostCastType));
5372 if (!selectFirstBitSet32(ResVReg: FBSReg, ResType: PostCastType, I, SrcReg: BitcastReg, BitSetOpcode))
5373 return false;
5374
5375 // 3. Split result vector into high bits and low bits
5376 Register HighReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5377 Register LowReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5378
5379 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
5380 if (IsScalarRes) {
5381 // if scalar do a vector extract
5382 if (!selectOpWithSrcs(ResVReg: HighReg, ResType, I, Srcs: {FBSReg, ConstIntZero},
5383 Opcode: SPIRV::OpVectorExtractDynamic))
5384 return false;
5385 if (!selectOpWithSrcs(ResVReg: LowReg, ResType, I, Srcs: {FBSReg, ConstIntOne},
5386 Opcode: SPIRV::OpVectorExtractDynamic))
5387 return false;
5388 } else {
5389 // if vector do a shufflevector
5390 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
5391 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
5392 .addDef(RegNo: HighReg)
5393 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5394 .addUse(RegNo: FBSReg)
5395 // Per the spec, repeat the vector if only one vec is needed
5396 .addUse(RegNo: FBSReg);
5397
5398 // high bits are stored in even indexes. Extract them from FBSReg
5399 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
5400 MIB.addImm(Val: J);
5401 }
5402
5403 MIB.constrainAllUses(TII, TRI, RBI);
5404
5405 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
5406 MCID: TII.get(Opcode: SPIRV::OpVectorShuffle))
5407 .addDef(RegNo: LowReg)
5408 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5409 .addUse(RegNo: FBSReg)
5410 // Per the spec, repeat the vector if only one vec is needed
5411 .addUse(RegNo: FBSReg);
5412
5413 // low bits are stored in odd indexes. Extract them from FBSReg
5414 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
5415 MIB.addImm(Val: J);
5416 }
5417 MIB.constrainAllUses(TII, TRI, RBI);
5418 }
5419
5420 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
5421 // primary
5422 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
5423 Register NegOneReg;
5424 Register Reg0;
5425 Register Reg32;
5426 unsigned SelectOp;
5427 unsigned AddOp;
5428
5429 if (IsScalarRes) {
5430 NegOneReg =
5431 GR.getOrCreateConstInt(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
5432 Reg0 = GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
5433 Reg32 = GR.getOrCreateConstInt(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
5434 SelectOp = SPIRV::OpSelectSISCond;
5435 AddOp = SPIRV::OpIAddS;
5436 } else {
5437 BoolType = GR.getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: ComponentCount,
5438 MIRBuilder, EmitIR: false);
5439 NegOneReg =
5440 GR.getOrCreateConstVector(Val: (unsigned)-1, I, SpvType: ResType, TII, ZeroAsNull);
5441 Reg0 = GR.getOrCreateConstVector(Val: 0, I, SpvType: ResType, TII, ZeroAsNull);
5442 Reg32 = GR.getOrCreateConstVector(Val: 32, I, SpvType: ResType, TII, ZeroAsNull);
5443 SelectOp = SPIRV::OpSelectVIVCond;
5444 AddOp = SPIRV::OpIAddV;
5445 }
5446
5447 Register PrimaryReg = HighReg;
5448 Register SecondaryReg = LowReg;
5449 Register PrimaryShiftReg = Reg32;
5450 Register SecondaryShiftReg = Reg0;
5451
5452 // By default the emitted opcodes check for the set bit from the MSB side.
5453 // Setting SwapPrimarySide checks the set bit from the LSB side
5454 if (SwapPrimarySide) {
5455 PrimaryReg = LowReg;
5456 SecondaryReg = HighReg;
5457 PrimaryShiftReg = Reg0;
5458 SecondaryShiftReg = Reg32;
5459 }
5460
5461 // Check if the primary bits are == -1
5462 Register BReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: BoolType));
5463 if (!selectOpWithSrcs(ResVReg: BReg, ResType: BoolType, I, Srcs: {PrimaryReg, NegOneReg},
5464 Opcode: SPIRV::OpIEqual))
5465 return false;
5466
5467 // Select secondary bits if true in BReg, otherwise primary bits
5468 Register TmpReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5469 if (!selectOpWithSrcs(ResVReg: TmpReg, ResType, I, Srcs: {BReg, SecondaryReg, PrimaryReg},
5470 Opcode: SelectOp))
5471 return false;
5472
5473 // 5. Add 32 when high bits are used, otherwise 0 for low bits
5474 Register ValReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5475 if (!selectOpWithSrcs(ResVReg: ValReg, ResType, I,
5476 Srcs: {BReg, SecondaryShiftReg, PrimaryShiftReg}, Opcode: SelectOp))
5477 return false;
5478
5479 return selectOpWithSrcs(ResVReg, ResType, I, Srcs: {ValReg, TmpReg}, Opcode: AddOp);
5480}
5481
5482bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
5483 SPIRVTypeInst ResType,
5484 MachineInstr &I,
5485 bool IsSigned) const {
5486 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
5487 Register OpReg = I.getOperand(i: 2).getReg();
5488 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
5489 // zero or sign extend
5490 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5491 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
5492
5493 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
5494 case 16:
5495 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
5496 case 32:
5497 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
5498 case 64:
5499 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
5500 /*SwapPrimarySide=*/false);
5501 default:
5502 report_fatal_error(
5503 reason: "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
5504 }
5505}
5506
5507bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
5508 SPIRVTypeInst ResType,
5509 MachineInstr &I) const {
5510 // FindILsb intrinsic only supports 32 bit integers
5511 Register OpReg = I.getOperand(i: 2).getReg();
5512 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(VReg: OpReg);
5513 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
5514 // to an unsigned i32. As this leaves all the least significant bits unchanged
5515 // so the first set bit from the LSB side doesn't change.
5516 unsigned ExtendOpcode = SPIRV::OpUConvert;
5517 unsigned BitSetOpcode = GL::FindILsb;
5518
5519 switch (GR.getScalarOrVectorBitWidth(Type: OpType)) {
5520 case 16:
5521 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
5522 case 32:
5523 return selectFirstBitSet32(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode);
5524 case 64:
5525 return selectFirstBitSet64(ResVReg, ResType, I, SrcReg: OpReg, BitSetOpcode,
5526 /*SwapPrimarySide=*/true);
5527 default:
5528 report_fatal_error(reason: "spv_firstbitlow only supports 16,32,64 bits.");
5529 }
5530}
5531
5532bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
5533 SPIRVTypeInst ResType,
5534 MachineInstr &I) const {
5535 // there was an allocation size parameter to the allocation instruction
5536 // that is not 1
5537 MachineBasicBlock &BB = *I.getParent();
5538 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariableLengthArrayINTEL))
5539 .addDef(RegNo: ResVReg)
5540 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5541 .addUse(RegNo: I.getOperand(i: 2).getReg())
5542 .constrainAllUses(TII, TRI, RBI);
5543 if (!STI.isShader()) {
5544 unsigned Alignment = I.getOperand(i: 3).getImm();
5545 buildOpDecorate(Reg: ResVReg, I, TII, Dec: SPIRV::Decoration::Alignment, DecArgs: {Alignment});
5546 }
5547 return true;
5548}
5549
5550bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
5551 SPIRVTypeInst ResType,
5552 MachineInstr &I) const {
5553 // Change order of instructions if needed: all OpVariable instructions in a
5554 // function must be the first instructions in the first block
5555 auto It = getOpVariableMBBIt(I);
5556 BuildMI(BB&: *It->getParent(), I: It, MIMD: It->getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
5557 .addDef(RegNo: ResVReg)
5558 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5559 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function))
5560 .constrainAllUses(TII, TRI, RBI);
5561 if (!STI.isShader()) {
5562 unsigned Alignment = I.getOperand(i: 2).getImm();
5563 buildOpDecorate(Reg: ResVReg, I&: *It, TII, Dec: SPIRV::Decoration::Alignment,
5564 DecArgs: {Alignment});
5565 }
5566 return true;
5567}
5568
5569bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
5570 // InstructionSelector walks backwards through the instructions. We can use
5571 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
5572 // first, so can generate an OpBranchConditional here. If there is no
5573 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
5574 const MachineInstr *PrevI = I.getPrevNode();
5575 MachineBasicBlock &MBB = *I.getParent();
5576 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
5577 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
5578 .addUse(RegNo: PrevI->getOperand(i: 0).getReg())
5579 .addMBB(MBB: PrevI->getOperand(i: 1).getMBB())
5580 .addMBB(MBB: I.getOperand(i: 0).getMBB())
5581 .constrainAllUses(TII, TRI, RBI);
5582 return true;
5583 }
5584 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranch))
5585 .addMBB(MBB: I.getOperand(i: 0).getMBB())
5586 .constrainAllUses(TII, TRI, RBI);
5587 return true;
5588}
5589
5590bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
5591 // InstructionSelector walks backwards through the instructions. For an
5592 // explicit conditional branch with no fallthrough, we use both a G_BR and a
5593 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
5594 // generate the OpBranchConditional in selectBranch above.
5595 //
5596 // If an OpBranchConditional has been generated, we simply return, as the work
5597 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
5598 // implicit fallthrough to the next basic block, so we need to create an
5599 // OpBranchConditional with an explicit "false" argument pointing to the next
5600 // basic block that LLVM would fall through to.
5601 const MachineInstr *NextI = I.getNextNode();
5602 // Check if this has already been successfully selected.
5603 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
5604 return true;
5605 // Must be relying on implicit block fallthrough, so generate an
5606 // OpBranchConditional with the "next" basic block as the "false" target.
5607 MachineBasicBlock &MBB = *I.getParent();
5608 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
5609 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(N: NextMBBNum);
5610 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpBranchConditional))
5611 .addUse(RegNo: I.getOperand(i: 0).getReg())
5612 .addMBB(MBB: I.getOperand(i: 1).getMBB())
5613 .addMBB(MBB: NextMBB)
5614 .constrainAllUses(TII, TRI, RBI);
5615 return true;
5616}
5617
5618bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
5619 MachineInstr &I) const {
5620 auto MIB =
5621 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::PHI))
5622 .addDef(RegNo: ResVReg);
5623 const unsigned NumOps = I.getNumOperands();
5624 for (unsigned i = 1; i < NumOps; i += 2) {
5625 MIB.addUse(RegNo: I.getOperand(i: i + 0).getReg());
5626 MIB.addMBB(MBB: I.getOperand(i: i + 1).getMBB());
5627 }
5628 MIB.constrainAllUses(TII, TRI, RBI);
5629 return true;
5630}
5631
5632bool SPIRVInstructionSelector::selectGlobalValue(
5633 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
5634 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
5635 MachineIRBuilder MIRBuilder(I);
5636 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
5637 Type *GVType = toTypedPointer(Ty: GR.getDeducedGlobalValueType(Global: GV));
5638
5639 std::string GlobalIdent;
5640 if (!GV->hasName()) {
5641 unsigned &ID = UnnamedGlobalIDs[GV];
5642 if (ID == 0)
5643 ID = UnnamedGlobalIDs.size();
5644 GlobalIdent = "__unnamed_" + Twine(ID).str();
5645 } else {
5646 GlobalIdent = GV->getName();
5647 }
5648
5649 // Behaviour of functions as operands depends on availability of the
5650 // corresponding extension (SPV_INTEL_function_pointers):
5651 // - If there is an extension to operate with functions as operands:
5652 // We create a proper constant operand and evaluate a correct type for a
5653 // function pointer.
5654 // - Without the required extension:
5655 // We have functions as operands in tests with blocks of instruction e.g. in
5656 // transcoding/global_block.ll. These operands are not used and should be
5657 // substituted by zero constants. Their type is expected to be always
5658 // OpTypePointer Function %uchar.
5659 if (isa<Function>(Val: GV)) {
5660 const Constant *ConstVal = GV;
5661 MachineBasicBlock &BB = *I.getParent();
5662 Register NewReg = GR.find(V: ConstVal, MF: GR.CurMF);
5663 if (!NewReg.isValid()) {
5664 const Function *GVFun =
5665 STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
5666 ? dyn_cast<Function>(Val: GV)
5667 : nullptr;
5668 SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType(
5669 BaseType: GVType, I,
5670 SC: GVFun ? SPIRV::StorageClass::CodeSectionINTEL
5671 : addressSpaceToStorageClass(AddrSpace: GV->getAddressSpace(), STI));
5672 if (GVFun) {
5673 // References to a function via function pointers generate virtual
5674 // registers without a definition. We will resolve it later, during
5675 // module analysis stage.
5676 Register ResTypeReg = GR.getSPIRVTypeID(SpirvType: ResType);
5677 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5678 Register FuncVReg =
5679 MRI->createGenericVirtualRegister(Ty: GR.getRegType(SpvType: ResType));
5680 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::pIDRegClass);
5681 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: FuncVReg, MF: *GR.CurMF);
5682 MachineInstrBuilder MIB1 =
5683 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpUndef))
5684 .addDef(RegNo: FuncVReg)
5685 .addUse(RegNo: ResTypeReg);
5686 MachineInstrBuilder MIB2 =
5687 BuildMI(BB, I, MIMD: I.getDebugLoc(),
5688 MCID: TII.get(Opcode: SPIRV::OpConstantFunctionPointerINTEL))
5689 .addDef(RegNo: ResVReg)
5690 .addUse(RegNo: ResTypeReg)
5691 .addUse(RegNo: FuncVReg);
5692 GR.add(V: ConstVal, MI: MIB2);
5693 // mapping the function pointer to the used Function
5694 GR.recordFunctionPointer(MO: &MIB2.getInstr()->getOperand(i: 2), F: GVFun);
5695 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF: *GR.CurMF);
5696 MIB1.constrainAllUses(TII, TRI, RBI);
5697 MIB2.constrainAllUses(TII, TRI, RBI);
5698 return true;
5699 }
5700 MachineInstrBuilder MIB3 =
5701 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpConstantNull))
5702 .addDef(RegNo: ResVReg)
5703 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType));
5704 GR.add(V: ConstVal, MI: MIB3);
5705 MIB3.constrainAllUses(TII, TRI, RBI);
5706 return true;
5707 }
5708 assert(NewReg != ResVReg);
5709 return BuildCOPY(DestReg: ResVReg, SrcReg: NewReg, I);
5710 }
5711 auto GlobalVar = cast<GlobalVariable>(Val: GV);
5712 assert(GlobalVar->getName() != "llvm.global.annotations");
5713
5714 // Skip empty declaration for GVs with initializers till we get the decl with
5715 // passed initializer.
5716 if (hasInitializer(GV: GlobalVar) && !Init)
5717 return true;
5718
5719 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
5720 getSpirvLinkageTypeFor(ST: STI, GV: *GV);
5721
5722 const unsigned AddrSpace = GV->getAddressSpace();
5723 SPIRV::StorageClass::StorageClass StorageClass =
5724 addressSpaceToStorageClass(AddrSpace, STI);
5725 SPIRVTypeInst ResType =
5726 GR.getOrCreateSPIRVPointerType(BaseType: GVType, I, SC: StorageClass);
5727 Register Reg = GR.buildGlobalVariable(
5728 Reg: ResVReg, BaseType: ResType, Name: GlobalIdent, GV, Storage: StorageClass, Init,
5729 IsConst: GlobalVar->isConstant(), LinkageType: LnkType, MIRBuilder, IsInstSelector: true);
5730 // TODO: For AMDGCN, we pipe externally_initialized through via
5731 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
5732 // reverse translation. We should remove this once SPIR-V gains the ability to
5733 // express the concept.
5734 if (GlobalVar->isExternallyInitialized() &&
5735 STI.getTargetTriple().getVendor() == Triple::AMD) {
5736 constexpr unsigned ReadWriteINTEL = 3u;
5737 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::HostAccessINTEL,
5738 DecArgs: {ReadWriteINTEL});
5739 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
5740 addStringImm(Str: GV->getName(), MIB);
5741 }
5742 return Reg.isValid();
5743}
5744
5745bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
5746 SPIRVTypeInst ResType,
5747 MachineInstr &I) const {
5748 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5749 return selectExtInst(ResVReg, ResType, I, CLInst: CL::log10);
5750 }
5751
5752 // There is no log10 instruction in the GLSL Extended Instruction set, so it
5753 // is implemented as:
5754 // log10(x) = log2(x) * (1 / log2(10))
5755 // = log2(x) * 0.30103
5756
5757 MachineIRBuilder MIRBuilder(I);
5758 MachineBasicBlock &BB = *I.getParent();
5759
5760 // Build log2(x).
5761 Register VarReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5762 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5763 .addDef(RegNo: VarReg)
5764 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5765 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
5766 .addImm(Val: GL::Log2)
5767 .add(MO: I.getOperand(i: 1))
5768 .constrainAllUses(TII, TRI, RBI);
5769
5770 // Build 0.30103.
5771 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
5772 ResType->getOpcode() == SPIRV::OpTypeFloat);
5773 // TODO: Add matrix implementation once supported by the HLSL frontend.
5774 SPIRVTypeInst SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector
5775 ? SPIRVTypeInst(GR.getSPIRVTypeForVReg(
5776 VReg: ResType->getOperand(i: 1).getReg()))
5777 : ResType;
5778 Register ScaleReg =
5779 GR.buildConstantFP(Val: APFloat(0.30103f), MIRBuilder, SpvType: SpirvScalarType);
5780
5781 // Multiply log2(x) by 0.30103 to get log10(x) result.
5782 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
5783 ? SPIRV::OpVectorTimesScalar
5784 : SPIRV::OpFMulS;
5785 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode))
5786 .addDef(RegNo: ResVReg)
5787 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5788 .addUse(RegNo: VarReg)
5789 .addUse(RegNo: ScaleReg)
5790 .constrainAllUses(TII, TRI, RBI);
5791 return true;
5792}
5793
5794bool SPIRVInstructionSelector::selectFpowi(Register ResVReg,
5795 SPIRVTypeInst ResType,
5796 MachineInstr &I) const {
5797 // On OpenCL targets, pown(gentype x, intn n) maps directly.
5798 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std))
5799 return selectExtInst(ResVReg, ResType, I, CLInst: CL::pown);
5800
5801 // On GLSL (Vulkan) targets, there is no integer-exponent power instruction.
5802 // Lower as: Pow(base, OpConvertSToF(exp)).
5803 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
5804 Register BaseReg = I.getOperand(i: 1).getReg();
5805 Register ExpReg = I.getOperand(i: 2).getReg();
5806 Register FloatExpReg = MRI->createVirtualRegister(RegClass: GR.getRegClass(SpvType: ResType));
5807 if (!selectOpWithSrcs(ResVReg: FloatExpReg, ResType, I, Srcs: {ExpReg},
5808 Opcode: SPIRV::OpConvertSToF))
5809 return false;
5810 return selectExtInst(ResVReg, ResType, I, GLInst: GL::Pow,
5811 /*setMIFlags=*/true, /*useMISrc=*/false,
5812 SrcRegs: {BaseReg, FloatExpReg});
5813 }
5814 return false;
5815}
5816
5817bool SPIRVInstructionSelector::selectModf(Register ResVReg,
5818 SPIRVTypeInst ResType,
5819 MachineInstr &I) const {
5820 // llvm.modf has a single arg --the number to be decomposed-- and returns a
5821 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
5822 // number to be decomposed and a pointer--, returns the fractional part and
5823 // the integral part is stored in the pointer argument. Therefore, we can't
5824 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
5825 // scaffolding to make it work. The idea is to create an alloca instruction
5826 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
5827 // from this ptr to place it in the struct. llvm.modf returns the fractional
5828 // part as the first element of the result, and the integral part as the
5829 // second element of the result.
5830
5831 // At this point, the return type is not a struct anymore, but rather two
5832 // independent elements of SPIRVResType. We can get each independent element
5833 // from I.getDefs() or I.getOperands().
5834 if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
5835 MachineIRBuilder MIRBuilder(I);
5836 // Get pointer type for alloca variable.
5837 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5838 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Function);
5839 // Create new register for the pointer type of alloca variable.
5840 Register PtrTyReg =
5841 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5842 MIRBuilder.getMRI()->setType(
5843 VReg: PtrTyReg,
5844 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Function),
5845 SizeInBits: GR.getPointerSize()));
5846
5847 // Assign SPIR-V type of the pointer type of the alloca variable to the
5848 // new register.
5849 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: PtrTyReg, MF: MIRBuilder.getMF());
5850 MachineBasicBlock &EntryBB = I.getMF()->front();
5851 MachineBasicBlock::iterator VarPos =
5852 getFirstValidInstructionInsertPoint(BB&: EntryBB);
5853 auto AllocaMIB =
5854 BuildMI(BB&: EntryBB, I: VarPos, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpVariable))
5855 .addDef(RegNo: PtrTyReg)
5856 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: PtrType))
5857 .addImm(Val: static_cast<uint32_t>(SPIRV::StorageClass::Function));
5858 Register Variable = AllocaMIB->getOperand(i: 0).getReg();
5859
5860 MachineBasicBlock &BB = *I.getParent();
5861 // Create the OpenCLLIB::modf instruction.
5862 auto MIB =
5863 BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpExtInst))
5864 .addDef(RegNo: ResVReg)
5865 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5866 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
5867 .addImm(Val: CL::modf)
5868 .setMIFlags(I.getFlags())
5869 .add(MO: I.getOperand(i: I.getNumExplicitDefs())) // Floating point value.
5870 .addUse(RegNo: Variable); // Pointer to integral part.
5871 // Assign the integral part stored in the ptr to the second element of the
5872 // result.
5873 Register IntegralPartReg = I.getOperand(i: 1).getReg();
5874 if (IntegralPartReg.isValid()) {
5875 // Load the value from the pointer to integral part.
5876 auto LoadMIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5877 .addDef(RegNo: IntegralPartReg)
5878 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5879 .addUse(RegNo: Variable);
5880 LoadMIB.constrainAllUses(TII, TRI, RBI);
5881 return true;
5882 }
5883
5884 MIB.constrainAllUses(TII, TRI, RBI);
5885 return true;
5886 } else if (STI.canUseExtInstSet(E: SPIRV::InstructionSet::GLSL_std_450)) {
5887 assert(false && "GLSL::Modf is deprecated.");
5888 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
5889 return false;
5890 }
5891 return false;
5892}
5893
5894// Generate the instructions to load 3-element vector builtin input
5895// IDs/Indices.
5896// Like: GlobalInvocationId, LocalInvocationId, etc....
5897
5898bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
5899 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5900 SPIRVTypeInst ResType, MachineInstr &I) const {
5901 MachineIRBuilder MIRBuilder(I);
5902 const SPIRVTypeInst Vec3Ty =
5903 GR.getOrCreateSPIRVVectorType(BaseType: ResType, NumElements: 3, MIRBuilder, EmitIR: false);
5904 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5905 BaseType: Vec3Ty, MIRBuilder, SC: SPIRV::StorageClass::Input);
5906
5907 // Create new register for the input ID builtin variable.
5908 Register NewRegister =
5909 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5910 MIRBuilder.getMRI()->setType(VReg: NewRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5911 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5912
5913 // Build global variable with the necessary decorations for the input ID
5914 // builtin variable.
5915 Register Variable = GR.buildGlobalVariable(
5916 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5917 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5918 IsInstSelector: false);
5919
5920 // Create new register for loading value.
5921 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5922 Register LoadedRegister = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
5923 MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
5924 GR.assignSPIRVTypeToVReg(Type: Vec3Ty, VReg: LoadedRegister, MF: MIRBuilder.getMF());
5925
5926 // Load v3uint value from the global variable.
5927 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5928 .addDef(RegNo: LoadedRegister)
5929 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: Vec3Ty))
5930 .addUse(RegNo: Variable);
5931
5932 // Get the input ID index. Expecting operand is a constant immediate value,
5933 // wrapped in a type assignment.
5934 assert(I.getOperand(2).isReg());
5935 const uint32_t ThreadId = foldImm(MO: I.getOperand(i: 2), MRI);
5936
5937 // Extract the input ID from the loaded vector value.
5938 MachineBasicBlock &BB = *I.getParent();
5939 auto MIB = BuildMI(BB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpCompositeExtract))
5940 .addDef(RegNo: ResVReg)
5941 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5942 .addUse(RegNo: LoadedRegister)
5943 .addImm(Val: ThreadId);
5944 MIB.constrainAllUses(TII, TRI, RBI);
5945 return true;
5946}
5947
5948// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
5949// Like LocalInvocationIndex
5950bool SPIRVInstructionSelector::loadBuiltinInputID(
5951 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
5952 SPIRVTypeInst ResType, MachineInstr &I) const {
5953 MachineIRBuilder MIRBuilder(I);
5954 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
5955 BaseType: ResType, MIRBuilder, SC: SPIRV::StorageClass::Input);
5956
5957 // Create new register for the input ID builtin variable.
5958 Register NewRegister =
5959 MIRBuilder.getMRI()->createVirtualRegister(RegClass: GR.getRegClass(SpvType: PtrType));
5960 MIRBuilder.getMRI()->setType(
5961 VReg: NewRegister,
5962 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Input),
5963 SizeInBits: GR.getPointerSize()));
5964 GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
5965
5966 // Build global variable with the necessary decorations for the input ID
5967 // builtin variable.
5968 Register Variable = GR.buildGlobalVariable(
5969 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue), GV: nullptr,
5970 Storage: SPIRV::StorageClass::Input, Init: nullptr, IsConst: true, LinkageType: std::nullopt, MIRBuilder,
5971 IsInstSelector: false);
5972
5973 // Load uint value from the global variable.
5974 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpLoad))
5975 .addDef(RegNo: ResVReg)
5976 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
5977 .addUse(RegNo: Variable);
5978
5979 MIB.constrainAllUses(TII, TRI, RBI);
5980 return true;
5981}
5982
5983SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type,
5984 MachineInstr &I) const {
5985 MachineIRBuilder MIRBuilder(I);
5986 if (Type->getOpcode() != SPIRV::OpTypeVector)
5987 return GR.getOrCreateSPIRVVectorType(BaseType: Type, NumElements: 4, MIRBuilder, EmitIR: false);
5988
5989 uint64_t VectorSize = Type->getOperand(i: 2).getImm();
5990 if (VectorSize == 4)
5991 return Type;
5992
5993 Register ScalarTypeReg = Type->getOperand(i: 1).getReg();
5994 const SPIRVTypeInst ScalarType = GR.getSPIRVTypeForVReg(VReg: ScalarTypeReg);
5995 return GR.getOrCreateSPIRVVectorType(BaseType: ScalarType, NumElements: 4, MIRBuilder, EmitIR: false);
5996}
5997
5998bool SPIRVInstructionSelector::loadHandleBeforePosition(
5999 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
6000 MachineInstr &Pos) const {
6001
6002 assert(HandleDef.getIntrinsicID() ==
6003 Intrinsic::spv_resource_handlefrombinding);
6004 uint32_t Set = foldImm(MO: HandleDef.getOperand(i: 2), MRI);
6005 uint32_t Binding = foldImm(MO: HandleDef.getOperand(i: 3), MRI);
6006 uint32_t ArraySize = foldImm(MO: HandleDef.getOperand(i: 4), MRI);
6007 Register IndexReg = HandleDef.getOperand(i: 5).getReg();
6008 std::string Name =
6009 getStringValueFromReg(Reg: HandleDef.getOperand(i: 6).getReg(), MRI&: *MRI);
6010
6011 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
6012 MachineIRBuilder MIRBuilder(HandleDef);
6013 SPIRVTypeInst VarType = ResType;
6014 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
6015
6016 if (IsStructuredBuffer) {
6017 VarType = GR.getPointeeType(PtrType: ResType);
6018 SC = GR.getPointerStorageClass(Type: ResType);
6019 }
6020
6021 if (ResType->getOpcode() == SPIRV::OpTypeImage && ArraySize == 0)
6022 MIRBuilder.buildInstr(Opcode: SPIRV::OpCapability)
6023 .addImm(Val: SPIRV::Capability::RuntimeDescriptorArrayEXT);
6024
6025 Register VarReg =
6026 buildPointerToResource(SpirvResType: SPIRVTypeInst(VarType), SC, Set, Binding,
6027 ArraySize, IndexReg, Name, MIRBuilder);
6028
6029 // The handle for the buffer is the pointer to the resource. For an image, the
6030 // handle is the image object. So images get an extra load.
6031 uint32_t LoadOpcode =
6032 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
6033 GR.assignSPIRVTypeToVReg(Type: ResType, VReg: HandleReg, MF: *Pos.getMF());
6034 BuildMI(BB&: *Pos.getParent(), I&: Pos, MIMD: HandleDef.getDebugLoc(), MCID: TII.get(Opcode: LoadOpcode))
6035 .addDef(RegNo: HandleReg)
6036 .addUse(RegNo: GR.getSPIRVTypeID(SpirvType: ResType))
6037 .addUse(RegNo: VarReg)
6038 .constrainAllUses(TII, TRI, RBI);
6039 return true;
6040}
6041
6042void SPIRVInstructionSelector::errorIfInstrOutsideShader(
6043 MachineInstr &I) const {
6044 if (!STI.isShader()) {
6045 std::string DiagMsg;
6046 raw_string_ostream OS(DiagMsg);
6047 I.print(OS, IsStandalone: true, SkipOpers: false, SkipDebugLoc: false, AddNewLine: false);
6048 DiagMsg += " is only supported in shaders.\n";
6049 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
6050 }
6051}
6052
6053namespace llvm {
6054InstructionSelector *
6055createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
6056 const SPIRVSubtarget &Subtarget,
6057 const RegisterBankInfo &RBI) {
6058 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
6059}
6060} // namespace llvm
6061