1//===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVUtils.h"
14#include "MCTargetDesc/SPIRVBaseInfo.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRVInstrInfo.h"
19#include "SPIRVSubtarget.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/Demangle/Demangle.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/TargetParser/AtomicScope.h"
31#include <queue>
32#include <vector>
33
34namespace llvm {
35namespace SPIRV {
36static MDNode *findNamedMDOperand(NamedMDNode *NMD, StringRef Name) {
37 auto It = find_if(Range: NMD->operands(), P: [Name](MDNode *N) {
38 if (auto *MDS = dyn_cast_or_null<MDString>(Val: N->getOperand(I: 0)))
39 return MDS->getString() == Name;
40 return false;
41 });
42 return It == NMD->op_end() ? nullptr : *It;
43}
44
45// This code restores function args/retvalue types for composite cases
46// because the final types should still be aggregate whereas they're i32
47// during the translation to cope with aggregate flattening etc.
48// TODO: should these just return nullptr when there's no metadata?
49static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
50 FunctionType *FTy,
51 StringRef Name) {
52 if (!NMD)
53 return FTy;
54
55 MDNode *Match = findNamedMDOperand(NMD, Name);
56 if (!Match)
57 return FTy;
58
59 Type *RetTy = FTy->getReturnType();
60 SmallVector<Type *, 4> PTys(FTy->params());
61
62 for (unsigned I = 1; I != Match->getNumOperands(); ++I) {
63 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I));
64 assert(MD && "MDNode operand is expected");
65
66 if (auto *Const = getMDOperandAsConstInt(N: MD, I: 0)) {
67 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1));
68 assert(CMeta && "ConstantAsMetadata operand is expected");
69 int64_t Idx = Const->getSExtValue();
70 // Currently -1 indicates return value, greater values mean
71 // argument numbers.
72 if (Idx == -1) {
73 RetTy = CMeta->getType();
74 continue;
75 }
76 if (Idx >= 0 && static_cast<uint64_t>(Idx) < PTys.size()) {
77 PTys[Idx] = CMeta->getType();
78 continue;
79 }
80 report_fatal_error(reason: "invalid argument index in function type metadata");
81 }
82 }
83
84 return FunctionType::get(Result: RetTy, Params: PTys, isVarArg: FTy->isVarArg());
85}
86
87static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
88 StringRef Constraints,
89 StringRef Name) {
90 if (!NMD)
91 return Constraints;
92
93 MDNode *Match = findNamedMDOperand(NMD, Name);
94 if (!Match)
95 return Constraints;
96
97 // By convention, the constraints string is stored in the final MD operand.
98 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I: Match->getNumOperands() - 1));
99 assert(MD && "MDNode operand is expected");
100
101 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
102 Constraints = MDS->getString();
103
104 return Constraints;
105}
106
107FunctionType *getOriginalFunctionType(const Function &F) {
108 return extractFunctionTypeFromMetadata(
109 NMD: F.getParent()->getNamedMetadata(Name: "spv.cloned_funcs"), FTy: F.getFunctionType(),
110 Name: F.getName());
111}
112
113// Keyed via instruction metadata, not a name.
114static std::optional<StringRef> getMutatedCallsiteKey(const CallBase &CB) {
115 if (MDNode *MD = CB.getMetadata(Kind: "spv.mutated_callsite"))
116 if (MD->getNumOperands() > 0)
117 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
118 return MDS->getString();
119 return std::nullopt;
120}
121
122FunctionType *getOriginalFunctionType(const CallBase &CB) {
123 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
124 if (!Key)
125 return CB.getFunctionType();
126 return extractFunctionTypeFromMetadata(
127 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"),
128 FTy: CB.getFunctionType(), Name: *Key);
129}
130
131StringRef getOriginalAsmConstraints(const CallBase &CB) {
132 StringRef Constraints =
133 cast<InlineAsm>(Val: CB.getCalledOperand())->getConstraintString();
134 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
135 if (!Key)
136 return Constraints;
137 return extractAsmConstraintsFromMetadata(
138 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"), Constraints,
139 Name: *Key);
140}
141} // Namespace SPIRV
142
143// The following functions are used to add these string literals as a series of
144// 32-bit integer operands with the correct format, and unpack them if necessary
145// when making string comparisons in compiler passes.
146// SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment.
147static uint32_t convertCharsToWord(StringRef Str, unsigned i) {
148 uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars.
149 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {
150 unsigned StrIndex = i + WordIndex;
151 uint8_t CharToAdd = 0; // Initilize char as padding/null.
152 if (StrIndex < Str.size()) { // If it's within the string, get a real char.
153 CharToAdd = Str[StrIndex];
154 }
155 Word |= (CharToAdd << (WordIndex * 8));
156 }
157 return Word;
158}
159
160// Get length including padding and null terminator.
161static size_t getPaddedLen(StringRef Str) { return alignTo(Value: Str.size() + 1, Align: 4); }
162
163void addStringImm(StringRef Str, MCInst &Inst) {
164 const size_t PaddedLen = getPaddedLen(Str);
165 for (unsigned i = 0; i < PaddedLen; i += 4) {
166 // Add an operand for the 32-bits of chars or padding.
167 Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i)));
168 }
169}
170
171void addStringImm(StringRef Str, MachineInstrBuilder &MIB) {
172 const size_t PaddedLen = getPaddedLen(Str);
173 for (unsigned i = 0; i < PaddedLen; i += 4) {
174 // Add an operand for the 32-bits of chars or padding.
175 MIB.addImm(Val: convertCharsToWord(Str, i));
176 }
177}
178
179std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
180 return getSPIRVStringOperand(MI, StartIndex);
181}
182
183std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
184 MachineInstr *Def = getVRegDef(MRI, Reg);
185 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
186 "Expected G_GLOBAL_VALUE");
187 const GlobalValue *GV = Def->getOperand(i: 1).getGlobal();
188 Value *V = GV->getOperand(i: 0);
189 const ConstantDataArray *CDA = cast<ConstantDataArray>(Val: V);
190 return CDA->getAsCString().str();
191}
192
193void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
194 const auto Bitwidth = Imm.getBitWidth();
195 if (Bitwidth == 1)
196 return; // Already handled
197 else if (Bitwidth <= 32) {
198 MIB.addImm(Val: Imm.getZExtValue());
199 // Asm Printer needs this info to print floating-type correctly
200 if (Bitwidth == 16)
201 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16);
202 return;
203 } else if (Bitwidth <= 64) {
204 uint64_t FullImm = Imm.getZExtValue();
205 MIB.addImm(Val: Lo_32(Value: FullImm)).addImm(Val: Hi_32(Value: FullImm));
206 // Asm Printer needs this info to print 64-bit operands correctly
207 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH64);
208 return;
209 } else {
210 // Emit ceil(Bitwidth / 32) words to conform SPIR-V spec.
211 unsigned NumWords = divideCeil(Numerator: Bitwidth, Denominator: 32);
212 for (unsigned I = 0; I < NumWords; ++I) {
213 unsigned LimbIdx = I / 2;
214 unsigned LimbShift = (I % 2) * 32;
215 uint32_t Word = (Imm.getRawData()[LimbIdx] >> LimbShift) & 0xffffffff;
216 MIB.addImm(Val: Word);
217 }
218 return;
219 }
220}
221
222void buildOpName(Register Target, StringRef Name,
223 MachineIRBuilder &MIRBuilder) {
224 if (!Name.empty()) {
225 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpName).addUse(RegNo: Target);
226 addStringImm(Str: Name, MIB);
227 }
228}
229
230void buildOpName(Register Target, StringRef Name, MachineInstr &I,
231 const SPIRVInstrInfo &TII) {
232 if (!Name.empty()) {
233 auto MIB =
234 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName))
235 .addUse(RegNo: Target);
236 addStringImm(Str: Name, MIB);
237 }
238}
239
240static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
241 ArrayRef<uint32_t> DecArgs,
242 StringRef StrImm) {
243 if (!StrImm.empty())
244 addStringImm(Str: StrImm, MIB);
245 for (const auto &DecArg : DecArgs)
246 MIB.addImm(Val: DecArg);
247}
248
249void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
250 SPIRV::Decoration::Decoration Dec,
251 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
252 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
253 .addUse(RegNo: Reg)
254 .addImm(Val: static_cast<uint32_t>(Dec));
255 finishBuildOpDecorate(MIB, DecArgs, StrImm);
256}
257
258void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
259 SPIRV::Decoration::Decoration Dec,
260 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
261 MachineBasicBlock &MBB = *I.getParent();
262 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDecorate))
263 .addUse(RegNo: Reg)
264 .addImm(Val: static_cast<uint32_t>(Dec));
265 finishBuildOpDecorate(MIB, DecArgs, StrImm);
266}
267
268void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
269 SPIRV::Decoration::Decoration Dec, uint32_t Member,
270 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
271 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpMemberDecorate)
272 .addUse(RegNo: Reg)
273 .addImm(Val: Member)
274 .addImm(Val: static_cast<uint32_t>(Dec));
275 finishBuildOpDecorate(MIB, DecArgs, StrImm);
276}
277
278void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
279 const MDNode *GVarMD, const SPIRVSubtarget &ST) {
280 for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
281 auto *OpMD = dyn_cast<MDNode>(Val: GVarMD->getOperand(I));
282 if (!OpMD)
283 report_fatal_error(reason: "Invalid decoration");
284 if (OpMD->getNumOperands() == 0)
285 report_fatal_error(reason: "Expect operand(s) of the decoration");
286 ConstantInt *DecorationId =
287 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 0));
288 if (!DecorationId)
289 report_fatal_error(reason: "Expect SPIR-V <Decoration> operand to be the first "
290 "element of the decoration");
291
292 // The goal of `spirv.Decorations` metadata is to provide a way to
293 // represent SPIR-V entities that do not map to LLVM in an obvious way.
294 // FP flags do have obvious matches between LLVM IR and SPIR-V.
295 // Additionally, we have no guarantee at this point that the flags passed
296 // through the decoration are not violated already in the optimizer passes.
297 // Therefore, we simply ignore FP flags, including NoContraction, and
298 // FPFastMathMode.
299 if (DecorationId->getZExtValue() ==
300 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||
301 DecorationId->getZExtValue() ==
302 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {
303 continue; // Ignored.
304 }
305 uint32_t Dec = static_cast<uint32_t>(DecorationId->getZExtValue());
306 if (Dec == static_cast<uint32_t>(SPIRV::Decoration::UniformId)) {
307 ConstantInt *ScopeV =
308 OpMD->getNumOperands() == 2
309 ? mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 1))
310 : nullptr;
311 assert(ScopeV && isUInt<32>(ScopeV->getZExtValue()) &&
312 "Expect Scope <id> operand of the UniformId decoration");
313 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
314 SPIRVTypeInst SpvTypeInt32 =
315 GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
316 Register ScopeReg = GR->buildConstantInt(
317 Val: ScopeV->getZExtValue(), MIRBuilder, SpvType: SpvTypeInt32, /*EmitIR=*/false);
318 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorateId)
319 .addUse(RegNo: Reg)
320 .addImm(Val: Dec)
321 .addUse(RegNo: ScopeReg);
322 continue;
323 }
324 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate).addUse(RegNo: Reg).addImm(Val: Dec);
325 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
326 if (ConstantInt *OpV =
327 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: OpI)))
328 MIB.addImm(Val: static_cast<uint32_t>(OpV->getZExtValue()));
329 else if (MDString *OpV = dyn_cast<MDString>(Val: OpMD->getOperand(I: OpI)))
330 addStringImm(Str: OpV->getString(), MIB);
331 else
332 report_fatal_error(reason: "Unexpected operand of the decoration");
333 }
334 }
335}
336
337MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF) {
338 MachineBasicBlock &MBB = MF.front();
339 // Find the position to insert the OpVariable instruction.
340 // We will insert it after the last OpFunctionParameter, if any, or
341 // after OpFunction otherwise.
342 auto IsPreamble = [](const MachineInstr &MI) {
343 switch (MI.getOpcode()) {
344 case SPIRV::OpFunction:
345 case SPIRV::OpFunctionParameter:
346 case SPIRV::OpLabel:
347 case SPIRV::ASSIGN_TYPE:
348 return true;
349 default:
350 return false;
351 }
352 };
353 MachineBasicBlock::iterator VarPos = MBB.SkipPHIsAndLabels(I: MBB.begin());
354 while (VarPos != MBB.end() && VarPos->getOpcode() != SPIRV::OpFunction)
355 ++VarPos;
356 // Advance past the preamble.
357 while (VarPos != MBB.end() && IsPreamble(*VarPos))
358 ++VarPos;
359 return VarPos;
360}
361
362MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB) {
363 MachineBasicBlock::iterator I = MBB->end();
364 if (I == MBB->begin())
365 return I;
366 --I;
367 while (I->isTerminator() || I->isDebugValue()) {
368 if (I == MBB->begin())
369 break;
370 --I;
371 }
372 return I;
373}
374
375SPIRV::StorageClass::StorageClass
376addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
377 switch (AddrSpace) {
378 case 0:
379 return SPIRV::StorageClass::Function;
380 case 1:
381 return SPIRV::StorageClass::CrossWorkgroup;
382 case 2:
383 return SPIRV::StorageClass::UniformConstant;
384 case 3:
385 return SPIRV::StorageClass::Workgroup;
386 case 4:
387 return SPIRV::StorageClass::Generic;
388 case 5:
389 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
390 ? SPIRV::StorageClass::DeviceOnlyINTEL
391 : SPIRV::StorageClass::CrossWorkgroup;
392 case 6:
393 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
394 ? SPIRV::StorageClass::HostOnlyINTEL
395 : SPIRV::StorageClass::CrossWorkgroup;
396 case 7:
397 return SPIRV::StorageClass::Input;
398 case 8:
399 return SPIRV::StorageClass::Output;
400 case 9:
401 return SPIRV::StorageClass::CodeSectionINTEL;
402 case 10:
403 return SPIRV::StorageClass::Private;
404 case 11:
405 return SPIRV::StorageClass::StorageBuffer;
406 case 12:
407 return SPIRV::StorageClass::Uniform;
408 case 13:
409 return SPIRV::StorageClass::PushConstant;
410 default:
411 report_fatal_error(reason: "Unknown address space");
412 }
413}
414
415SPIRV::MemorySemantics::MemorySemantics
416getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) {
417 switch (SC) {
418 case SPIRV::StorageClass::StorageBuffer:
419 case SPIRV::StorageClass::Uniform:
420 return SPIRV::MemorySemantics::UniformMemory;
421 case SPIRV::StorageClass::Workgroup:
422 return SPIRV::MemorySemantics::WorkgroupMemory;
423 case SPIRV::StorageClass::CrossWorkgroup:
424 return SPIRV::MemorySemantics::CrossWorkgroupMemory;
425 case SPIRV::StorageClass::AtomicCounter:
426 return SPIRV::MemorySemantics::AtomicCounterMemory;
427 case SPIRV::StorageClass::Image:
428 return SPIRV::MemorySemantics::ImageMemory;
429 default:
430 return SPIRV::MemorySemantics::None;
431 }
432}
433
434SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
435 switch (Ord) {
436 case AtomicOrdering::Acquire:
437 return SPIRV::MemorySemantics::Acquire;
438 case AtomicOrdering::Release:
439 return SPIRV::MemorySemantics::Release;
440 case AtomicOrdering::AcquireRelease:
441 return SPIRV::MemorySemantics::AcquireRelease;
442 case AtomicOrdering::SequentiallyConsistent:
443 return SPIRV::MemorySemantics::SequentiallyConsistent;
444 case AtomicOrdering::Unordered:
445 case AtomicOrdering::Monotonic:
446 case AtomicOrdering::NotAtomic:
447 return SPIRV::MemorySemantics::None;
448 }
449 llvm_unreachable(nullptr);
450}
451
452uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem,
453 uint32_t StorageClassSem) {
454 bool DropStorageClass =
455 TT.isVulkanOS() &&
456 OrderSem == static_cast<uint32_t>(SPIRV::MemorySemantics::None);
457 return OrderSem | (DropStorageClass ? 0 : StorageClassSem);
458}
459
460SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
461 SyncScope::ID Id) {
462 // Named by
463 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_scope_id.
464 // We don't need aliases for Invocation and CrossDevice, as we already have
465 // them covered by "singlethread" and "" strings respectively (see
466 // implementation of LLVMContext::LLVMContext()).
467 auto ScopeID = [&](AtomicScope Scope) {
468 return Ctx.getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(T: TT, S: Scope));
469 };
470 static const llvm::SyncScope::ID SubGroup = ScopeID(AtomicScope::Wavefront);
471 static const llvm::SyncScope::ID WorkGroup = ScopeID(AtomicScope::Workgroup);
472 static const llvm::SyncScope::ID Device = ScopeID(AtomicScope::Device);
473
474 if (Id == llvm::SyncScope::SingleThread)
475 return SPIRV::Scope::Invocation;
476 else if (Id == llvm::SyncScope::System)
477 return SPIRV::Scope::CrossDevice;
478 else if (Id == SubGroup)
479 return SPIRV::Scope::Subgroup;
480 else if (Id == WorkGroup)
481 return SPIRV::Scope::Workgroup;
482 else if (Id == Device)
483 return SPIRV::Scope::Device;
484 return SPIRV::Scope::CrossDevice;
485}
486
487MachineInstr *getDefInstrMaybeConstant(Register &ConstReg,
488 const MachineRegisterInfo *MRI) {
489 MachineInstr *MI = MRI->getVRegDef(Reg: ConstReg);
490 MachineInstr *ConstInstr =
491 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT
492 ? MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg())
493 : MI;
494 if (auto *GI = dyn_cast<GIntrinsic>(Val: ConstInstr)) {
495 if (GI->is(ID: Intrinsic::spv_track_constant)) {
496 ConstReg = ConstInstr->getOperand(i: 2).getReg();
497 return MRI->getVRegDef(Reg: ConstReg);
498 }
499 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {
500 ConstReg = ConstInstr->getOperand(i: 1).getReg();
501 return MRI->getVRegDef(Reg: ConstReg);
502 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||
503 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {
504 ConstReg = ConstInstr->getOperand(i: 0).getReg();
505 return ConstInstr;
506 }
507 return MRI->getVRegDef(Reg: ConstReg);
508}
509
510uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) {
511 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
512 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
513 return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
514}
515
516int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI) {
517 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
518 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
519 return MI->getOperand(i: 1).getCImm()->getSExtValue();
520}
521
522bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
523 if (const auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
524 return GI->is(ID: IntrinsicID);
525 return false;
526}
527
528Type *getMDOperandAsType(const MDNode *N, unsigned I) {
529 Type *ElementTy = cast<ValueAsMetadata>(Val: N->getOperand(I))->getType();
530 return toTypedPointer(Ty: ElementTy);
531}
532
533ConstantInt *getMDOperandAsConstInt(const MDNode *N, unsigned I) {
534 if (N->getNumOperands() <= I)
535 return nullptr;
536 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I)))
537 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
538 return nullptr;
539}
540
541static bool isEnqueueKernelBI(StringRef MangledName) {
542 return MangledName == "__enqueue_kernel_basic" ||
543 MangledName == "__enqueue_kernel_basic_events" ||
544 MangledName == "__enqueue_kernel_varargs" ||
545 MangledName == "__enqueue_kernel_events_varargs";
546}
547
548static bool isKernelQueryBI(StringRef MangledName) {
549 return MangledName == "__get_kernel_work_group_size_impl" ||
550 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
551 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
552 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
553}
554
555static bool isNonMangledOCLBuiltin(StringRef Name) {
556 if (!Name.starts_with(Prefix: "__"))
557 return false;
558
559 return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) ||
560 SPIRV::isPipeOrAddressSpaceCastBuiltin(Name) ||
561 Name == "__translate_sampler_initializer";
562}
563
564std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
565 bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
566 bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_");
567 bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_");
568 bool IsMangled = Name.starts_with(Prefix: "_Z");
569
570 // Otherwise use simple demangling to return the function name.
571 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
572 return Name.str();
573
574 // Try to use the itanium demangler.
575 if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) {
576 std::string Result = DemangledName;
577 free(ptr: DemangledName);
578 return Result;
579 }
580
581 // Autocheck C++, maybe need to do explicit check of the source language.
582 // OpenCL C++ built-ins are declared in cl namespace.
583 // TODO: consider using 'St' abbriviation for cl namespace mangling.
584 // Similar to ::std:: in C++.
585 size_t Start, Len = 0;
586 size_t DemangledNameLenStart = 2;
587 if (Name.starts_with(Prefix: "_ZN")) {
588 // Skip CV and ref qualifiers.
589 size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO", From: 3);
590 // All built-ins are in the ::cl:: namespace.
591 if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv")
592 return std::string();
593 DemangledNameLenStart = NameSpaceStart + 11;
594 }
595 Start = Name.find_first_not_of(Chars: "0123456789", From: DemangledNameLenStart);
596 bool Error = Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart)
597 .getAsInteger(Radix: 10, Result&: Len);
598 if (Error)
599 return std::string();
600 return Name.substr(Start, N: Len).str();
601}
602
603bool hasBuiltinTypePrefix(StringRef Name) {
604 if (Name.starts_with(Prefix: "opencl.") || Name.starts_with(Prefix: "ocl_") ||
605 Name.starts_with(Prefix: "spirv."))
606 return true;
607 return false;
608}
609
610bool isSpecialOpaqueType(const Type *Ty) {
611 if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Val: Ty))
612 return isTypedPointerWrapper(ExtTy)
613 ? false
614 : hasBuiltinTypePrefix(Name: ExtTy->getName());
615
616 return false;
617}
618
619bool isEntryPoint(const Function &F) {
620 // OpenCL handling: any function with the SPIR_KERNEL
621 // calling convention will be a potential entry point.
622 if (F.getCallingConv() == CallingConv::SPIR_KERNEL)
623 return true;
624
625 // HLSL handling: special attribute are emitted from the
626 // front-end.
627 if (F.getFnAttribute(Kind: "hlsl.shader").isValid())
628 return true;
629
630 return false;
631}
632
633Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
634 TypeName.consume_front(Prefix: "atomic_");
635 if (TypeName.consume_front(Prefix: "void"))
636 return Type::getVoidTy(C&: Ctx);
637 else if (TypeName.consume_front(Prefix: "bool") || TypeName.consume_front(Prefix: "_Bool"))
638 return Type::getIntNTy(C&: Ctx, N: 1);
639 else if (TypeName.consume_front(Prefix: "char") ||
640 TypeName.consume_front(Prefix: "signed char") ||
641 TypeName.consume_front(Prefix: "unsigned char") ||
642 TypeName.consume_front(Prefix: "uchar"))
643 return Type::getInt8Ty(C&: Ctx);
644 else if (TypeName.consume_front(Prefix: "short") ||
645 TypeName.consume_front(Prefix: "signed short") ||
646 TypeName.consume_front(Prefix: "unsigned short") ||
647 TypeName.consume_front(Prefix: "ushort"))
648 return Type::getInt16Ty(C&: Ctx);
649 else if (TypeName.consume_front(Prefix: "int") ||
650 TypeName.consume_front(Prefix: "signed int") ||
651 TypeName.consume_front(Prefix: "unsigned int") ||
652 TypeName.consume_front(Prefix: "uint"))
653 return Type::getInt32Ty(C&: Ctx);
654 else if (TypeName.consume_front(Prefix: "long") ||
655 TypeName.consume_front(Prefix: "signed long") ||
656 TypeName.consume_front(Prefix: "unsigned long") ||
657 TypeName.consume_front(Prefix: "ulong"))
658 return Type::getInt64Ty(C&: Ctx);
659 else if (TypeName.consume_front(Prefix: "half") ||
660 TypeName.consume_front(Prefix: "_Float16") ||
661 TypeName.consume_front(Prefix: "__fp16"))
662 return Type::getHalfTy(C&: Ctx);
663 else if (TypeName.consume_front(Prefix: "float"))
664 return Type::getFloatTy(C&: Ctx);
665 else if (TypeName.consume_front(Prefix: "double"))
666 return Type::getDoubleTy(C&: Ctx);
667
668 // Unable to recognize SPIRV type name
669 return nullptr;
670}
671
672SmallPtrSet<BasicBlock *, 0>
673PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
674 std::queue<BasicBlock *> ToVisit;
675 ToVisit.push(x: Start);
676
677 SmallPtrSet<BasicBlock *, 0> Output;
678 while (ToVisit.size() != 0) {
679 BasicBlock *BB = ToVisit.front();
680 ToVisit.pop();
681
682 if (Output.count(Ptr: BB) != 0)
683 continue;
684 Output.insert(Ptr: BB);
685
686 for (BasicBlock *Successor : successors(BB)) {
687 if (DT.dominates(A: Successor, B: BB))
688 continue;
689 ToVisit.push(x: Successor);
690 }
691 }
692
693 return Output;
694}
695
696bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
697 for (BasicBlock *P : predecessors(BB)) {
698 // Ignore back-edges.
699 if (DT.dominates(A: BB, B: P))
700 continue;
701
702 // One of the predecessor hasn't been visited. Not ready yet.
703 if (BlockToOrder.count(Val: P) == 0)
704 return false;
705
706 // If the block is a loop exit, the loop must be finished before
707 // we can continue.
708 Loop *L = LI.getLoopFor(BB: P);
709 if (L == nullptr || L->contains(BB))
710 continue;
711
712 // SPIR-V requires a single back-edge. And the backend first
713 // step transforms loops into the simplified format. If we have
714 // more than 1 back-edge, something is wrong.
715 assert(L->getNumBackEdges() <= 1);
716
717 // If the loop has no latch, loop's rank won't matter, so we can
718 // proceed.
719 BasicBlock *Latch = L->getLoopLatch();
720 assert(Latch);
721 if (Latch == nullptr)
722 continue;
723
724 // The latch is not ready yet, let's wait.
725 if (BlockToOrder.count(Val: Latch) == 0)
726 return false;
727 }
728
729 return true;
730}
731
732size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
733 auto It = BlockToOrder.find(Val: BB);
734 if (It != BlockToOrder.end())
735 return It->second.Rank;
736
737 size_t result = 0;
738 for (BasicBlock *P : predecessors(BB)) {
739 // Ignore back-edges.
740 if (DT.dominates(A: BB, B: P))
741 continue;
742
743 auto Iterator = BlockToOrder.end();
744 Loop *L = LI.getLoopFor(BB: P);
745 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
746
747 // If the predecessor is either outside a loop, or part of
748 // the same loop, simply take its rank + 1.
749 if (L == nullptr || L->contains(BB) || Latch == nullptr) {
750 Iterator = BlockToOrder.find(Val: P);
751 } else {
752 // Otherwise, take the loop's rank (highest rank in the loop) as base.
753 // Since loops have a single latch, highest rank is easy to find.
754 // If the loop has no latch, then it doesn't matter.
755 Iterator = BlockToOrder.find(Val: Latch);
756 }
757
758 assert(Iterator != BlockToOrder.end());
759 result = std::max(a: result, b: Iterator->second.Rank + 1);
760 }
761
762 return result;
763}
764
765size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
766 ToVisit.push(x: BB);
767 Queued.insert(Ptr: BB);
768
769 size_t QueueIndex = 0;
770 while (ToVisit.size() != 0) {
771 BasicBlock *BB = ToVisit.front();
772 ToVisit.pop();
773
774 if (!CanBeVisited(BB)) {
775 ToVisit.push(x: BB);
776 if (QueueIndex >= ToVisit.size())
777 llvm::report_fatal_error(
778 reason: "No valid candidate in the queue. Is the graph reducible?");
779 QueueIndex++;
780 continue;
781 }
782
783 QueueIndex = 0;
784 size_t Rank = GetNodeRank(BB);
785 OrderInfo Info = {.Rank: Rank, .TraversalIndex: BlockToOrder.size()};
786 BlockToOrder.try_emplace(Key: BB, Args&: Info);
787
788 for (BasicBlock *S : successors(BB)) {
789 if (Queued.count(Ptr: S) != 0)
790 continue;
791 ToVisit.push(x: S);
792 Queued.insert(Ptr: S);
793 }
794 }
795
796 return 0;
797}
798
799PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
800 DT.recalculate(Func&: F);
801 LI = LoopInfo(DT);
802
803 visit(BB: &*F.begin(), Unused: 0);
804
805 Order.reserve(n: F.size());
806 for (auto &[BB, Info] : BlockToOrder)
807 Order.emplace_back(args&: BB);
808
809 llvm::sort(C&: Order, Comp: [&](const auto &LHS, const auto &RHS) {
810 return compare(LHS, RHS);
811 });
812}
813
814bool PartialOrderingVisitor::compare(const BasicBlock *LHS,
815 const BasicBlock *RHS) const {
816 const OrderInfo &InfoLHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(LHS));
817 const OrderInfo &InfoRHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(RHS));
818 if (InfoLHS.Rank != InfoRHS.Rank)
819 return InfoLHS.Rank < InfoRHS.Rank;
820 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;
821}
822
823void PartialOrderingVisitor::partialOrderVisit(
824 BasicBlock &Start, std::function<bool(BasicBlock *)> Op) {
825 SmallPtrSet<BasicBlock *, 0> Reachable = getReachableFrom(Start: &Start);
826 assert(BlockToOrder.count(&Start) != 0);
827
828 // Skipping blocks with a rank inferior to |Start|'s rank.
829 auto It = Order.begin();
830 while (It != Order.end() && *It != &Start)
831 ++It;
832
833 // This is unexpected. Worst case |Start| is the last block,
834 // so It should point to the last block, not past-end.
835 assert(It != Order.end());
836
837 // By default, there is no rank limit. Setting it to the maximum value.
838 std::optional<size_t> EndRank = std::nullopt;
839 for (; It != Order.end(); ++It) {
840 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)
841 break;
842
843 if (Reachable.count(Ptr: *It) == 0) {
844 continue;
845 }
846
847 if (!Op(*It)) {
848 EndRank = BlockToOrder[*It].Rank;
849 }
850 }
851}
852
853bool sortBlocks(Function &F) {
854 if (F.size() == 0)
855 return false;
856
857 bool Modified = false;
858 std::vector<BasicBlock *> Order;
859 Order.reserve(n: F.size());
860
861 ReversePostOrderTraversal<Function *> RPOT(&F);
862 llvm::append_range(C&: Order, R&: RPOT);
863
864 assert(&*F.begin() == Order[0]);
865 BasicBlock *LastBlock = &*F.begin();
866 for (BasicBlock *BB : Order) {
867 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {
868 Modified = true;
869 BB->moveAfter(MovePos: LastBlock);
870 }
871 LastBlock = BB;
872 }
873
874 return Modified;
875}
876
877AllocaInst *createVariable(Function &F, Type *Type) {
878 const DataLayout &DL = F.getDataLayout();
879 return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
880 F.begin()->getFirstInsertionPt());
881}
882
883Value *
884createExitVariable(BasicBlock *BB,
885 const DenseMap<BasicBlock *, ConstantInt *> &TargetToValue) {
886 auto *T = BB->getTerminator();
887 if (isa<ReturnInst>(Val: T))
888 return nullptr;
889 if (auto *BI = dyn_cast<UncondBrInst>(Val: T))
890 return TargetToValue.lookup(Val: BI->getSuccessor());
891
892 IRBuilder<> Builder(BB);
893 Builder.SetInsertPoint(T);
894
895 if (auto *BI = dyn_cast<CondBrInst>(Val: T)) {
896 Value *LHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 0));
897 Value *RHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 1));
898
899 if (LHS == nullptr || RHS == nullptr)
900 return LHS == nullptr ? RHS : LHS;
901 return Builder.CreateSelect(C: BI->getCondition(), True: LHS, False: RHS);
902 }
903
904 if (auto *SI = dyn_cast<SwitchInst>(Val: T)) {
905 Value *Condition = SI->getCondition();
906 // The default destination acts as the fallback value of the select chain.
907 Value *Result = TargetToValue.lookup(Val: SI->getDefaultDest());
908 for (const auto &Case : SI->cases()) {
909 Value *CaseValue = TargetToValue.lookup(Val: Case.getCaseSuccessor());
910 // Successors that are internal to the region have no exit value.
911 if (CaseValue == nullptr)
912 continue;
913 // The first known exit value becomes the base of the select chain.
914 if (Result == nullptr) {
915 Result = CaseValue;
916 continue;
917 }
918 Value *Cmp = Builder.CreateICmpEQ(LHS: Condition, RHS: Case.getCaseValue());
919 Result = Builder.CreateSelect(C: Cmp, True: CaseValue, False: Result);
920 }
921 return Result;
922 }
923
924 llvm_unreachable("Unhandled terminator type.");
925}
926
927MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
928 MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
929 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
930 MaybeDef = MRI.getVRegDef(Reg: MaybeDef->getOperand(i: 1).getReg());
931 return MaybeDef;
932}
933
934static bool getVacantFunctionName(Module &M, std::string &Name) {
935 // It's a bit of paranoia, but still we don't want to have even a chance that
936 // the loop will work for too long.
937 constexpr unsigned MaxIters = 1024;
938 for (unsigned I = 0; I < MaxIters; ++I) {
939 std::string OrdName = Name + Twine(I).str();
940 if (!M.getFunction(Name: OrdName)) {
941 Name = std::move(OrdName);
942 return true;
943 }
944 }
945 return false;
946}
947
948// Assign SPIR-V type to the register. If the register has no valid assigned
949// class, set register LLT type and class according to the SPIR-V type.
950void setRegClassType(Register Reg, SPIRVTypeInst SpvType,
951 SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
952 const MachineFunction &MF, bool Force) {
953 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
954 if (!MRI->getRegClassOrNull(Reg) || Force) {
955 MRI->setRegClass(Reg, RC: GR->getRegClass(SpvType));
956 LLT RegType = GR->getRegType(SpvType);
957 if (Force || !MRI->getType(Reg).isValid())
958 MRI->setType(VReg: Reg, Ty: RegType);
959 }
960}
961
962// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
963// no valid assigned class, set register LLT type and class according to the
964// SPIR-V type.
965void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
966 MachineIRBuilder &MIRBuilder,
967 SPIRV::AccessQualifier::AccessQualifier AccessQual,
968 bool EmitIR, bool Force) {
969 setRegClassType(Reg,
970 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR),
971 GR, MRI: MIRBuilder.getMRI(), MF: MIRBuilder.getMF(), Force);
972}
973
974// Create a virtual register and assign SPIR-V type to the register. Set
975// register LLT type and class according to the SPIR-V type.
976Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
977 MachineRegisterInfo *MRI,
978 const MachineFunction &MF) {
979 Register Reg = MRI->createVirtualRegister(RegClass: GR->getRegClass(SpvType));
980 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
981 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
982 return Reg;
983}
984
985// Create a virtual register and assign SPIR-V type to the register. Set
986// register LLT type and class according to the SPIR-V type.
987Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
988 MachineIRBuilder &MIRBuilder) {
989 return createVirtualRegister(SpvType, GR, MRI: MIRBuilder.getMRI(),
990 MF: MIRBuilder.getMF());
991}
992
993// Create a SPIR-V type, virtual register and assign SPIR-V type to the
994// register. Set register LLT type and class according to the SPIR-V type.
995Register createVirtualRegister(
996 const Type *Ty, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIRBuilder,
997 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
998 return createVirtualRegister(
999 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR), GR,
1000 MIRBuilder);
1001}
1002
1003bool isVectorType(SPIRVTypeInst SPVTy) {
1004 return SPVTy->getOpcode() == SPIRV::OpTypeVector ||
1005 SPVTy->getOpcode() == SPIRV::OpTypeVectorIdEXT;
1006}
1007
1008CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
1009 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
1010 IRBuilder<> &B) {
1011 SmallVector<Value *, 4> Args;
1012 Args.push_back(Elt: Arg2);
1013 Args.push_back(Elt: buildMD(Arg));
1014 llvm::append_range(C&: Args, R&: Imms);
1015 return B.CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: {Types}, Args);
1016}
1017
1018// Return true if there is an opaque pointer type nested in the argument.
1019bool isNestedPointer(const Type *Ty) {
1020 if (Ty->isPtrOrPtrVectorTy())
1021 return true;
1022 if (const FunctionType *RefTy = dyn_cast<FunctionType>(Val: Ty)) {
1023 if (isNestedPointer(Ty: RefTy->getReturnType()))
1024 return true;
1025 for (const Type *ArgTy : RefTy->params())
1026 if (isNestedPointer(Ty: ArgTy))
1027 return true;
1028 return false;
1029 }
1030 if (const ArrayType *RefTy = dyn_cast<ArrayType>(Val: Ty))
1031 return isNestedPointer(Ty: RefTy->getElementType());
1032 return false;
1033}
1034
1035bool isSpvIntrinsic(const Value *Arg) {
1036 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Arg))
1037 if (Function *F = II->getCalledFunction())
1038 if (F->getName().starts_with(Prefix: "llvm.spv."))
1039 return true;
1040 return false;
1041}
1042
1043// Function to create continued instructions for SPV_INTEL_long_composites
1044// extension
1045SmallVector<MachineInstr *, 4>
1046createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode,
1047 unsigned MinWC, unsigned ContinuedOpcode,
1048 ArrayRef<Register> Args, Register ReturnRegister,
1049 Register TypeID) {
1050
1051 SmallVector<MachineInstr *, 4> Instructions;
1052 constexpr unsigned MaxWordCount = UINT16_MAX;
1053 const size_t NumElements = Args.size();
1054 size_t MaxNumElements = MaxWordCount - MinWC;
1055 size_t SPIRVStructNumElements = NumElements;
1056
1057 if (NumElements > MaxNumElements) {
1058 // Do adjustments for continued instructions which always had only one
1059 // minumum word count.
1060 SPIRVStructNumElements = MaxNumElements;
1061 MaxNumElements = MaxWordCount - 1;
1062 }
1063
1064 auto MIB =
1065 MIRBuilder.buildInstr(Opcode).addDef(RegNo: ReturnRegister).addUse(RegNo: TypeID);
1066
1067 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1068 MIB.addUse(RegNo: Args[I]);
1069
1070 Instructions.push_back(Elt: MIB.getInstr());
1071
1072 for (size_t I = SPIRVStructNumElements; I < NumElements;
1073 I += MaxNumElements) {
1074 auto MIB = MIRBuilder.buildInstr(Opcode: ContinuedOpcode);
1075 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
1076 MIB.addUse(RegNo: Args[J]);
1077 Instructions.push_back(Elt: MIB.getInstr());
1078 }
1079 return Instructions;
1080}
1081
1082SmallVector<unsigned, 1>
1083getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD) {
1084 unsigned LC = SPIRV::LoopControl::None;
1085 // Currently used only to store PartialCount value. Later when other
1086 // LoopControls are added - this map should be sorted before making
1087 // them loop_merge operands to satisfy 3.23. Loop Control requirements.
1088 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;
1089 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.disable")) {
1090 LC |= SPIRV::LoopControl::DontUnroll;
1091 } else {
1092 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.enable") ||
1093 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.full")) {
1094 LC |= SPIRV::LoopControl::Unroll;
1095 }
1096 if (MDNode *CountMD =
1097 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.count")) {
1098 if (auto *CI =
1099 mdconst::extract_or_null<ConstantInt>(MD: CountMD->getOperand(I: 1))) {
1100 unsigned Count = CI->getZExtValue();
1101 if (Count != 1) {
1102 LC |= SPIRV::LoopControl::PartialCount;
1103 MaskToValueMap.emplace_back(
1104 args: std::make_pair(x: SPIRV::LoopControl::PartialCount, y&: Count));
1105 }
1106 }
1107 }
1108 }
1109 SmallVector<unsigned, 1> Result = {LC};
1110 for (auto &[Mask, Val] : MaskToValueMap)
1111 Result.push_back(Elt: Val);
1112 return Result;
1113}
1114
1115SmallVector<unsigned, 1> getSpirvLoopControlOperandsFromLoopMetadata(Loop *L) {
1116 return getSpirvLoopControlOperandsFromLoopMetadata(LoopMD: L->getLoopID());
1117}
1118
1119const std::set<unsigned> &getTypeFoldingSupportedOpcodes() {
1120 // clang-format off
1121 static const std::set<unsigned> TypeFoldingSupportingOpcs = {
1122 TargetOpcode::G_ADD,
1123 TargetOpcode::G_FADD,
1124 TargetOpcode::G_STRICT_FADD,
1125 TargetOpcode::G_SUB,
1126 TargetOpcode::G_FSUB,
1127 TargetOpcode::G_STRICT_FSUB,
1128 TargetOpcode::G_MUL,
1129 TargetOpcode::G_FMUL,
1130 TargetOpcode::G_STRICT_FMUL,
1131 TargetOpcode::G_SDIV,
1132 TargetOpcode::G_UDIV,
1133 TargetOpcode::G_FDIV,
1134 TargetOpcode::G_STRICT_FDIV,
1135 TargetOpcode::G_SREM,
1136 TargetOpcode::G_UREM,
1137 TargetOpcode::G_FREM,
1138 TargetOpcode::G_STRICT_FREM,
1139 TargetOpcode::G_FNEG,
1140 TargetOpcode::G_CONSTANT,
1141 TargetOpcode::G_FCONSTANT,
1142 TargetOpcode::G_AND,
1143 TargetOpcode::G_OR,
1144 TargetOpcode::G_XOR,
1145 TargetOpcode::G_SHL,
1146 TargetOpcode::G_ASHR,
1147 TargetOpcode::G_LSHR,
1148 TargetOpcode::G_SELECT,
1149 TargetOpcode::G_EXTRACT_VECTOR_ELT,
1150 };
1151 // clang-format on
1152 return TypeFoldingSupportingOpcs;
1153}
1154
1155bool isTypeFoldingSupported(unsigned Opcode) {
1156 return getTypeFoldingSupportedOpcodes().count(x: Opcode) > 0;
1157}
1158
1159// Traversing [g]MIR accounting for pseudo-instructions.
1160MachineInstr *passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI) {
1161 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||
1162 Def->getOpcode() == TargetOpcode::COPY)
1163 ? MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())
1164 : Def;
1165}
1166
1167MachineInstr *getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1168 if (MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg()))
1169 return passCopy(Def, MRI);
1170 return nullptr;
1171}
1172
1173MachineInstr *getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1174 if (MachineInstr *Def = getDef(MO, MRI)) {
1175 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
1176 Def->getOpcode() == SPIRV::OpConstantI)
1177 return Def;
1178 }
1179 return nullptr;
1180}
1181
1182int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1183 if (MachineInstr *Def = getImm(MO, MRI)) {
1184 if (Def->getOpcode() == SPIRV::OpConstantI)
1185 return Def->getOperand(i: 2).getImm();
1186 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1187 return Def->getOperand(i: 1).getCImm()->getZExtValue();
1188 }
1189 llvm_unreachable("Unexpected integer constant pattern");
1190}
1191
1192unsigned getArrayComponentCount(const MachineRegisterInfo *MRI,
1193 const MachineInstr *ResType) {
1194 return foldImm(MO: ResType->getOperand(i: 2), MRI);
1195}
1196
1197bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType,
1198 uint64_t &TotalSize) {
1199 // An array of N padded structs is represented as {[N-1 x <{T, pad}>], T}.
1200 if (Ty->getStructNumElements() != 2)
1201 return false;
1202
1203 Type *FirstElement = Ty->getStructElementType(N: 0);
1204 Type *SecondElement = Ty->getStructElementType(N: 1);
1205
1206 if (!FirstElement->isArrayTy())
1207 return false;
1208
1209 Type *ArrayElementType = FirstElement->getArrayElementType();
1210 if (!ArrayElementType->isStructTy() ||
1211 ArrayElementType->getStructNumElements() != 2)
1212 return false;
1213
1214 Type *T_in_struct = ArrayElementType->getStructElementType(N: 0);
1215 if (T_in_struct != SecondElement)
1216 return false;
1217
1218 auto *Padding_in_struct =
1219 dyn_cast<TargetExtType>(Val: ArrayElementType->getStructElementType(N: 1));
1220 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")
1221 return false;
1222
1223 const uint64_t ArraySize = FirstElement->getArrayNumElements();
1224 TotalSize = ArraySize + 1;
1225 OriginalElementType = ArrayElementType;
1226 return true;
1227}
1228
1229Type *reconstitutePeeledArrayType(Type *Ty) {
1230 if (!Ty->isStructTy())
1231 return Ty;
1232
1233 auto *STy = cast<StructType>(Val: Ty);
1234 Type *OriginalElementType = nullptr;
1235 uint64_t TotalSize = 0;
1236 if (matchPeeledArrayPattern(Ty: STy, OriginalElementType, TotalSize)) {
1237 Type *ResultTy = ArrayType::get(
1238 ElementType: reconstitutePeeledArrayType(Ty: OriginalElementType), NumElements: TotalSize);
1239 return ResultTy;
1240 }
1241
1242 SmallVector<Type *, 4> NewElementTypes;
1243 bool Changed = false;
1244 for (Type *ElementTy : STy->elements()) {
1245 Type *NewElementTy = reconstitutePeeledArrayType(Ty: ElementTy);
1246 if (NewElementTy != ElementTy)
1247 Changed = true;
1248 NewElementTypes.push_back(Elt: NewElementTy);
1249 }
1250
1251 if (!Changed)
1252 return Ty;
1253
1254 Type *ResultTy;
1255 if (STy->isLiteral()) {
1256 ResultTy =
1257 StructType::get(Context&: STy->getContext(), Elements: NewElementTypes, isPacked: STy->isPacked());
1258 } else {
1259 ResultTy = StructType::create(Context&: STy->getContext(), Elements: NewElementTypes,
1260 Name: STy->getName(), isPacked: STy->isPacked());
1261 }
1262 return ResultTy;
1263}
1264
1265std::optional<SPIRV::LinkageType::LinkageType>
1266getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV) {
1267 if (GV.hasLocalLinkage())
1268 return std::nullopt;
1269
1270 if (GV.isDeclarationForLinker()) {
1271 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: &GV)) {
1272 auto SC = addressSpaceToStorageClass(AddrSpace: GVar->getAddressSpace(), STI: ST);
1273 // Interface variables must not get Import linkage.
1274 if (SC == SPIRV::StorageClass::Input ||
1275 SC == SPIRV::StorageClass::Output ||
1276 SC == SPIRV::StorageClass::PushConstant)
1277 return std::nullopt;
1278 // Shaders have no linker, so module-internal storage
1279 // (e.g. HLSL groupshared) can't be imported
1280 if (ST.isShader() && (SC == SPIRV::StorageClass::Workgroup ||
1281 SC == SPIRV::StorageClass::Private))
1282 return std::nullopt;
1283 }
1284 return SPIRV::LinkageType::Import;
1285 }
1286
1287 if (GV.hasHiddenVisibility())
1288 return std::nullopt;
1289
1290 if (GV.hasLinkOnceODRLinkage() &&
1291 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr))
1292 return SPIRV::LinkageType::LinkOnceODR;
1293
1294 if (GV.hasWeakLinkage() &&
1295 ST.canUseExtension(E: SPIRV::Extension::SPV_AMD_weak_linkage))
1296 return SPIRV::LinkageType::WeakAMD;
1297
1298 return SPIRV::LinkageType::Export;
1299}
1300
1301Function *getOrCreateBackendServiceFunction(Module &M) {
1302 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
1303 if (!getVacantFunctionName(M, Name&: ServiceFunName))
1304 report_fatal_error(
1305 reason: "cannot allocate a name for the internal service function");
1306 if (Function *SF = M.getFunction(Name: ServiceFunName)) {
1307 if (SF->getInstructionCount() > 0)
1308 report_fatal_error(
1309 reason: "Unexpected combination of global variables and function pointers");
1310 return SF;
1311 }
1312 Function *SF = Function::Create(
1313 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: {}, isVarArg: false),
1314 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
1315 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
1316 return SF;
1317}
1318
1319} // namespace llvm
1320