1//===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVUtils.h"
14#include "MCTargetDesc/SPIRVBaseInfo.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRVInstrInfo.h"
19#include "SPIRVSubtarget.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/Demangle/Demangle.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30#include <queue>
31#include <vector>
32
33namespace llvm {
34namespace SPIRV {
35static MDNode *findNamedMDOperand(NamedMDNode *NMD, StringRef Name) {
36 auto It = find_if(Range: NMD->operands(), P: [Name](MDNode *N) {
37 if (auto *MDS = dyn_cast_or_null<MDString>(Val: N->getOperand(I: 0)))
38 return MDS->getString() == Name;
39 return false;
40 });
41 return It == NMD->op_end() ? nullptr : *It;
42}
43
44// This code restores function args/retvalue types for composite cases
45// because the final types should still be aggregate whereas they're i32
46// during the translation to cope with aggregate flattening etc.
47// TODO: should these just return nullptr when there's no metadata?
48static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
49 FunctionType *FTy,
50 StringRef Name) {
51 if (!NMD)
52 return FTy;
53
54 MDNode *Match = findNamedMDOperand(NMD, Name);
55 if (!Match)
56 return FTy;
57
58 Type *RetTy = FTy->getReturnType();
59 SmallVector<Type *, 4> PTys(FTy->params());
60
61 for (unsigned I = 1; I != Match->getNumOperands(); ++I) {
62 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I));
63 assert(MD && "MDNode operand is expected");
64
65 if (auto *Const = getMDOperandAsConstInt(N: MD, I: 0)) {
66 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1));
67 assert(CMeta && "ConstantAsMetadata operand is expected");
68 int64_t Idx = Const->getSExtValue();
69 // Currently -1 indicates return value, greater values mean
70 // argument numbers.
71 if (Idx == -1) {
72 RetTy = CMeta->getType();
73 continue;
74 }
75 if (Idx >= 0 && static_cast<uint64_t>(Idx) < PTys.size()) {
76 PTys[Idx] = CMeta->getType();
77 continue;
78 }
79 report_fatal_error(reason: "invalid argument index in function type metadata");
80 }
81 }
82
83 return FunctionType::get(Result: RetTy, Params: PTys, isVarArg: FTy->isVarArg());
84}
85
86static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
87 StringRef Constraints,
88 StringRef Name) {
89 if (!NMD)
90 return Constraints;
91
92 MDNode *Match = findNamedMDOperand(NMD, Name);
93 if (!Match)
94 return Constraints;
95
96 // By convention, the constraints string is stored in the final MD operand.
97 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I: Match->getNumOperands() - 1));
98 assert(MD && "MDNode operand is expected");
99
100 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
101 Constraints = MDS->getString();
102
103 return Constraints;
104}
105
106FunctionType *getOriginalFunctionType(const Function &F) {
107 return extractFunctionTypeFromMetadata(
108 NMD: F.getParent()->getNamedMetadata(Name: "spv.cloned_funcs"), FTy: F.getFunctionType(),
109 Name: F.getName());
110}
111
112// Keyed via instruction metadata, not a name.
113static std::optional<StringRef> getMutatedCallsiteKey(const CallBase &CB) {
114 if (MDNode *MD = CB.getMetadata(Kind: "spv.mutated_callsite"))
115 if (MD->getNumOperands() > 0)
116 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
117 return MDS->getString();
118 return std::nullopt;
119}
120
121FunctionType *getOriginalFunctionType(const CallBase &CB) {
122 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
123 if (!Key)
124 return CB.getFunctionType();
125 return extractFunctionTypeFromMetadata(
126 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"),
127 FTy: CB.getFunctionType(), Name: *Key);
128}
129
130StringRef getOriginalAsmConstraints(const CallBase &CB) {
131 StringRef Constraints =
132 cast<InlineAsm>(Val: CB.getCalledOperand())->getConstraintString();
133 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
134 if (!Key)
135 return Constraints;
136 return extractAsmConstraintsFromMetadata(
137 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"), Constraints,
138 Name: *Key);
139}
140} // Namespace SPIRV
141
142// The following functions are used to add these string literals as a series of
143// 32-bit integer operands with the correct format, and unpack them if necessary
144// when making string comparisons in compiler passes.
145// SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment.
146static uint32_t convertCharsToWord(StringRef Str, unsigned i) {
147 uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars.
148 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {
149 unsigned StrIndex = i + WordIndex;
150 uint8_t CharToAdd = 0; // Initilize char as padding/null.
151 if (StrIndex < Str.size()) { // If it's within the string, get a real char.
152 CharToAdd = Str[StrIndex];
153 }
154 Word |= (CharToAdd << (WordIndex * 8));
155 }
156 return Word;
157}
158
159// Get length including padding and null terminator.
160static size_t getPaddedLen(StringRef Str) { return alignTo(Value: Str.size() + 1, Align: 4); }
161
162void addStringImm(StringRef Str, MCInst &Inst) {
163 const size_t PaddedLen = getPaddedLen(Str);
164 for (unsigned i = 0; i < PaddedLen; i += 4) {
165 // Add an operand for the 32-bits of chars or padding.
166 Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i)));
167 }
168}
169
170void addStringImm(StringRef Str, MachineInstrBuilder &MIB) {
171 const size_t PaddedLen = getPaddedLen(Str);
172 for (unsigned i = 0; i < PaddedLen; i += 4) {
173 // Add an operand for the 32-bits of chars or padding.
174 MIB.addImm(Val: convertCharsToWord(Str, i));
175 }
176}
177
178std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
179 return getSPIRVStringOperand(MI, StartIndex);
180}
181
182std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
183 MachineInstr *Def = getVRegDef(MRI, Reg);
184 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
185 "Expected G_GLOBAL_VALUE");
186 const GlobalValue *GV = Def->getOperand(i: 1).getGlobal();
187 Value *V = GV->getOperand(i: 0);
188 const ConstantDataArray *CDA = cast<ConstantDataArray>(Val: V);
189 return CDA->getAsCString().str();
190}
191
192void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
193 const auto Bitwidth = Imm.getBitWidth();
194 if (Bitwidth == 1)
195 return; // Already handled
196 else if (Bitwidth <= 32) {
197 MIB.addImm(Val: Imm.getZExtValue());
198 // Asm Printer needs this info to print floating-type correctly
199 if (Bitwidth == 16)
200 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16);
201 return;
202 } else if (Bitwidth <= 64) {
203 uint64_t FullImm = Imm.getZExtValue();
204 MIB.addImm(Val: Lo_32(Value: FullImm)).addImm(Val: Hi_32(Value: FullImm));
205 // Asm Printer needs this info to print 64-bit operands correctly
206 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH64);
207 return;
208 } else {
209 // Emit ceil(Bitwidth / 32) words to conform SPIR-V spec.
210 unsigned NumWords = divideCeil(Numerator: Bitwidth, Denominator: 32);
211 for (unsigned I = 0; I < NumWords; ++I) {
212 unsigned LimbIdx = I / 2;
213 unsigned LimbShift = (I % 2) * 32;
214 uint32_t Word = (Imm.getRawData()[LimbIdx] >> LimbShift) & 0xffffffff;
215 MIB.addImm(Val: Word);
216 }
217 return;
218 }
219}
220
221void buildOpName(Register Target, StringRef Name,
222 MachineIRBuilder &MIRBuilder) {
223 if (!Name.empty()) {
224 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpName).addUse(RegNo: Target);
225 addStringImm(Str: Name, MIB);
226 }
227}
228
229void buildOpName(Register Target, StringRef Name, MachineInstr &I,
230 const SPIRVInstrInfo &TII) {
231 if (!Name.empty()) {
232 auto MIB =
233 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName))
234 .addUse(RegNo: Target);
235 addStringImm(Str: Name, MIB);
236 }
237}
238
239static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
240 ArrayRef<uint32_t> DecArgs,
241 StringRef StrImm) {
242 if (!StrImm.empty())
243 addStringImm(Str: StrImm, MIB);
244 for (const auto &DecArg : DecArgs)
245 MIB.addImm(Val: DecArg);
246}
247
248void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
249 SPIRV::Decoration::Decoration Dec,
250 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
251 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
252 .addUse(RegNo: Reg)
253 .addImm(Val: static_cast<uint32_t>(Dec));
254 finishBuildOpDecorate(MIB, DecArgs, StrImm);
255}
256
257void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
258 SPIRV::Decoration::Decoration Dec,
259 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
260 MachineBasicBlock &MBB = *I.getParent();
261 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDecorate))
262 .addUse(RegNo: Reg)
263 .addImm(Val: static_cast<uint32_t>(Dec));
264 finishBuildOpDecorate(MIB, DecArgs, StrImm);
265}
266
267void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
268 SPIRV::Decoration::Decoration Dec, uint32_t Member,
269 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
270 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpMemberDecorate)
271 .addUse(RegNo: Reg)
272 .addImm(Val: Member)
273 .addImm(Val: static_cast<uint32_t>(Dec));
274 finishBuildOpDecorate(MIB, DecArgs, StrImm);
275}
276
277void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
278 const MDNode *GVarMD, const SPIRVSubtarget &ST) {
279 for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
280 auto *OpMD = dyn_cast<MDNode>(Val: GVarMD->getOperand(I));
281 if (!OpMD)
282 report_fatal_error(reason: "Invalid decoration");
283 if (OpMD->getNumOperands() == 0)
284 report_fatal_error(reason: "Expect operand(s) of the decoration");
285 ConstantInt *DecorationId =
286 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 0));
287 if (!DecorationId)
288 report_fatal_error(reason: "Expect SPIR-V <Decoration> operand to be the first "
289 "element of the decoration");
290
291 // The goal of `spirv.Decorations` metadata is to provide a way to
292 // represent SPIR-V entities that do not map to LLVM in an obvious way.
293 // FP flags do have obvious matches between LLVM IR and SPIR-V.
294 // Additionally, we have no guarantee at this point that the flags passed
295 // through the decoration are not violated already in the optimizer passes.
296 // Therefore, we simply ignore FP flags, including NoContraction, and
297 // FPFastMathMode.
298 if (DecorationId->getZExtValue() ==
299 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||
300 DecorationId->getZExtValue() ==
301 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {
302 continue; // Ignored.
303 }
304 uint32_t Dec = static_cast<uint32_t>(DecorationId->getZExtValue());
305 if (Dec == static_cast<uint32_t>(SPIRV::Decoration::UniformId)) {
306 ConstantInt *ScopeV =
307 OpMD->getNumOperands() == 2
308 ? mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 1))
309 : nullptr;
310 assert(ScopeV && isUInt<32>(ScopeV->getZExtValue()) &&
311 "Expect Scope <id> operand of the UniformId decoration");
312 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
313 SPIRVTypeInst SpvTypeInt32 =
314 GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
315 Register ScopeReg = GR->buildConstantInt(
316 Val: ScopeV->getZExtValue(), MIRBuilder, SpvType: SpvTypeInt32, /*EmitIR=*/false);
317 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorateId)
318 .addUse(RegNo: Reg)
319 .addImm(Val: Dec)
320 .addUse(RegNo: ScopeReg);
321 continue;
322 }
323 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate).addUse(RegNo: Reg).addImm(Val: Dec);
324 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
325 if (ConstantInt *OpV =
326 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: OpI)))
327 MIB.addImm(Val: static_cast<uint32_t>(OpV->getZExtValue()));
328 else if (MDString *OpV = dyn_cast<MDString>(Val: OpMD->getOperand(I: OpI)))
329 addStringImm(Str: OpV->getString(), MIB);
330 else
331 report_fatal_error(reason: "Unexpected operand of the decoration");
332 }
333 }
334}
335
336MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF) {
337 MachineBasicBlock &MBB = MF.front();
338 // Find the position to insert the OpVariable instruction.
339 // We will insert it after the last OpFunctionParameter, if any, or
340 // after OpFunction otherwise.
341 auto IsPreamble = [](const MachineInstr &MI) {
342 switch (MI.getOpcode()) {
343 case SPIRV::OpFunction:
344 case SPIRV::OpFunctionParameter:
345 case SPIRV::OpLabel:
346 case SPIRV::ASSIGN_TYPE:
347 return true;
348 default:
349 return false;
350 }
351 };
352 MachineBasicBlock::iterator VarPos = MBB.SkipPHIsAndLabels(I: MBB.begin());
353 while (VarPos != MBB.end() && VarPos->getOpcode() != SPIRV::OpFunction)
354 ++VarPos;
355 // Advance past the preamble.
356 while (VarPos != MBB.end() && IsPreamble(*VarPos))
357 ++VarPos;
358 return VarPos;
359}
360
361MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB) {
362 MachineBasicBlock::iterator I = MBB->end();
363 if (I == MBB->begin())
364 return I;
365 --I;
366 while (I->isTerminator() || I->isDebugValue()) {
367 if (I == MBB->begin())
368 break;
369 --I;
370 }
371 return I;
372}
373
374SPIRV::StorageClass::StorageClass
375addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
376 switch (AddrSpace) {
377 case 0:
378 return SPIRV::StorageClass::Function;
379 case 1:
380 return SPIRV::StorageClass::CrossWorkgroup;
381 case 2:
382 return SPIRV::StorageClass::UniformConstant;
383 case 3:
384 return SPIRV::StorageClass::Workgroup;
385 case 4:
386 return SPIRV::StorageClass::Generic;
387 case 5:
388 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
389 ? SPIRV::StorageClass::DeviceOnlyINTEL
390 : SPIRV::StorageClass::CrossWorkgroup;
391 case 6:
392 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
393 ? SPIRV::StorageClass::HostOnlyINTEL
394 : SPIRV::StorageClass::CrossWorkgroup;
395 case 7:
396 return SPIRV::StorageClass::Input;
397 case 8:
398 return SPIRV::StorageClass::Output;
399 case 9:
400 return SPIRV::StorageClass::CodeSectionINTEL;
401 case 10:
402 return SPIRV::StorageClass::Private;
403 case 11:
404 return SPIRV::StorageClass::StorageBuffer;
405 case 12:
406 return SPIRV::StorageClass::Uniform;
407 case 13:
408 return SPIRV::StorageClass::PushConstant;
409 default:
410 report_fatal_error(reason: "Unknown address space");
411 }
412}
413
414SPIRV::MemorySemantics::MemorySemantics
415getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) {
416 switch (SC) {
417 case SPIRV::StorageClass::StorageBuffer:
418 case SPIRV::StorageClass::Uniform:
419 return SPIRV::MemorySemantics::UniformMemory;
420 case SPIRV::StorageClass::Workgroup:
421 return SPIRV::MemorySemantics::WorkgroupMemory;
422 case SPIRV::StorageClass::CrossWorkgroup:
423 return SPIRV::MemorySemantics::CrossWorkgroupMemory;
424 case SPIRV::StorageClass::AtomicCounter:
425 return SPIRV::MemorySemantics::AtomicCounterMemory;
426 case SPIRV::StorageClass::Image:
427 return SPIRV::MemorySemantics::ImageMemory;
428 default:
429 return SPIRV::MemorySemantics::None;
430 }
431}
432
433SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
434 switch (Ord) {
435 case AtomicOrdering::Acquire:
436 return SPIRV::MemorySemantics::Acquire;
437 case AtomicOrdering::Release:
438 return SPIRV::MemorySemantics::Release;
439 case AtomicOrdering::AcquireRelease:
440 return SPIRV::MemorySemantics::AcquireRelease;
441 case AtomicOrdering::SequentiallyConsistent:
442 return SPIRV::MemorySemantics::SequentiallyConsistent;
443 case AtomicOrdering::Unordered:
444 case AtomicOrdering::Monotonic:
445 case AtomicOrdering::NotAtomic:
446 return SPIRV::MemorySemantics::None;
447 }
448 llvm_unreachable(nullptr);
449}
450
451SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id) {
452 // Named by
453 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_scope_id.
454 // We don't need aliases for Invocation and CrossDevice, as we already have
455 // them covered by "singlethread" and "" strings respectively (see
456 // implementation of LLVMContext::LLVMContext()).
457 static const llvm::SyncScope::ID SubGroup =
458 Ctx.getOrInsertSyncScopeID(SSN: "subgroup");
459 static const llvm::SyncScope::ID WorkGroup =
460 Ctx.getOrInsertSyncScopeID(SSN: "workgroup");
461 static const llvm::SyncScope::ID Device =
462 Ctx.getOrInsertSyncScopeID(SSN: "device");
463
464 if (Id == llvm::SyncScope::SingleThread)
465 return SPIRV::Scope::Invocation;
466 else if (Id == llvm::SyncScope::System)
467 return SPIRV::Scope::CrossDevice;
468 else if (Id == SubGroup)
469 return SPIRV::Scope::Subgroup;
470 else if (Id == WorkGroup)
471 return SPIRV::Scope::Workgroup;
472 else if (Id == Device)
473 return SPIRV::Scope::Device;
474 return SPIRV::Scope::CrossDevice;
475}
476
477MachineInstr *getDefInstrMaybeConstant(Register &ConstReg,
478 const MachineRegisterInfo *MRI) {
479 MachineInstr *MI = MRI->getVRegDef(Reg: ConstReg);
480 MachineInstr *ConstInstr =
481 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT
482 ? MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg())
483 : MI;
484 if (auto *GI = dyn_cast<GIntrinsic>(Val: ConstInstr)) {
485 if (GI->is(ID: Intrinsic::spv_track_constant)) {
486 ConstReg = ConstInstr->getOperand(i: 2).getReg();
487 return MRI->getVRegDef(Reg: ConstReg);
488 }
489 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {
490 ConstReg = ConstInstr->getOperand(i: 1).getReg();
491 return MRI->getVRegDef(Reg: ConstReg);
492 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||
493 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {
494 ConstReg = ConstInstr->getOperand(i: 0).getReg();
495 return ConstInstr;
496 }
497 return MRI->getVRegDef(Reg: ConstReg);
498}
499
500uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) {
501 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
502 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
503 return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
504}
505
506int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI) {
507 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
508 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
509 return MI->getOperand(i: 1).getCImm()->getSExtValue();
510}
511
512bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
513 if (const auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
514 return GI->is(ID: IntrinsicID);
515 return false;
516}
517
518Type *getMDOperandAsType(const MDNode *N, unsigned I) {
519 Type *ElementTy = cast<ValueAsMetadata>(Val: N->getOperand(I))->getType();
520 return toTypedPointer(Ty: ElementTy);
521}
522
523ConstantInt *getMDOperandAsConstInt(const MDNode *N, unsigned I) {
524 if (N->getNumOperands() <= I)
525 return nullptr;
526 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I)))
527 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
528 return nullptr;
529}
530
531static bool isEnqueueKernelBI(StringRef MangledName) {
532 return MangledName == "__enqueue_kernel_basic" ||
533 MangledName == "__enqueue_kernel_basic_events" ||
534 MangledName == "__enqueue_kernel_varargs" ||
535 MangledName == "__enqueue_kernel_events_varargs";
536}
537
538static bool isKernelQueryBI(StringRef MangledName) {
539 return MangledName == "__get_kernel_work_group_size_impl" ||
540 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
541 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
542 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
543}
544
545static bool isNonMangledOCLBuiltin(StringRef Name) {
546 if (!Name.starts_with(Prefix: "__"))
547 return false;
548
549 return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) ||
550 SPIRV::isPipeOrAddressSpaceCastBuiltin(Name) ||
551 Name == "__translate_sampler_initializer";
552}
553
554std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
555 bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
556 bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_");
557 bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_");
558 bool IsMangled = Name.starts_with(Prefix: "_Z");
559
560 // Otherwise use simple demangling to return the function name.
561 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
562 return Name.str();
563
564 // Try to use the itanium demangler.
565 if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) {
566 std::string Result = DemangledName;
567 free(ptr: DemangledName);
568 return Result;
569 }
570
571 // Autocheck C++, maybe need to do explicit check of the source language.
572 // OpenCL C++ built-ins are declared in cl namespace.
573 // TODO: consider using 'St' abbriviation for cl namespace mangling.
574 // Similar to ::std:: in C++.
575 size_t Start, Len = 0;
576 size_t DemangledNameLenStart = 2;
577 if (Name.starts_with(Prefix: "_ZN")) {
578 // Skip CV and ref qualifiers.
579 size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO", From: 3);
580 // All built-ins are in the ::cl:: namespace.
581 if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv")
582 return std::string();
583 DemangledNameLenStart = NameSpaceStart + 11;
584 }
585 Start = Name.find_first_not_of(Chars: "0123456789", From: DemangledNameLenStart);
586 bool Error = Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart)
587 .getAsInteger(Radix: 10, Result&: Len);
588 if (Error)
589 return std::string();
590 return Name.substr(Start, N: Len).str();
591}
592
593bool hasBuiltinTypePrefix(StringRef Name) {
594 if (Name.starts_with(Prefix: "opencl.") || Name.starts_with(Prefix: "ocl_") ||
595 Name.starts_with(Prefix: "spirv."))
596 return true;
597 return false;
598}
599
600bool isSpecialOpaqueType(const Type *Ty) {
601 if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Val: Ty))
602 return isTypedPointerWrapper(ExtTy)
603 ? false
604 : hasBuiltinTypePrefix(Name: ExtTy->getName());
605
606 return false;
607}
608
609bool isEntryPoint(const Function &F) {
610 // OpenCL handling: any function with the SPIR_KERNEL
611 // calling convention will be a potential entry point.
612 if (F.getCallingConv() == CallingConv::SPIR_KERNEL)
613 return true;
614
615 // HLSL handling: special attribute are emitted from the
616 // front-end.
617 if (F.getFnAttribute(Kind: "hlsl.shader").isValid())
618 return true;
619
620 return false;
621}
622
623Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
624 TypeName.consume_front(Prefix: "atomic_");
625 if (TypeName.consume_front(Prefix: "void"))
626 return Type::getVoidTy(C&: Ctx);
627 else if (TypeName.consume_front(Prefix: "bool") || TypeName.consume_front(Prefix: "_Bool"))
628 return Type::getIntNTy(C&: Ctx, N: 1);
629 else if (TypeName.consume_front(Prefix: "char") ||
630 TypeName.consume_front(Prefix: "signed char") ||
631 TypeName.consume_front(Prefix: "unsigned char") ||
632 TypeName.consume_front(Prefix: "uchar"))
633 return Type::getInt8Ty(C&: Ctx);
634 else if (TypeName.consume_front(Prefix: "short") ||
635 TypeName.consume_front(Prefix: "signed short") ||
636 TypeName.consume_front(Prefix: "unsigned short") ||
637 TypeName.consume_front(Prefix: "ushort"))
638 return Type::getInt16Ty(C&: Ctx);
639 else if (TypeName.consume_front(Prefix: "int") ||
640 TypeName.consume_front(Prefix: "signed int") ||
641 TypeName.consume_front(Prefix: "unsigned int") ||
642 TypeName.consume_front(Prefix: "uint"))
643 return Type::getInt32Ty(C&: Ctx);
644 else if (TypeName.consume_front(Prefix: "long") ||
645 TypeName.consume_front(Prefix: "signed long") ||
646 TypeName.consume_front(Prefix: "unsigned long") ||
647 TypeName.consume_front(Prefix: "ulong"))
648 return Type::getInt64Ty(C&: Ctx);
649 else if (TypeName.consume_front(Prefix: "half") ||
650 TypeName.consume_front(Prefix: "_Float16") ||
651 TypeName.consume_front(Prefix: "__fp16"))
652 return Type::getHalfTy(C&: Ctx);
653 else if (TypeName.consume_front(Prefix: "float"))
654 return Type::getFloatTy(C&: Ctx);
655 else if (TypeName.consume_front(Prefix: "double"))
656 return Type::getDoubleTy(C&: Ctx);
657
658 // Unable to recognize SPIRV type name
659 return nullptr;
660}
661
662SmallPtrSet<BasicBlock *, 0>
663PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
664 std::queue<BasicBlock *> ToVisit;
665 ToVisit.push(x: Start);
666
667 SmallPtrSet<BasicBlock *, 0> Output;
668 while (ToVisit.size() != 0) {
669 BasicBlock *BB = ToVisit.front();
670 ToVisit.pop();
671
672 if (Output.count(Ptr: BB) != 0)
673 continue;
674 Output.insert(Ptr: BB);
675
676 for (BasicBlock *Successor : successors(BB)) {
677 if (DT.dominates(A: Successor, B: BB))
678 continue;
679 ToVisit.push(x: Successor);
680 }
681 }
682
683 return Output;
684}
685
686bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
687 for (BasicBlock *P : predecessors(BB)) {
688 // Ignore back-edges.
689 if (DT.dominates(A: BB, B: P))
690 continue;
691
692 // One of the predecessor hasn't been visited. Not ready yet.
693 if (BlockToOrder.count(Val: P) == 0)
694 return false;
695
696 // If the block is a loop exit, the loop must be finished before
697 // we can continue.
698 Loop *L = LI.getLoopFor(BB: P);
699 if (L == nullptr || L->contains(BB))
700 continue;
701
702 // SPIR-V requires a single back-edge. And the backend first
703 // step transforms loops into the simplified format. If we have
704 // more than 1 back-edge, something is wrong.
705 assert(L->getNumBackEdges() <= 1);
706
707 // If the loop has no latch, loop's rank won't matter, so we can
708 // proceed.
709 BasicBlock *Latch = L->getLoopLatch();
710 assert(Latch);
711 if (Latch == nullptr)
712 continue;
713
714 // The latch is not ready yet, let's wait.
715 if (BlockToOrder.count(Val: Latch) == 0)
716 return false;
717 }
718
719 return true;
720}
721
722size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
723 auto It = BlockToOrder.find(Val: BB);
724 if (It != BlockToOrder.end())
725 return It->second.Rank;
726
727 size_t result = 0;
728 for (BasicBlock *P : predecessors(BB)) {
729 // Ignore back-edges.
730 if (DT.dominates(A: BB, B: P))
731 continue;
732
733 auto Iterator = BlockToOrder.end();
734 Loop *L = LI.getLoopFor(BB: P);
735 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
736
737 // If the predecessor is either outside a loop, or part of
738 // the same loop, simply take its rank + 1.
739 if (L == nullptr || L->contains(BB) || Latch == nullptr) {
740 Iterator = BlockToOrder.find(Val: P);
741 } else {
742 // Otherwise, take the loop's rank (highest rank in the loop) as base.
743 // Since loops have a single latch, highest rank is easy to find.
744 // If the loop has no latch, then it doesn't matter.
745 Iterator = BlockToOrder.find(Val: Latch);
746 }
747
748 assert(Iterator != BlockToOrder.end());
749 result = std::max(a: result, b: Iterator->second.Rank + 1);
750 }
751
752 return result;
753}
754
755size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
756 ToVisit.push(x: BB);
757 Queued.insert(Ptr: BB);
758
759 size_t QueueIndex = 0;
760 while (ToVisit.size() != 0) {
761 BasicBlock *BB = ToVisit.front();
762 ToVisit.pop();
763
764 if (!CanBeVisited(BB)) {
765 ToVisit.push(x: BB);
766 if (QueueIndex >= ToVisit.size())
767 llvm::report_fatal_error(
768 reason: "No valid candidate in the queue. Is the graph reducible?");
769 QueueIndex++;
770 continue;
771 }
772
773 QueueIndex = 0;
774 size_t Rank = GetNodeRank(BB);
775 OrderInfo Info = {.Rank: Rank, .TraversalIndex: BlockToOrder.size()};
776 BlockToOrder.try_emplace(Key: BB, Args&: Info);
777
778 for (BasicBlock *S : successors(BB)) {
779 if (Queued.count(Ptr: S) != 0)
780 continue;
781 ToVisit.push(x: S);
782 Queued.insert(Ptr: S);
783 }
784 }
785
786 return 0;
787}
788
789PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
790 DT.recalculate(Func&: F);
791 LI = LoopInfo(DT);
792
793 visit(BB: &*F.begin(), Unused: 0);
794
795 Order.reserve(n: F.size());
796 for (auto &[BB, Info] : BlockToOrder)
797 Order.emplace_back(args&: BB);
798
799 llvm::sort(C&: Order, Comp: [&](const auto &LHS, const auto &RHS) {
800 return compare(LHS, RHS);
801 });
802}
803
804bool PartialOrderingVisitor::compare(const BasicBlock *LHS,
805 const BasicBlock *RHS) const {
806 const OrderInfo &InfoLHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(LHS));
807 const OrderInfo &InfoRHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(RHS));
808 if (InfoLHS.Rank != InfoRHS.Rank)
809 return InfoLHS.Rank < InfoRHS.Rank;
810 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;
811}
812
813void PartialOrderingVisitor::partialOrderVisit(
814 BasicBlock &Start, std::function<bool(BasicBlock *)> Op) {
815 SmallPtrSet<BasicBlock *, 0> Reachable = getReachableFrom(Start: &Start);
816 assert(BlockToOrder.count(&Start) != 0);
817
818 // Skipping blocks with a rank inferior to |Start|'s rank.
819 auto It = Order.begin();
820 while (It != Order.end() && *It != &Start)
821 ++It;
822
823 // This is unexpected. Worst case |Start| is the last block,
824 // so It should point to the last block, not past-end.
825 assert(It != Order.end());
826
827 // By default, there is no rank limit. Setting it to the maximum value.
828 std::optional<size_t> EndRank = std::nullopt;
829 for (; It != Order.end(); ++It) {
830 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)
831 break;
832
833 if (Reachable.count(Ptr: *It) == 0) {
834 continue;
835 }
836
837 if (!Op(*It)) {
838 EndRank = BlockToOrder[*It].Rank;
839 }
840 }
841}
842
843bool sortBlocks(Function &F) {
844 if (F.size() == 0)
845 return false;
846
847 bool Modified = false;
848 std::vector<BasicBlock *> Order;
849 Order.reserve(n: F.size());
850
851 ReversePostOrderTraversal<Function *> RPOT(&F);
852 llvm::append_range(C&: Order, R&: RPOT);
853
854 assert(&*F.begin() == Order[0]);
855 BasicBlock *LastBlock = &*F.begin();
856 for (BasicBlock *BB : Order) {
857 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {
858 Modified = true;
859 BB->moveAfter(MovePos: LastBlock);
860 }
861 LastBlock = BB;
862 }
863
864 return Modified;
865}
866
867AllocaInst *createVariable(Function &F, Type *Type) {
868 const DataLayout &DL = F.getDataLayout();
869 return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
870 F.begin()->getFirstInsertionPt());
871}
872
873Value *
874createExitVariable(BasicBlock *BB,
875 const DenseMap<BasicBlock *, ConstantInt *> &TargetToValue) {
876 auto *T = BB->getTerminator();
877 if (isa<ReturnInst>(Val: T))
878 return nullptr;
879 if (auto *BI = dyn_cast<UncondBrInst>(Val: T))
880 return TargetToValue.lookup(Val: BI->getSuccessor());
881
882 IRBuilder<> Builder(BB);
883 Builder.SetInsertPoint(T);
884
885 if (auto *BI = dyn_cast<CondBrInst>(Val: T)) {
886 Value *LHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 0));
887 Value *RHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 1));
888
889 if (LHS == nullptr || RHS == nullptr)
890 return LHS == nullptr ? RHS : LHS;
891 return Builder.CreateSelect(C: BI->getCondition(), True: LHS, False: RHS);
892 }
893
894 // TODO: add support for switch cases.
895 llvm_unreachable("Unhandled terminator type.");
896}
897
898MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
899 MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
900 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
901 MaybeDef = MRI.getVRegDef(Reg: MaybeDef->getOperand(i: 1).getReg());
902 return MaybeDef;
903}
904
905static bool getVacantFunctionName(Module &M, std::string &Name) {
906 // It's a bit of paranoia, but still we don't want to have even a chance that
907 // the loop will work for too long.
908 constexpr unsigned MaxIters = 1024;
909 for (unsigned I = 0; I < MaxIters; ++I) {
910 std::string OrdName = Name + Twine(I).str();
911 if (!M.getFunction(Name: OrdName)) {
912 Name = std::move(OrdName);
913 return true;
914 }
915 }
916 return false;
917}
918
919// Assign SPIR-V type to the register. If the register has no valid assigned
920// class, set register LLT type and class according to the SPIR-V type.
921void setRegClassType(Register Reg, SPIRVTypeInst SpvType,
922 SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
923 const MachineFunction &MF, bool Force) {
924 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
925 if (!MRI->getRegClassOrNull(Reg) || Force) {
926 MRI->setRegClass(Reg, RC: GR->getRegClass(SpvType));
927 LLT RegType = GR->getRegType(SpvType);
928 if (Force || !MRI->getType(Reg).isValid())
929 MRI->setType(VReg: Reg, Ty: RegType);
930 }
931}
932
933// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
934// no valid assigned class, set register LLT type and class according to the
935// SPIR-V type.
936void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
937 MachineIRBuilder &MIRBuilder,
938 SPIRV::AccessQualifier::AccessQualifier AccessQual,
939 bool EmitIR, bool Force) {
940 setRegClassType(Reg,
941 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR),
942 GR, MRI: MIRBuilder.getMRI(), MF: MIRBuilder.getMF(), Force);
943}
944
945// Create a virtual register and assign SPIR-V type to the register. Set
946// register LLT type and class according to the SPIR-V type.
947Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
948 MachineRegisterInfo *MRI,
949 const MachineFunction &MF) {
950 Register Reg = MRI->createVirtualRegister(RegClass: GR->getRegClass(SpvType));
951 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
952 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
953 return Reg;
954}
955
956// Create a virtual register and assign SPIR-V type to the register. Set
957// register LLT type and class according to the SPIR-V type.
958Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
959 MachineIRBuilder &MIRBuilder) {
960 return createVirtualRegister(SpvType, GR, MRI: MIRBuilder.getMRI(),
961 MF: MIRBuilder.getMF());
962}
963
964// Create a SPIR-V type, virtual register and assign SPIR-V type to the
965// register. Set register LLT type and class according to the SPIR-V type.
966Register createVirtualRegister(
967 const Type *Ty, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIRBuilder,
968 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
969 return createVirtualRegister(
970 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR), GR,
971 MIRBuilder);
972}
973
974CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
975 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
976 IRBuilder<> &B) {
977 SmallVector<Value *, 4> Args;
978 Args.push_back(Elt: Arg2);
979 Args.push_back(Elt: buildMD(Arg));
980 llvm::append_range(C&: Args, R&: Imms);
981 return B.CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: {Types}, Args);
982}
983
984// Return true if there is an opaque pointer type nested in the argument.
985bool isNestedPointer(const Type *Ty) {
986 if (Ty->isPtrOrPtrVectorTy())
987 return true;
988 if (const FunctionType *RefTy = dyn_cast<FunctionType>(Val: Ty)) {
989 if (isNestedPointer(Ty: RefTy->getReturnType()))
990 return true;
991 for (const Type *ArgTy : RefTy->params())
992 if (isNestedPointer(Ty: ArgTy))
993 return true;
994 return false;
995 }
996 if (const ArrayType *RefTy = dyn_cast<ArrayType>(Val: Ty))
997 return isNestedPointer(Ty: RefTy->getElementType());
998 return false;
999}
1000
1001bool isSpvIntrinsic(const Value *Arg) {
1002 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Arg))
1003 if (Function *F = II->getCalledFunction())
1004 if (F->getName().starts_with(Prefix: "llvm.spv."))
1005 return true;
1006 return false;
1007}
1008
1009// Function to create continued instructions for SPV_INTEL_long_composites
1010// extension
1011SmallVector<MachineInstr *, 4>
1012createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode,
1013 unsigned MinWC, unsigned ContinuedOpcode,
1014 ArrayRef<Register> Args, Register ReturnRegister,
1015 Register TypeID) {
1016
1017 SmallVector<MachineInstr *, 4> Instructions;
1018 constexpr unsigned MaxWordCount = UINT16_MAX;
1019 const size_t NumElements = Args.size();
1020 size_t MaxNumElements = MaxWordCount - MinWC;
1021 size_t SPIRVStructNumElements = NumElements;
1022
1023 if (NumElements > MaxNumElements) {
1024 // Do adjustments for continued instructions which always had only one
1025 // minumum word count.
1026 SPIRVStructNumElements = MaxNumElements;
1027 MaxNumElements = MaxWordCount - 1;
1028 }
1029
1030 auto MIB =
1031 MIRBuilder.buildInstr(Opcode).addDef(RegNo: ReturnRegister).addUse(RegNo: TypeID);
1032
1033 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1034 MIB.addUse(RegNo: Args[I]);
1035
1036 Instructions.push_back(Elt: MIB.getInstr());
1037
1038 for (size_t I = SPIRVStructNumElements; I < NumElements;
1039 I += MaxNumElements) {
1040 auto MIB = MIRBuilder.buildInstr(Opcode: ContinuedOpcode);
1041 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
1042 MIB.addUse(RegNo: Args[J]);
1043 Instructions.push_back(Elt: MIB.getInstr());
1044 }
1045 return Instructions;
1046}
1047
1048SmallVector<unsigned, 1>
1049getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD) {
1050 unsigned LC = SPIRV::LoopControl::None;
1051 // Currently used only to store PartialCount value. Later when other
1052 // LoopControls are added - this map should be sorted before making
1053 // them loop_merge operands to satisfy 3.23. Loop Control requirements.
1054 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;
1055 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.disable")) {
1056 LC |= SPIRV::LoopControl::DontUnroll;
1057 } else {
1058 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.enable") ||
1059 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.full")) {
1060 LC |= SPIRV::LoopControl::Unroll;
1061 }
1062 if (MDNode *CountMD =
1063 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.count")) {
1064 if (auto *CI =
1065 mdconst::extract_or_null<ConstantInt>(MD: CountMD->getOperand(I: 1))) {
1066 unsigned Count = CI->getZExtValue();
1067 if (Count != 1) {
1068 LC |= SPIRV::LoopControl::PartialCount;
1069 MaskToValueMap.emplace_back(
1070 args: std::make_pair(x: SPIRV::LoopControl::PartialCount, y&: Count));
1071 }
1072 }
1073 }
1074 }
1075 SmallVector<unsigned, 1> Result = {LC};
1076 for (auto &[Mask, Val] : MaskToValueMap)
1077 Result.push_back(Elt: Val);
1078 return Result;
1079}
1080
1081SmallVector<unsigned, 1> getSpirvLoopControlOperandsFromLoopMetadata(Loop *L) {
1082 return getSpirvLoopControlOperandsFromLoopMetadata(LoopMD: L->getLoopID());
1083}
1084
1085const std::set<unsigned> &getTypeFoldingSupportedOpcodes() {
1086 // clang-format off
1087 static const std::set<unsigned> TypeFoldingSupportingOpcs = {
1088 TargetOpcode::G_ADD,
1089 TargetOpcode::G_FADD,
1090 TargetOpcode::G_STRICT_FADD,
1091 TargetOpcode::G_SUB,
1092 TargetOpcode::G_FSUB,
1093 TargetOpcode::G_STRICT_FSUB,
1094 TargetOpcode::G_MUL,
1095 TargetOpcode::G_FMUL,
1096 TargetOpcode::G_STRICT_FMUL,
1097 TargetOpcode::G_SDIV,
1098 TargetOpcode::G_UDIV,
1099 TargetOpcode::G_FDIV,
1100 TargetOpcode::G_STRICT_FDIV,
1101 TargetOpcode::G_SREM,
1102 TargetOpcode::G_UREM,
1103 TargetOpcode::G_FREM,
1104 TargetOpcode::G_STRICT_FREM,
1105 TargetOpcode::G_FNEG,
1106 TargetOpcode::G_CONSTANT,
1107 TargetOpcode::G_FCONSTANT,
1108 TargetOpcode::G_AND,
1109 TargetOpcode::G_OR,
1110 TargetOpcode::G_XOR,
1111 TargetOpcode::G_SHL,
1112 TargetOpcode::G_ASHR,
1113 TargetOpcode::G_LSHR,
1114 TargetOpcode::G_SELECT,
1115 TargetOpcode::G_EXTRACT_VECTOR_ELT,
1116 };
1117 // clang-format on
1118 return TypeFoldingSupportingOpcs;
1119}
1120
1121bool isTypeFoldingSupported(unsigned Opcode) {
1122 return getTypeFoldingSupportedOpcodes().count(x: Opcode) > 0;
1123}
1124
1125// Traversing [g]MIR accounting for pseudo-instructions.
1126MachineInstr *passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI) {
1127 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||
1128 Def->getOpcode() == TargetOpcode::COPY)
1129 ? MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())
1130 : Def;
1131}
1132
1133MachineInstr *getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1134 if (MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg()))
1135 return passCopy(Def, MRI);
1136 return nullptr;
1137}
1138
1139MachineInstr *getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1140 if (MachineInstr *Def = getDef(MO, MRI)) {
1141 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
1142 Def->getOpcode() == SPIRV::OpConstantI)
1143 return Def;
1144 }
1145 return nullptr;
1146}
1147
1148int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1149 if (MachineInstr *Def = getImm(MO, MRI)) {
1150 if (Def->getOpcode() == SPIRV::OpConstantI)
1151 return Def->getOperand(i: 2).getImm();
1152 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1153 return Def->getOperand(i: 1).getCImm()->getZExtValue();
1154 }
1155 llvm_unreachable("Unexpected integer constant pattern");
1156}
1157
1158unsigned getArrayComponentCount(const MachineRegisterInfo *MRI,
1159 const MachineInstr *ResType) {
1160 return foldImm(MO: ResType->getOperand(i: 2), MRI);
1161}
1162
1163bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType,
1164 uint64_t &TotalSize) {
1165 // An array of N padded structs is represented as {[N-1 x <{T, pad}>], T}.
1166 if (Ty->getStructNumElements() != 2)
1167 return false;
1168
1169 Type *FirstElement = Ty->getStructElementType(N: 0);
1170 Type *SecondElement = Ty->getStructElementType(N: 1);
1171
1172 if (!FirstElement->isArrayTy())
1173 return false;
1174
1175 Type *ArrayElementType = FirstElement->getArrayElementType();
1176 if (!ArrayElementType->isStructTy() ||
1177 ArrayElementType->getStructNumElements() != 2)
1178 return false;
1179
1180 Type *T_in_struct = ArrayElementType->getStructElementType(N: 0);
1181 if (T_in_struct != SecondElement)
1182 return false;
1183
1184 auto *Padding_in_struct =
1185 dyn_cast<TargetExtType>(Val: ArrayElementType->getStructElementType(N: 1));
1186 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")
1187 return false;
1188
1189 const uint64_t ArraySize = FirstElement->getArrayNumElements();
1190 TotalSize = ArraySize + 1;
1191 OriginalElementType = ArrayElementType;
1192 return true;
1193}
1194
1195Type *reconstitutePeeledArrayType(Type *Ty) {
1196 if (!Ty->isStructTy())
1197 return Ty;
1198
1199 auto *STy = cast<StructType>(Val: Ty);
1200 Type *OriginalElementType = nullptr;
1201 uint64_t TotalSize = 0;
1202 if (matchPeeledArrayPattern(Ty: STy, OriginalElementType, TotalSize)) {
1203 Type *ResultTy = ArrayType::get(
1204 ElementType: reconstitutePeeledArrayType(Ty: OriginalElementType), NumElements: TotalSize);
1205 return ResultTy;
1206 }
1207
1208 SmallVector<Type *, 4> NewElementTypes;
1209 bool Changed = false;
1210 for (Type *ElementTy : STy->elements()) {
1211 Type *NewElementTy = reconstitutePeeledArrayType(Ty: ElementTy);
1212 if (NewElementTy != ElementTy)
1213 Changed = true;
1214 NewElementTypes.push_back(Elt: NewElementTy);
1215 }
1216
1217 if (!Changed)
1218 return Ty;
1219
1220 Type *ResultTy;
1221 if (STy->isLiteral())
1222 ResultTy =
1223 StructType::get(Context&: STy->getContext(), Elements: NewElementTypes, isPacked: STy->isPacked());
1224 else {
1225 auto *NewTy = StructType::create(Context&: STy->getContext(), Name: STy->getName());
1226 NewTy->setBody(Elements: NewElementTypes, isPacked: STy->isPacked());
1227 ResultTy = NewTy;
1228 }
1229 return ResultTy;
1230}
1231
1232std::optional<SPIRV::LinkageType::LinkageType>
1233getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV) {
1234 if (GV.hasLocalLinkage())
1235 return std::nullopt;
1236
1237 if (GV.isDeclarationForLinker()) {
1238 // Interface variables must not get Import linkage.
1239 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: &GV)) {
1240 auto SC = addressSpaceToStorageClass(AddrSpace: GVar->getAddressSpace(), STI: ST);
1241 if (SC == SPIRV::StorageClass::Input ||
1242 SC == SPIRV::StorageClass::Output ||
1243 SC == SPIRV::StorageClass::PushConstant)
1244 return std::nullopt;
1245 }
1246 return SPIRV::LinkageType::Import;
1247 }
1248
1249 if (GV.hasHiddenVisibility())
1250 return std::nullopt;
1251
1252 if (GV.hasLinkOnceODRLinkage() &&
1253 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr))
1254 return SPIRV::LinkageType::LinkOnceODR;
1255
1256 if (GV.hasWeakLinkage() &&
1257 ST.canUseExtension(E: SPIRV::Extension::SPV_AMD_weak_linkage))
1258 return SPIRV::LinkageType::WeakAMD;
1259
1260 return SPIRV::LinkageType::Export;
1261}
1262
1263Function *getOrCreateBackendServiceFunction(Module &M) {
1264 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
1265 if (!getVacantFunctionName(M, Name&: ServiceFunName))
1266 report_fatal_error(
1267 reason: "cannot allocate a name for the internal service function");
1268 if (Function *SF = M.getFunction(Name: ServiceFunName)) {
1269 if (SF->getInstructionCount() > 0)
1270 report_fatal_error(
1271 reason: "Unexpected combination of global variables and function pointers");
1272 return SF;
1273 }
1274 Function *SF = Function::Create(
1275 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: {}, isVarArg: false),
1276 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
1277 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
1278 return SF;
1279}
1280
1281} // namespace llvm
1282