1//===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVUtils.h"
14#include "MCTargetDesc/SPIRVBaseInfo.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRVInstrInfo.h"
19#include "SPIRVSubtarget.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/Demangle/Demangle.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/TargetParser/AtomicScope.h"
31#include <queue>
32#include <vector>
33
34namespace llvm {
35namespace SPIRV {
36static MDNode *findNamedMDOperand(NamedMDNode *NMD, StringRef Name) {
37 auto It = find_if(Range: NMD->operands(), P: [Name](MDNode *N) {
38 if (auto *MDS = dyn_cast_or_null<MDString>(Val: N->getOperand(I: 0)))
39 return MDS->getString() == Name;
40 return false;
41 });
42 return It == NMD->op_end() ? nullptr : *It;
43}
44
45// This code restores function args/retvalue types for composite cases
46// because the final types should still be aggregate whereas they're i32
47// during the translation to cope with aggregate flattening etc.
48// TODO: should these just return nullptr when there's no metadata?
49static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
50 FunctionType *FTy,
51 StringRef Name) {
52 if (!NMD)
53 return FTy;
54
55 MDNode *Match = findNamedMDOperand(NMD, Name);
56 if (!Match)
57 return FTy;
58
59 Type *RetTy = FTy->getReturnType();
60 SmallVector<Type *, 4> PTys(FTy->params());
61
62 for (unsigned I = 1; I != Match->getNumOperands(); ++I) {
63 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I));
64 assert(MD && "MDNode operand is expected");
65
66 if (auto *Const = getMDOperandAsConstInt(N: MD, I: 0)) {
67 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1));
68 assert(CMeta && "ConstantAsMetadata operand is expected");
69 int64_t Idx = Const->getSExtValue();
70 // Currently -1 indicates return value, greater values mean
71 // argument numbers.
72 if (Idx == -1) {
73 RetTy = CMeta->getType();
74 continue;
75 }
76 if (Idx >= 0 && static_cast<uint64_t>(Idx) < PTys.size()) {
77 PTys[Idx] = CMeta->getType();
78 continue;
79 }
80 report_fatal_error(reason: "invalid argument index in function type metadata");
81 }
82 }
83
84 return FunctionType::get(Result: RetTy, Params: PTys, isVarArg: FTy->isVarArg());
85}
86
87static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
88 StringRef Constraints,
89 StringRef Name) {
90 if (!NMD)
91 return Constraints;
92
93 MDNode *Match = findNamedMDOperand(NMD, Name);
94 if (!Match)
95 return Constraints;
96
97 // By convention, the constraints string is stored in the final MD operand.
98 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I: Match->getNumOperands() - 1));
99 assert(MD && "MDNode operand is expected");
100
101 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
102 Constraints = MDS->getString();
103
104 return Constraints;
105}
106
107FunctionType *getOriginalFunctionType(const Function &F) {
108 return extractFunctionTypeFromMetadata(
109 NMD: F.getParent()->getNamedMetadata(Name: "spv.cloned_funcs"), FTy: F.getFunctionType(),
110 Name: F.getName());
111}
112
113// Keyed via instruction metadata, not a name.
114static std::optional<StringRef> getMutatedCallsiteKey(const CallBase &CB) {
115 if (MDNode *MD = CB.getMetadata(Kind: "spv.mutated_callsite"))
116 if (MD->getNumOperands() > 0)
117 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
118 return MDS->getString();
119 return std::nullopt;
120}
121
122FunctionType *getOriginalFunctionType(const CallBase &CB) {
123 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
124 if (!Key)
125 return CB.getFunctionType();
126 return extractFunctionTypeFromMetadata(
127 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"),
128 FTy: CB.getFunctionType(), Name: *Key);
129}
130
131StringRef getOriginalAsmConstraints(const CallBase &CB) {
132 StringRef Constraints =
133 cast<InlineAsm>(Val: CB.getCalledOperand())->getConstraintString();
134 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
135 if (!Key)
136 return Constraints;
137 return extractAsmConstraintsFromMetadata(
138 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"), Constraints,
139 Name: *Key);
140}
141} // Namespace SPIRV
142
143// The following functions are used to add these string literals as a series of
144// 32-bit integer operands with the correct format, and unpack them if necessary
145// when making string comparisons in compiler passes.
146// SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment.
147static uint32_t convertCharsToWord(StringRef Str, unsigned i) {
148 uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars.
149 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {
150 unsigned StrIndex = i + WordIndex;
151 uint8_t CharToAdd = 0; // Initilize char as padding/null.
152 if (StrIndex < Str.size()) { // If it's within the string, get a real char.
153 CharToAdd = Str[StrIndex];
154 }
155 Word |= (CharToAdd << (WordIndex * 8));
156 }
157 return Word;
158}
159
160// Get length including padding and null terminator.
161static size_t getPaddedLen(StringRef Str) { return alignTo(Value: Str.size() + 1, Align: 4); }
162
163void addStringImm(StringRef Str, MCInst &Inst) {
164 const size_t PaddedLen = getPaddedLen(Str);
165 for (unsigned i = 0; i < PaddedLen; i += 4) {
166 // Add an operand for the 32-bits of chars or padding.
167 Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i)));
168 }
169}
170
171void addStringImm(StringRef Str, MachineInstrBuilder &MIB) {
172 const size_t PaddedLen = getPaddedLen(Str);
173 for (unsigned i = 0; i < PaddedLen; i += 4) {
174 // Add an operand for the 32-bits of chars or padding.
175 MIB.addImm(Val: convertCharsToWord(Str, i));
176 }
177}
178
179std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
180 return getSPIRVStringOperand(MI, StartIndex);
181}
182
183std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
184 MachineInstr *Def = getVRegDef(MRI, Reg);
185 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
186 "Expected G_GLOBAL_VALUE");
187 const GlobalValue *GV = Def->getOperand(i: 1).getGlobal();
188 Value *V = GV->getOperand(i: 0);
189 const ConstantDataArray *CDA = cast<ConstantDataArray>(Val: V);
190 return CDA->getAsCString().str();
191}
192
193void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
194 const auto Bitwidth = Imm.getBitWidth();
195 if (Bitwidth == 1)
196 return; // Already handled
197 else if (Bitwidth <= 32) {
198 MIB.addImm(Val: Imm.getZExtValue());
199 // Asm Printer needs this info to print floating-type correctly
200 if (Bitwidth == 16)
201 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16);
202 return;
203 } else if (Bitwidth <= 64) {
204 uint64_t FullImm = Imm.getZExtValue();
205 MIB.addImm(Val: Lo_32(Value: FullImm)).addImm(Val: Hi_32(Value: FullImm));
206 // Asm Printer needs this info to print 64-bit operands correctly
207 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH64);
208 return;
209 } else {
210 // Emit ceil(Bitwidth / 32) words to conform SPIR-V spec.
211 unsigned NumWords = divideCeil(Numerator: Bitwidth, Denominator: 32);
212 for (unsigned I = 0; I < NumWords; ++I) {
213 unsigned LimbIdx = I / 2;
214 unsigned LimbShift = (I % 2) * 32;
215 uint32_t Word = (Imm.getRawData()[LimbIdx] >> LimbShift) & 0xffffffff;
216 MIB.addImm(Val: Word);
217 }
218 return;
219 }
220}
221
222void buildOpName(Register Target, StringRef Name,
223 MachineIRBuilder &MIRBuilder) {
224 if (!Name.empty()) {
225 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpName).addUse(RegNo: Target);
226 addStringImm(Str: Name, MIB);
227 }
228}
229
230static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
231 ArrayRef<uint32_t> DecArgs,
232 StringRef StrImm) {
233 if (!StrImm.empty())
234 addStringImm(Str: StrImm, MIB);
235 for (const auto &DecArg : DecArgs)
236 MIB.addImm(Val: DecArg);
237}
238
239void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
240 SPIRV::Decoration::Decoration Dec,
241 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
242 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
243 .addUse(RegNo: Reg)
244 .addImm(Val: static_cast<uint32_t>(Dec));
245 finishBuildOpDecorate(MIB, DecArgs, StrImm);
246}
247
248void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
249 SPIRV::Decoration::Decoration Dec,
250 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
251 MachineBasicBlock &MBB = *I.getParent();
252 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDecorate))
253 .addUse(RegNo: Reg)
254 .addImm(Val: static_cast<uint32_t>(Dec));
255 finishBuildOpDecorate(MIB, DecArgs, StrImm);
256}
257
258void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
259 SPIRV::Decoration::Decoration Dec, uint32_t Member,
260 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
261 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpMemberDecorate)
262 .addUse(RegNo: Reg)
263 .addImm(Val: Member)
264 .addImm(Val: static_cast<uint32_t>(Dec));
265 finishBuildOpDecorate(MIB, DecArgs, StrImm);
266}
267
268void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
269 const MDNode *GVarMD, const SPIRVSubtarget &ST) {
270 for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
271 auto *OpMD = dyn_cast<MDNode>(Val: GVarMD->getOperand(I));
272 if (!OpMD)
273 report_fatal_error(reason: "Invalid decoration");
274 if (OpMD->getNumOperands() == 0)
275 report_fatal_error(reason: "Expect operand(s) of the decoration");
276 ConstantInt *DecorationId =
277 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 0));
278 if (!DecorationId)
279 report_fatal_error(reason: "Expect SPIR-V <Decoration> operand to be the first "
280 "element of the decoration");
281
282 // The goal of `spirv.Decorations` metadata is to provide a way to
283 // represent SPIR-V entities that do not map to LLVM in an obvious way.
284 // FP flags do have obvious matches between LLVM IR and SPIR-V.
285 // Additionally, we have no guarantee at this point that the flags passed
286 // through the decoration are not violated already in the optimizer passes.
287 // Therefore, we simply ignore FP flags, including NoContraction, and
288 // FPFastMathMode.
289 if (DecorationId->getZExtValue() ==
290 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||
291 DecorationId->getZExtValue() ==
292 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {
293 continue; // Ignored.
294 }
295 uint32_t Dec = static_cast<uint32_t>(DecorationId->getZExtValue());
296 if (Dec == static_cast<uint32_t>(SPIRV::Decoration::UniformId) ||
297 Dec == static_cast<uint32_t>(SPIRV::Decoration::AlignmentId) ||
298 Dec == static_cast<uint32_t>(SPIRV::Decoration::MaxByteOffsetId)) {
299 ConstantInt *IdV =
300 OpMD->getNumOperands() == 2
301 ? mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 1))
302 : nullptr;
303 if (!IdV || !isUInt<32>(x: IdV->getZExtValue()))
304 report_fatal_error(reason: "Expect a single integer <id> operand of the "
305 "decoration");
306 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
307 SPIRVTypeInst SpvTypeInt32 =
308 GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
309 Register IdReg = GR->buildConstantInt(Val: IdV->getZExtValue(), MIRBuilder,
310 SpvType: SpvTypeInt32, /*EmitIR=*/false);
311 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorateId)
312 .addUse(RegNo: Reg)
313 .addImm(Val: Dec)
314 .addUse(RegNo: IdReg);
315 continue;
316 }
317 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate).addUse(RegNo: Reg).addImm(Val: Dec);
318 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
319 if (ConstantInt *OpV =
320 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: OpI)))
321 MIB.addImm(Val: static_cast<uint32_t>(OpV->getZExtValue()));
322 else if (MDString *OpV = dyn_cast<MDString>(Val: OpMD->getOperand(I: OpI)))
323 addStringImm(Str: OpV->getString(), MIB);
324 else
325 report_fatal_error(reason: "Unexpected operand of the decoration");
326 }
327 }
328}
329
330MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF) {
331 MachineBasicBlock &MBB = MF.front();
332 // Find the position to insert the OpVariable instruction.
333 // We will insert it after the last OpFunctionParameter, if any, or
334 // after OpFunction otherwise.
335 auto IsPreamble = [](const MachineInstr &MI) {
336 switch (MI.getOpcode()) {
337 case SPIRV::OpFunction:
338 case SPIRV::OpFunctionParameter:
339 case SPIRV::OpLabel:
340 case SPIRV::ASSIGN_TYPE:
341 return true;
342 default:
343 return false;
344 }
345 };
346 MachineBasicBlock::iterator VarPos = MBB.SkipPHIsAndLabels(I: MBB.begin());
347 while (VarPos != MBB.end() && VarPos->getOpcode() != SPIRV::OpFunction)
348 ++VarPos;
349 // Advance past the preamble.
350 while (VarPos != MBB.end() && IsPreamble(*VarPos))
351 ++VarPos;
352 return VarPos;
353}
354
355MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB) {
356 MachineBasicBlock::iterator I = MBB->end();
357 if (I == MBB->begin())
358 return I;
359 --I;
360 while (I->isTerminator() || I->isDebugValue()) {
361 if (I == MBB->begin())
362 break;
363 --I;
364 }
365 return I;
366}
367
368SPIRV::StorageClass::StorageClass
369addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
370 switch (AddrSpace) {
371 case 0:
372 return SPIRV::StorageClass::Function;
373 case 1:
374 return SPIRV::StorageClass::CrossWorkgroup;
375 case 2:
376 return SPIRV::StorageClass::UniformConstant;
377 case 3:
378 return SPIRV::StorageClass::Workgroup;
379 case 4:
380 return SPIRV::StorageClass::Generic;
381 case 5:
382 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
383 ? SPIRV::StorageClass::DeviceOnlyINTEL
384 : SPIRV::StorageClass::CrossWorkgroup;
385 case 6:
386 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
387 ? SPIRV::StorageClass::HostOnlyINTEL
388 : SPIRV::StorageClass::CrossWorkgroup;
389 case 7:
390 return SPIRV::StorageClass::Input;
391 case 8:
392 return SPIRV::StorageClass::Output;
393 case 9:
394 return SPIRV::StorageClass::CodeSectionINTEL;
395 case 10:
396 return SPIRV::StorageClass::Private;
397 case 11:
398 return SPIRV::StorageClass::StorageBuffer;
399 case 12:
400 return SPIRV::StorageClass::Uniform;
401 case 13:
402 return SPIRV::StorageClass::PushConstant;
403 default:
404 report_fatal_error(reason: "Unknown address space");
405 }
406}
407
408SPIRV::MemorySemantics::MemorySemantics
409getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) {
410 switch (SC) {
411 case SPIRV::StorageClass::StorageBuffer:
412 case SPIRV::StorageClass::Uniform:
413 return SPIRV::MemorySemantics::UniformMemory;
414 case SPIRV::StorageClass::Workgroup:
415 return SPIRV::MemorySemantics::WorkgroupMemory;
416 case SPIRV::StorageClass::CrossWorkgroup:
417 return SPIRV::MemorySemantics::CrossWorkgroupMemory;
418 case SPIRV::StorageClass::Generic:
419 return SPIRV::MemorySemantics::MemorySemantics(
420 SPIRV::MemorySemantics::WorkgroupMemory |
421 SPIRV::MemorySemantics::CrossWorkgroupMemory);
422 case SPIRV::StorageClass::AtomicCounter:
423 return SPIRV::MemorySemantics::AtomicCounterMemory;
424 case SPIRV::StorageClass::Image:
425 return SPIRV::MemorySemantics::ImageMemory;
426 default:
427 return SPIRV::MemorySemantics::None;
428 }
429}
430
431SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
432 switch (Ord) {
433 case AtomicOrdering::Acquire:
434 return SPIRV::MemorySemantics::Acquire;
435 case AtomicOrdering::Release:
436 return SPIRV::MemorySemantics::Release;
437 case AtomicOrdering::AcquireRelease:
438 return SPIRV::MemorySemantics::AcquireRelease;
439 case AtomicOrdering::SequentiallyConsistent:
440 return SPIRV::MemorySemantics::SequentiallyConsistent;
441 case AtomicOrdering::Unordered:
442 case AtomicOrdering::Monotonic:
443 case AtomicOrdering::NotAtomic:
444 return SPIRV::MemorySemantics::None;
445 }
446 llvm_unreachable(nullptr);
447}
448
449uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem,
450 uint32_t StorageClassSem) {
451 bool DropStorageClass =
452 TT.isVulkanOS() &&
453 OrderSem == static_cast<uint32_t>(SPIRV::MemorySemantics::None);
454 return OrderSem | (DropStorageClass ? 0 : StorageClassSem);
455}
456
457SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
458 SyncScope::ID Id) {
459 // Named by
460 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_scope_id.
461 // We don't need aliases for Invocation and CrossDevice, as we already have
462 // them covered by "singlethread" and "" strings respectively (see
463 // implementation of LLVMContext::LLVMContext()).
464 auto ScopeID = [&](AtomicScope Scope) {
465 return Ctx.getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(T: TT, S: Scope));
466 };
467 const llvm::SyncScope::ID SubGroup = ScopeID(AtomicScope::Wavefront);
468 const llvm::SyncScope::ID WorkGroup = ScopeID(AtomicScope::Workgroup);
469 const llvm::SyncScope::ID Device = ScopeID(AtomicScope::Device);
470
471 if (Id == llvm::SyncScope::SingleThread)
472 return SPIRV::Scope::Invocation;
473 else if (Id == llvm::SyncScope::System)
474 return SPIRV::Scope::CrossDevice;
475 else if (Id == SubGroup)
476 return SPIRV::Scope::Subgroup;
477 else if (Id == WorkGroup)
478 return SPIRV::Scope::Workgroup;
479 else if (Id == Device)
480 return SPIRV::Scope::Device;
481 return SPIRV::Scope::CrossDevice;
482}
483
484MachineInstr *getDefInstrMaybeConstant(Register &ConstReg,
485 const MachineRegisterInfo *MRI) {
486 MachineInstr *MI = MRI->getVRegDef(Reg: ConstReg);
487 MachineInstr *ConstInstr =
488 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT
489 ? MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg())
490 : MI;
491 if (auto *GI = dyn_cast<GIntrinsic>(Val: ConstInstr)) {
492 if (GI->is(ID: Intrinsic::spv_track_constant)) {
493 ConstReg = ConstInstr->getOperand(i: 2).getReg();
494 return MRI->getVRegDef(Reg: ConstReg);
495 }
496 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {
497 ConstReg = ConstInstr->getOperand(i: 1).getReg();
498 return MRI->getVRegDef(Reg: ConstReg);
499 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||
500 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {
501 ConstReg = ConstInstr->getOperand(i: 0).getReg();
502 return ConstInstr;
503 }
504 return MRI->getVRegDef(Reg: ConstReg);
505}
506
507uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) {
508 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
509 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
510 return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
511}
512
513int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI) {
514 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
515 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
516 return MI->getOperand(i: 1).getCImm()->getSExtValue();
517}
518
519bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
520 if (const auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
521 return GI->is(ID: IntrinsicID);
522 return false;
523}
524
525Type *getMDOperandAsType(const MDNode *N, unsigned I) {
526 Type *ElementTy = cast<ValueAsMetadata>(Val: N->getOperand(I))->getType();
527 return toTypedPointer(Ty: ElementTy);
528}
529
530ConstantInt *getMDOperandAsConstInt(const MDNode *N, unsigned I) {
531 if (N->getNumOperands() <= I)
532 return nullptr;
533 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I)))
534 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
535 return nullptr;
536}
537
538static bool isEnqueueKernelBI(StringRef MangledName) {
539 return MangledName == "__enqueue_kernel_basic" ||
540 MangledName == "__enqueue_kernel_basic_events" ||
541 MangledName == "__enqueue_kernel_varargs" ||
542 MangledName == "__enqueue_kernel_events_varargs";
543}
544
545static bool isKernelQueryBI(StringRef MangledName) {
546 return MangledName == "__get_kernel_work_group_size_impl" ||
547 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
548 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
549 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
550}
551
552static bool isNonMangledOCLBuiltin(StringRef Name) {
553 if (!Name.starts_with(Prefix: "__"))
554 return false;
555
556 return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) ||
557 SPIRV::isPipeOrAddressSpaceCastBuiltin(Name) ||
558 Name == "__translate_sampler_initializer";
559}
560
561std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
562 bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
563 bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_");
564 bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_");
565 bool IsMangled = Name.starts_with(Prefix: "_Z");
566
567 // Otherwise use simple demangling to return the function name.
568 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
569 return Name.str();
570
571 // Try to use the itanium demangler.
572 if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) {
573 std::string Result = DemangledName;
574 free(ptr: DemangledName);
575 return Result;
576 }
577
578 // Autocheck C++, maybe need to do explicit check of the source language.
579 // OpenCL C++ built-ins are declared in cl namespace.
580 // TODO: consider using 'St' abbriviation for cl namespace mangling.
581 // Similar to ::std:: in C++.
582 size_t Start, Len = 0;
583 size_t DemangledNameLenStart = 2;
584 if (Name.starts_with(Prefix: "_ZN")) {
585 // Skip CV and ref qualifiers.
586 size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO", From: 3);
587 // All built-ins are in the ::cl:: namespace.
588 if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv")
589 return std::string();
590 DemangledNameLenStart = NameSpaceStart + 11;
591 }
592 Start = Name.find_first_not_of(Chars: "0123456789", From: DemangledNameLenStart);
593 bool Error = Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart)
594 .getAsInteger(Radix: 10, Result&: Len);
595 if (Error)
596 return std::string();
597 return Name.substr(Start, N: Len).str();
598}
599
600bool hasBuiltinTypePrefix(StringRef Name) {
601 if (Name.starts_with(Prefix: "opencl.") || Name.starts_with(Prefix: "ocl_") ||
602 Name.starts_with(Prefix: "spirv."))
603 return true;
604 return false;
605}
606
607bool isSpecialOpaqueType(const Type *Ty) {
608 if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Val: Ty))
609 return isTypedPointerWrapper(ExtTy)
610 ? false
611 : hasBuiltinTypePrefix(Name: ExtTy->getName());
612
613 return false;
614}
615
616bool isEntryPoint(const Function &F) {
617 // OpenCL handling: any function with the SPIR_KERNEL
618 // calling convention will be a potential entry point.
619 if (F.getCallingConv() == CallingConv::SPIR_KERNEL)
620 return true;
621
622 // HLSL handling: special attribute are emitted from the
623 // front-end.
624 if (F.getFnAttribute(Kind: "hlsl.shader").isValid())
625 return true;
626
627 return false;
628}
629
630Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
631 TypeName.consume_front(Prefix: "atomic_");
632 if (TypeName.consume_front(Prefix: "void"))
633 return Type::getVoidTy(C&: Ctx);
634 else if (TypeName.consume_front(Prefix: "bool") || TypeName.consume_front(Prefix: "_Bool"))
635 return Type::getIntNTy(C&: Ctx, N: 1);
636 else if (TypeName.consume_front(Prefix: "char") ||
637 TypeName.consume_front(Prefix: "signed char") ||
638 TypeName.consume_front(Prefix: "unsigned char") ||
639 TypeName.consume_front(Prefix: "uchar"))
640 return Type::getInt8Ty(C&: Ctx);
641 else if (TypeName.consume_front(Prefix: "short") ||
642 TypeName.consume_front(Prefix: "signed short") ||
643 TypeName.consume_front(Prefix: "unsigned short") ||
644 TypeName.consume_front(Prefix: "ushort"))
645 return Type::getInt16Ty(C&: Ctx);
646 else if (TypeName.consume_front(Prefix: "int") ||
647 TypeName.consume_front(Prefix: "signed int") ||
648 TypeName.consume_front(Prefix: "unsigned int") ||
649 TypeName.consume_front(Prefix: "uint"))
650 return Type::getInt32Ty(C&: Ctx);
651 else if (TypeName.consume_front(Prefix: "long") ||
652 TypeName.consume_front(Prefix: "signed long") ||
653 TypeName.consume_front(Prefix: "unsigned long") ||
654 TypeName.consume_front(Prefix: "ulong"))
655 return Type::getInt64Ty(C&: Ctx);
656 else if (TypeName.consume_front(Prefix: "half") ||
657 TypeName.consume_front(Prefix: "_Float16") ||
658 TypeName.consume_front(Prefix: "__fp16"))
659 return Type::getHalfTy(C&: Ctx);
660 else if (TypeName.consume_front(Prefix: "float"))
661 return Type::getFloatTy(C&: Ctx);
662 else if (TypeName.consume_front(Prefix: "double"))
663 return Type::getDoubleTy(C&: Ctx);
664
665 // Unable to recognize SPIRV type name
666 return nullptr;
667}
668
669SmallPtrSet<BasicBlock *, 0>
670PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
671 std::queue<BasicBlock *> ToVisit;
672 ToVisit.push(x: Start);
673
674 SmallPtrSet<BasicBlock *, 0> Output;
675 while (ToVisit.size() != 0) {
676 BasicBlock *BB = ToVisit.front();
677 ToVisit.pop();
678
679 if (Output.count(Ptr: BB) != 0)
680 continue;
681 Output.insert(Ptr: BB);
682
683 for (BasicBlock *Successor : successors(BB)) {
684 if (DT.dominates(A: Successor, B: BB))
685 continue;
686 ToVisit.push(x: Successor);
687 }
688 }
689
690 return Output;
691}
692
693bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
694 for (BasicBlock *P : predecessors(BB)) {
695 // Ignore back-edges.
696 if (DT.dominates(A: BB, B: P))
697 continue;
698
699 // One of the predecessor hasn't been visited. Not ready yet.
700 if (BlockToOrder.count(Val: P) == 0)
701 return false;
702
703 // If the block is a loop exit, the loop must be finished before
704 // we can continue.
705 Loop *L = LI.getLoopFor(BB: P);
706 if (L == nullptr || L->contains(BB))
707 continue;
708
709 // SPIR-V requires a single back-edge. And the backend first
710 // step transforms loops into the simplified format. If we have
711 // more than 1 back-edge, something is wrong.
712 assert(L->getNumBackEdges() <= 1);
713
714 // If the loop has no latch, loop's rank won't matter, so we can
715 // proceed.
716 BasicBlock *Latch = L->getLoopLatch();
717 assert(Latch);
718 if (Latch == nullptr)
719 continue;
720
721 // The latch is not ready yet, let's wait.
722 if (BlockToOrder.count(Val: Latch) == 0)
723 return false;
724 }
725
726 return true;
727}
728
729size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
730 auto It = BlockToOrder.find(Val: BB);
731 if (It != BlockToOrder.end())
732 return It->second.Rank;
733
734 size_t result = 0;
735 for (BasicBlock *P : predecessors(BB)) {
736 // Ignore back-edges.
737 if (DT.dominates(A: BB, B: P))
738 continue;
739
740 auto Iterator = BlockToOrder.end();
741 Loop *L = LI.getLoopFor(BB: P);
742 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
743
744 // If the predecessor is either outside a loop, or part of
745 // the same loop, simply take its rank + 1.
746 if (L == nullptr || L->contains(BB) || Latch == nullptr) {
747 Iterator = BlockToOrder.find(Val: P);
748 } else {
749 // Otherwise, take the loop's rank (highest rank in the loop) as base.
750 // Since loops have a single latch, highest rank is easy to find.
751 // If the loop has no latch, then it doesn't matter.
752 Iterator = BlockToOrder.find(Val: Latch);
753 }
754
755 assert(Iterator != BlockToOrder.end());
756 result = std::max(a: result, b: Iterator->second.Rank + 1);
757 }
758
759 return result;
760}
761
762size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
763 ToVisit.push(x: BB);
764 Queued.insert(Ptr: BB);
765
766 size_t QueueIndex = 0;
767 while (ToVisit.size() != 0) {
768 BasicBlock *BB = ToVisit.front();
769 ToVisit.pop();
770
771 if (!CanBeVisited(BB)) {
772 ToVisit.push(x: BB);
773 if (QueueIndex >= ToVisit.size())
774 llvm::report_fatal_error(
775 reason: "No valid candidate in the queue. Is the graph reducible?");
776 QueueIndex++;
777 continue;
778 }
779
780 QueueIndex = 0;
781 size_t Rank = GetNodeRank(BB);
782 OrderInfo Info = {.Rank: Rank, .TraversalIndex: BlockToOrder.size()};
783 BlockToOrder.try_emplace(Key: BB, Args&: Info);
784
785 for (BasicBlock *S : successors(BB)) {
786 if (Queued.count(Ptr: S) != 0)
787 continue;
788 ToVisit.push(x: S);
789 Queued.insert(Ptr: S);
790 }
791 }
792
793 return 0;
794}
795
796PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
797 DT.recalculate(Func&: F);
798 LI = LoopInfo(DT);
799
800 visit(BB: &*F.begin(), Unused: 0);
801
802 Order.reserve(n: F.size());
803 for (auto &[BB, Info] : BlockToOrder)
804 Order.emplace_back(args&: BB);
805
806 llvm::sort(C&: Order, Comp: [&](const auto &LHS, const auto &RHS) {
807 return compare(LHS, RHS);
808 });
809}
810
811bool PartialOrderingVisitor::compare(const BasicBlock *LHS,
812 const BasicBlock *RHS) const {
813 const OrderInfo &InfoLHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(LHS));
814 const OrderInfo &InfoRHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(RHS));
815 if (InfoLHS.Rank != InfoRHS.Rank)
816 return InfoLHS.Rank < InfoRHS.Rank;
817 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;
818}
819
820void PartialOrderingVisitor::partialOrderVisit(
821 BasicBlock &Start, std::function<bool(BasicBlock *)> Op) {
822 SmallPtrSet<BasicBlock *, 0> Reachable = getReachableFrom(Start: &Start);
823 assert(BlockToOrder.count(&Start) != 0);
824
825 // Skipping blocks with a rank inferior to |Start|'s rank.
826 auto It = Order.begin();
827 while (It != Order.end() && *It != &Start)
828 ++It;
829
830 // This is unexpected. Worst case |Start| is the last block,
831 // so It should point to the last block, not past-end.
832 assert(It != Order.end());
833
834 // By default, there is no rank limit. Setting it to the maximum value.
835 std::optional<size_t> EndRank = std::nullopt;
836 for (; It != Order.end(); ++It) {
837 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)
838 break;
839
840 if (Reachable.count(Ptr: *It) == 0) {
841 continue;
842 }
843
844 if (!Op(*It)) {
845 EndRank = BlockToOrder[*It].Rank;
846 }
847 }
848}
849
850bool sortBlocks(Function &F) {
851 if (F.size() == 0)
852 return false;
853
854 bool Modified = false;
855 std::vector<BasicBlock *> Order;
856 Order.reserve(n: F.size());
857
858 ReversePostOrderTraversal<Function *> RPOT(&F);
859 llvm::append_range(C&: Order, R&: RPOT);
860
861 assert(&*F.begin() == Order[0]);
862 BasicBlock *LastBlock = &*F.begin();
863 for (BasicBlock *BB : Order) {
864 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {
865 Modified = true;
866 BB->moveAfter(MovePos: LastBlock);
867 }
868 LastBlock = BB;
869 }
870
871 return Modified;
872}
873
874AllocaInst *createVariable(Function &F, Type *Type) {
875 const DataLayout &DL = F.getDataLayout();
876 return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
877 F.begin()->getFirstInsertionPt());
878}
879
880Value *
881createExitVariable(BasicBlock *BB,
882 const DenseMap<BasicBlock *, ConstantInt *> &TargetToValue) {
883 auto *T = BB->getTerminator();
884 if (isa<ReturnInst>(Val: T))
885 return nullptr;
886 if (auto *BI = dyn_cast<UncondBrInst>(Val: T))
887 return TargetToValue.lookup(Val: BI->getSuccessor());
888
889 IRBuilder<> Builder(BB);
890 Builder.SetInsertPoint(T);
891
892 if (auto *BI = dyn_cast<CondBrInst>(Val: T)) {
893 Value *LHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 0));
894 Value *RHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 1));
895
896 if (LHS == nullptr || RHS == nullptr)
897 return LHS == nullptr ? RHS : LHS;
898 return Builder.CreateSelect(C: BI->getCondition(), True: LHS, False: RHS);
899 }
900
901 if (auto *SI = dyn_cast<SwitchInst>(Val: T)) {
902 Value *Condition = SI->getCondition();
903 // The default destination acts as the fallback value of the select chain.
904 Value *Result = TargetToValue.lookup(Val: SI->getDefaultDest());
905 for (const auto &Case : SI->cases()) {
906 Value *CaseValue = TargetToValue.lookup(Val: Case.getCaseSuccessor());
907 // Successors that are internal to the region have no exit value.
908 if (CaseValue == nullptr)
909 continue;
910 // The first known exit value becomes the base of the select chain.
911 if (Result == nullptr) {
912 Result = CaseValue;
913 continue;
914 }
915 Value *Cmp = Builder.CreateICmpEQ(LHS: Condition, RHS: Case.getCaseValue());
916 Result = Builder.CreateSelect(C: Cmp, True: CaseValue, False: Result);
917 }
918 return Result;
919 }
920
921 llvm_unreachable("Unhandled terminator type.");
922}
923
924MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
925 MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
926 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
927 MaybeDef = MRI.getVRegDef(Reg: MaybeDef->getOperand(i: 1).getReg());
928 return MaybeDef;
929}
930
931static bool getVacantFunctionName(Module &M, std::string &Name) {
932 // It's a bit of paranoia, but still we don't want to have even a chance that
933 // the loop will work for too long.
934 constexpr unsigned MaxIters = 1024;
935 for (unsigned I = 0; I < MaxIters; ++I) {
936 std::string OrdName = Name + Twine(I).str();
937 if (!M.getFunction(Name: OrdName)) {
938 Name = std::move(OrdName);
939 return true;
940 }
941 }
942 return false;
943}
944
945// Assign SPIR-V type to the register. If the register has no valid assigned
946// class, set register LLT type and class according to the SPIR-V type.
947void setRegClassType(Register Reg, SPIRVTypeInst SpvType,
948 SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
949 const MachineFunction &MF, bool Force) {
950 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
951 if (!MRI->getRegClassOrNull(Reg) || Force) {
952 MRI->setRegClass(Reg, RC: GR->getRegClass(SpvType));
953 LLT RegType = GR->getRegType(SpvType);
954 if (Force || !MRI->getType(Reg).isValid())
955 MRI->setType(VReg: Reg, Ty: RegType);
956 }
957}
958
959// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
960// no valid assigned class, set register LLT type and class according to the
961// SPIR-V type.
962void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
963 MachineIRBuilder &MIRBuilder,
964 SPIRV::AccessQualifier::AccessQualifier AccessQual,
965 bool EmitIR, bool Force) {
966 setRegClassType(Reg,
967 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR),
968 GR, MRI: MIRBuilder.getMRI(), MF: MIRBuilder.getMF(), Force);
969}
970
971// Create a virtual register and assign SPIR-V type to the register. Set
972// register LLT type and class according to the SPIR-V type.
973Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
974 MachineRegisterInfo *MRI,
975 const MachineFunction &MF) {
976 Register Reg = MRI->createVirtualRegister(RegClass: GR->getRegClass(SpvType));
977 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
978 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
979 return Reg;
980}
981
982// Create a virtual register and assign SPIR-V type to the register. Set
983// register LLT type and class according to the SPIR-V type.
984Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
985 MachineIRBuilder &MIRBuilder) {
986 return createVirtualRegister(SpvType, GR, MRI: MIRBuilder.getMRI(),
987 MF: MIRBuilder.getMF());
988}
989
990// Create a SPIR-V type, virtual register and assign SPIR-V type to the
991// register. Set register LLT type and class according to the SPIR-V type.
992Register createVirtualRegister(
993 const Type *Ty, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIRBuilder,
994 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
995 return createVirtualRegister(
996 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR), GR,
997 MIRBuilder);
998}
999
1000bool isVectorType(SPIRVTypeInst SPVTy) {
1001 return SPVTy->getOpcode() == SPIRV::OpTypeVector ||
1002 SPVTy->getOpcode() == SPIRV::OpTypeVectorIdEXT;
1003}
1004
1005CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
1006 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
1007 IRBuilder<> &B) {
1008 SmallVector<Value *, 4> Args;
1009 Args.push_back(Elt: Arg2);
1010 Args.push_back(Elt: buildMD(Arg));
1011 llvm::append_range(C&: Args, R&: Imms);
1012 return B.CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: {Types}, Args);
1013}
1014
1015// Return true if there is an opaque pointer type nested in the argument.
1016bool isNestedPointer(const Type *Ty) {
1017 if (Ty->isPtrOrPtrVectorTy())
1018 return true;
1019 if (const FunctionType *RefTy = dyn_cast<FunctionType>(Val: Ty)) {
1020 if (isNestedPointer(Ty: RefTy->getReturnType()))
1021 return true;
1022 for (const Type *ArgTy : RefTy->params())
1023 if (isNestedPointer(Ty: ArgTy))
1024 return true;
1025 return false;
1026 }
1027 if (const ArrayType *RefTy = dyn_cast<ArrayType>(Val: Ty))
1028 return isNestedPointer(Ty: RefTy->getElementType());
1029 return false;
1030}
1031
1032bool isSpvIntrinsic(const Value *Arg) {
1033 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Arg))
1034 if (Function *F = II->getCalledFunction())
1035 if (F->getName().starts_with(Prefix: "llvm.spv."))
1036 return true;
1037 return false;
1038}
1039
1040// Function to create continued instructions for SPV_INTEL_long_composites
1041// extension
1042SmallVector<MachineInstr *, 4>
1043createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode,
1044 unsigned MinWC, unsigned ContinuedOpcode,
1045 ArrayRef<Register> Args, Register ReturnRegister,
1046 Register TypeID) {
1047
1048 SmallVector<MachineInstr *, 4> Instructions;
1049 constexpr unsigned MaxWordCount = UINT16_MAX;
1050 const size_t NumElements = Args.size();
1051 size_t MaxNumElements = MaxWordCount - MinWC;
1052 size_t SPIRVStructNumElements = NumElements;
1053
1054 if (NumElements > MaxNumElements) {
1055 // Do adjustments for continued instructions which always had only one
1056 // minumum word count.
1057 SPIRVStructNumElements = MaxNumElements;
1058 MaxNumElements = MaxWordCount - 1;
1059 }
1060
1061 auto MIB =
1062 MIRBuilder.buildInstr(Opcode).addDef(RegNo: ReturnRegister).addUse(RegNo: TypeID);
1063
1064 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1065 MIB.addUse(RegNo: Args[I]);
1066
1067 Instructions.push_back(Elt: MIB.getInstr());
1068
1069 for (size_t I = SPIRVStructNumElements; I < NumElements;
1070 I += MaxNumElements) {
1071 auto MIB = MIRBuilder.buildInstr(Opcode: ContinuedOpcode);
1072 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
1073 MIB.addUse(RegNo: Args[J]);
1074 Instructions.push_back(Elt: MIB.getInstr());
1075 }
1076 return Instructions;
1077}
1078
1079SmallVector<unsigned, 1>
1080getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD) {
1081 unsigned LC = SPIRV::LoopControl::None;
1082 // Currently used only to store PartialCount value. Later when other
1083 // LoopControls are added - this map should be sorted before making
1084 // them loop_merge operands to satisfy 3.23. Loop Control requirements.
1085 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;
1086 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.disable")) {
1087 LC |= SPIRV::LoopControl::DontUnroll;
1088 } else {
1089 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.enable") ||
1090 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.full")) {
1091 LC |= SPIRV::LoopControl::Unroll;
1092 }
1093 if (MDNode *CountMD =
1094 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.count")) {
1095 if (auto *CI =
1096 mdconst::extract_or_null<ConstantInt>(MD: CountMD->getOperand(I: 1))) {
1097 unsigned Count = CI->getZExtValue();
1098 if (Count != 1) {
1099 LC |= SPIRV::LoopControl::PartialCount;
1100 MaskToValueMap.emplace_back(
1101 args: std::make_pair(x: SPIRV::LoopControl::PartialCount, y&: Count));
1102 }
1103 }
1104 }
1105 }
1106 SmallVector<unsigned, 1> Result = {LC};
1107 for (auto &[Mask, Val] : MaskToValueMap)
1108 Result.push_back(Elt: Val);
1109 return Result;
1110}
1111
1112SmallVector<unsigned, 1> getSpirvLoopControlOperandsFromLoopMetadata(Loop *L) {
1113 return getSpirvLoopControlOperandsFromLoopMetadata(LoopMD: L->getLoopID());
1114}
1115
1116const std::set<unsigned> &getTypeFoldingSupportedOpcodes() {
1117 // clang-format off
1118 static const std::set<unsigned> TypeFoldingSupportingOpcs = {
1119 TargetOpcode::G_ADD,
1120 TargetOpcode::G_FADD,
1121 TargetOpcode::G_STRICT_FADD,
1122 TargetOpcode::G_SUB,
1123 TargetOpcode::G_FSUB,
1124 TargetOpcode::G_STRICT_FSUB,
1125 TargetOpcode::G_MUL,
1126 TargetOpcode::G_FMUL,
1127 TargetOpcode::G_STRICT_FMUL,
1128 TargetOpcode::G_SDIV,
1129 TargetOpcode::G_UDIV,
1130 TargetOpcode::G_FDIV,
1131 TargetOpcode::G_STRICT_FDIV,
1132 TargetOpcode::G_SREM,
1133 TargetOpcode::G_UREM,
1134 TargetOpcode::G_FREM,
1135 TargetOpcode::G_STRICT_FREM,
1136 TargetOpcode::G_FNEG,
1137 TargetOpcode::G_CONSTANT,
1138 TargetOpcode::G_FCONSTANT,
1139 TargetOpcode::G_AND,
1140 TargetOpcode::G_OR,
1141 TargetOpcode::G_XOR,
1142 TargetOpcode::G_SHL,
1143 TargetOpcode::G_ASHR,
1144 TargetOpcode::G_LSHR,
1145 TargetOpcode::G_SELECT,
1146 TargetOpcode::G_EXTRACT_VECTOR_ELT,
1147 };
1148 // clang-format on
1149 return TypeFoldingSupportingOpcs;
1150}
1151
1152bool isTypeFoldingSupported(unsigned Opcode) {
1153 return getTypeFoldingSupportedOpcodes().count(x: Opcode) > 0;
1154}
1155
1156// Traversing [g]MIR accounting for pseudo-instructions.
1157MachineInstr *passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI) {
1158 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||
1159 Def->getOpcode() == TargetOpcode::COPY)
1160 ? MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())
1161 : Def;
1162}
1163
1164MachineInstr *getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1165 if (MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg()))
1166 return passCopy(Def, MRI);
1167 return nullptr;
1168}
1169
1170MachineInstr *getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1171 if (MachineInstr *Def = getDef(MO, MRI)) {
1172 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
1173 Def->getOpcode() == SPIRV::OpConstantI)
1174 return Def;
1175 }
1176 return nullptr;
1177}
1178
1179int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1180 if (MachineInstr *Def = getImm(MO, MRI)) {
1181 if (Def->getOpcode() == SPIRV::OpConstantI)
1182 return Def->getOperand(i: 2).getImm();
1183 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1184 return Def->getOperand(i: 1).getCImm()->getZExtValue();
1185 }
1186 llvm_unreachable("Unexpected integer constant pattern");
1187}
1188
1189unsigned getArrayComponentCount(const MachineRegisterInfo *MRI,
1190 const MachineInstr *ResType) {
1191 return foldImm(MO: ResType->getOperand(i: 2), MRI);
1192}
1193
1194bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType,
1195 uint64_t &TotalSize) {
1196 // An array of N padded structs is represented as {[N-1 x <{T, pad}>], T}.
1197 if (Ty->getStructNumElements() != 2)
1198 return false;
1199
1200 Type *FirstElement = Ty->getStructElementType(N: 0);
1201 Type *SecondElement = Ty->getStructElementType(N: 1);
1202
1203 if (!FirstElement->isArrayTy())
1204 return false;
1205
1206 Type *ArrayElementType = FirstElement->getArrayElementType();
1207 if (!ArrayElementType->isStructTy() ||
1208 ArrayElementType->getStructNumElements() != 2)
1209 return false;
1210
1211 Type *T_in_struct = ArrayElementType->getStructElementType(N: 0);
1212 if (T_in_struct != SecondElement)
1213 return false;
1214
1215 auto *Padding_in_struct =
1216 dyn_cast<TargetExtType>(Val: ArrayElementType->getStructElementType(N: 1));
1217 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")
1218 return false;
1219
1220 const uint64_t ArraySize = FirstElement->getArrayNumElements();
1221 TotalSize = ArraySize + 1;
1222 OriginalElementType = ArrayElementType;
1223 return true;
1224}
1225
1226Type *reconstitutePeeledArrayType(Type *Ty) {
1227 if (!Ty->isStructTy())
1228 return Ty;
1229
1230 auto *STy = cast<StructType>(Val: Ty);
1231 Type *OriginalElementType = nullptr;
1232 uint64_t TotalSize = 0;
1233 if (matchPeeledArrayPattern(Ty: STy, OriginalElementType, TotalSize)) {
1234 Type *ResultTy = ArrayType::get(
1235 ElementType: reconstitutePeeledArrayType(Ty: OriginalElementType), NumElements: TotalSize);
1236 return ResultTy;
1237 }
1238
1239 SmallVector<Type *, 4> NewElementTypes;
1240 bool Changed = false;
1241 for (Type *ElementTy : STy->elements()) {
1242 Type *NewElementTy = reconstitutePeeledArrayType(Ty: ElementTy);
1243 if (NewElementTy != ElementTy)
1244 Changed = true;
1245 NewElementTypes.push_back(Elt: NewElementTy);
1246 }
1247
1248 if (!Changed)
1249 return Ty;
1250
1251 Type *ResultTy;
1252 if (STy->isLiteral()) {
1253 ResultTy =
1254 StructType::get(Context&: STy->getContext(), Elements: NewElementTypes, isPacked: STy->isPacked());
1255 } else {
1256 ResultTy = StructType::create(Context&: STy->getContext(), Elements: NewElementTypes,
1257 Name: STy->getName(), isPacked: STy->isPacked());
1258 }
1259 return ResultTy;
1260}
1261
1262std::optional<SPIRV::LinkageType::LinkageType>
1263getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV) {
1264 if (GV.hasLocalLinkage())
1265 return std::nullopt;
1266
1267 if (GV.isDeclarationForLinker()) {
1268 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: &GV)) {
1269 auto SC = addressSpaceToStorageClass(AddrSpace: GVar->getAddressSpace(), STI: ST);
1270 // Interface variables must not get Import linkage.
1271 if (SC == SPIRV::StorageClass::Input ||
1272 SC == SPIRV::StorageClass::Output ||
1273 SC == SPIRV::StorageClass::PushConstant)
1274 return std::nullopt;
1275 // Shaders have no linker, so module-internal storage
1276 // (e.g. HLSL groupshared) can't be imported
1277 if (ST.isShader() && (SC == SPIRV::StorageClass::Workgroup ||
1278 SC == SPIRV::StorageClass::Private))
1279 return std::nullopt;
1280 }
1281 return SPIRV::LinkageType::Import;
1282 }
1283
1284 if (GV.hasHiddenVisibility())
1285 return std::nullopt;
1286
1287 if (GV.hasLinkOnceODRLinkage() &&
1288 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr))
1289 return SPIRV::LinkageType::LinkOnceODR;
1290
1291 if (GV.hasWeakLinkage() &&
1292 ST.canUseExtension(E: SPIRV::Extension::SPV_AMD_weak_linkage))
1293 return SPIRV::LinkageType::WeakAMD;
1294
1295 return SPIRV::LinkageType::Export;
1296}
1297
1298Function *getOrCreateBackendServiceFunction(Module &M) {
1299 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
1300 if (!getVacantFunctionName(M, Name&: ServiceFunName))
1301 report_fatal_error(
1302 reason: "cannot allocate a name for the internal service function");
1303 if (Function *SF = M.getFunction(Name: ServiceFunName)) {
1304 if (SF->getInstructionCount() > 0)
1305 report_fatal_error(
1306 reason: "Unexpected combination of global variables and function pointers");
1307 return SF;
1308 }
1309 Function *SF = Function::Create(
1310 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: {}, isVarArg: false),
1311 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
1312 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
1313 return SF;
1314}
1315
1316} // namespace llvm
1317