1//===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVUtils.h"
14#include "MCTargetDesc/SPIRVBaseInfo.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRVInstrInfo.h"
19#include "SPIRVSubtarget.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/Demangle/Demangle.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/TargetParser/AtomicScope.h"
31#include <queue>
32#include <vector>
33
34namespace llvm {
35namespace SPIRV {
36static MDNode *findNamedMDOperand(NamedMDNode *NMD, StringRef Name) {
37 auto It = find_if(Range: NMD->operands(), P: [Name](MDNode *N) {
38 if (auto *MDS = dyn_cast_or_null<MDString>(Val: N->getOperand(I: 0)))
39 return MDS->getString() == Name;
40 return false;
41 });
42 return It == NMD->op_end() ? nullptr : *It;
43}
44
45// This code restores function args/retvalue types for composite cases
46// because the final types should still be aggregate whereas they're i32
47// during the translation to cope with aggregate flattening etc.
48// TODO: should these just return nullptr when there's no metadata?
49static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
50 FunctionType *FTy,
51 StringRef Name) {
52 if (!NMD)
53 return FTy;
54
55 MDNode *Match = findNamedMDOperand(NMD, Name);
56 if (!Match)
57 return FTy;
58
59 Type *RetTy = FTy->getReturnType();
60 SmallVector<Type *, 4> PTys(FTy->params());
61
62 for (unsigned I = 1; I != Match->getNumOperands(); ++I) {
63 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I));
64 assert(MD && "MDNode operand is expected");
65
66 if (auto *Const = getMDOperandAsConstInt(N: MD, I: 0)) {
67 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1));
68 assert(CMeta && "ConstantAsMetadata operand is expected");
69 int64_t Idx = Const->getSExtValue();
70 // Currently -1 indicates return value, greater values mean
71 // argument numbers.
72 if (Idx == -1) {
73 RetTy = CMeta->getType();
74 continue;
75 }
76 if (Idx >= 0 && static_cast<uint64_t>(Idx) < PTys.size()) {
77 PTys[Idx] = CMeta->getType();
78 continue;
79 }
80 report_fatal_error(reason: "invalid argument index in function type metadata");
81 }
82 }
83
84 return FunctionType::get(Result: RetTy, Params: PTys, isVarArg: FTy->isVarArg());
85}
86
87static StringRef extractAsmConstraintsFromMetadata(NamedMDNode *NMD,
88 StringRef Constraints,
89 StringRef Name) {
90 if (!NMD)
91 return Constraints;
92
93 MDNode *Match = findNamedMDOperand(NMD, Name);
94 if (!Match)
95 return Constraints;
96
97 // By convention, the constraints string is stored in the final MD operand.
98 MDNode *MD = dyn_cast<MDNode>(Val: Match->getOperand(I: Match->getNumOperands() - 1));
99 assert(MD && "MDNode operand is expected");
100
101 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
102 Constraints = MDS->getString();
103
104 return Constraints;
105}
106
107FunctionType *getOriginalFunctionType(const Function &F) {
108 return extractFunctionTypeFromMetadata(
109 NMD: F.getParent()->getNamedMetadata(Name: "spv.cloned_funcs"), FTy: F.getFunctionType(),
110 Name: F.getName());
111}
112
113// Keyed via instruction metadata, not a name.
114static std::optional<StringRef> getMutatedCallsiteKey(const CallBase &CB) {
115 if (MDNode *MD = CB.getMetadata(Kind: "spv.mutated_callsite"))
116 if (MD->getNumOperands() > 0)
117 if (auto *MDS = dyn_cast<MDString>(Val: MD->getOperand(I: 0)))
118 return MDS->getString();
119 return std::nullopt;
120}
121
122FunctionType *getOriginalFunctionType(const CallBase &CB) {
123 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
124 if (!Key)
125 return CB.getFunctionType();
126 return extractFunctionTypeFromMetadata(
127 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"),
128 FTy: CB.getFunctionType(), Name: *Key);
129}
130
131StringRef getOriginalAsmConstraints(const CallBase &CB) {
132 StringRef Constraints =
133 cast<InlineAsm>(Val: CB.getCalledOperand())->getConstraintString();
134 std::optional<StringRef> Key = getMutatedCallsiteKey(CB);
135 if (!Key)
136 return Constraints;
137 return extractAsmConstraintsFromMetadata(
138 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"), Constraints,
139 Name: *Key);
140}
141} // Namespace SPIRV
142
143// The following functions are used to add these string literals as a series of
144// 32-bit integer operands with the correct format, and unpack them if necessary
145// when making string comparisons in compiler passes.
146// SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment.
147static uint32_t convertCharsToWord(StringRef Str, unsigned i) {
148 uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars.
149 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {
150 unsigned StrIndex = i + WordIndex;
151 uint8_t CharToAdd = 0; // Initilize char as padding/null.
152 if (StrIndex < Str.size()) { // If it's within the string, get a real char.
153 CharToAdd = Str[StrIndex];
154 }
155 Word |= (CharToAdd << (WordIndex * 8));
156 }
157 return Word;
158}
159
160// Get length including padding and null terminator.
161static size_t getPaddedLen(StringRef Str) { return alignTo(Value: Str.size() + 1, Align: 4); }
162
163void addStringImm(StringRef Str, MCInst &Inst) {
164 const size_t PaddedLen = getPaddedLen(Str);
165 for (unsigned i = 0; i < PaddedLen; i += 4) {
166 // Add an operand for the 32-bits of chars or padding.
167 Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i)));
168 }
169}
170
171void addStringImm(StringRef Str, MachineInstrBuilder &MIB) {
172 const size_t PaddedLen = getPaddedLen(Str);
173 for (unsigned i = 0; i < PaddedLen; i += 4) {
174 // Add an operand for the 32-bits of chars or padding.
175 MIB.addImm(Val: convertCharsToWord(Str, i));
176 }
177}
178
179std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
180 return getSPIRVStringOperand(MI, StartIndex);
181}
182
183std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
184 MachineInstr *Def = getVRegDef(MRI, Reg);
185 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
186 "Expected G_GLOBAL_VALUE");
187 const GlobalValue *GV = Def->getOperand(i: 1).getGlobal();
188 Value *V = GV->getOperand(i: 0);
189 const ConstantDataArray *CDA = cast<ConstantDataArray>(Val: V);
190 return CDA->getAsCString().str();
191}
192
193void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
194 const auto Bitwidth = Imm.getBitWidth();
195 if (Bitwidth == 1)
196 return; // Already handled
197 else if (Bitwidth <= 32) {
198 MIB.addImm(Val: Imm.getZExtValue());
199 // Asm Printer needs this info to print floating-type correctly
200 if (Bitwidth == 16)
201 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16);
202 return;
203 } else if (Bitwidth <= 64) {
204 uint64_t FullImm = Imm.getZExtValue();
205 MIB.addImm(Val: Lo_32(Value: FullImm)).addImm(Val: Hi_32(Value: FullImm));
206 // Asm Printer needs this info to print 64-bit operands correctly
207 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH64);
208 return;
209 } else {
210 // Emit ceil(Bitwidth / 32) words to conform SPIR-V spec.
211 unsigned NumWords = divideCeil(Numerator: Bitwidth, Denominator: 32);
212 for (unsigned I = 0; I < NumWords; ++I) {
213 unsigned LimbIdx = I / 2;
214 unsigned LimbShift = (I % 2) * 32;
215 uint32_t Word = (Imm.getRawData()[LimbIdx] >> LimbShift) & 0xffffffff;
216 MIB.addImm(Val: Word);
217 }
218 return;
219 }
220}
221
222void buildOpName(Register Target, StringRef Name,
223 MachineIRBuilder &MIRBuilder) {
224 if (!Name.empty()) {
225 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpName).addUse(RegNo: Target);
226 addStringImm(Str: Name, MIB);
227 }
228}
229
230static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
231 ArrayRef<uint32_t> DecArgs,
232 StringRef StrImm) {
233 if (!StrImm.empty())
234 addStringImm(Str: StrImm, MIB);
235 for (const auto &DecArg : DecArgs)
236 MIB.addImm(Val: DecArg);
237}
238
239void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
240 SPIRV::Decoration::Decoration Dec,
241 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
242 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
243 .addUse(RegNo: Reg)
244 .addImm(Val: static_cast<uint32_t>(Dec));
245 finishBuildOpDecorate(MIB, DecArgs, StrImm);
246}
247
248void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
249 SPIRV::Decoration::Decoration Dec,
250 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
251 MachineBasicBlock &MBB = *I.getParent();
252 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDecorate))
253 .addUse(RegNo: Reg)
254 .addImm(Val: static_cast<uint32_t>(Dec));
255 finishBuildOpDecorate(MIB, DecArgs, StrImm);
256}
257
258void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
259 SPIRV::Decoration::Decoration Dec, uint32_t Member,
260 ArrayRef<uint32_t> DecArgs, StringRef StrImm) {
261 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpMemberDecorate)
262 .addUse(RegNo: Reg)
263 .addImm(Val: Member)
264 .addImm(Val: static_cast<uint32_t>(Dec));
265 finishBuildOpDecorate(MIB, DecArgs, StrImm);
266}
267
268void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
269 const MDNode *GVarMD, const SPIRVSubtarget &ST) {
270 for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
271 auto *OpMD = dyn_cast<MDNode>(Val: GVarMD->getOperand(I));
272 if (!OpMD)
273 report_fatal_error(reason: "Invalid decoration");
274 if (OpMD->getNumOperands() == 0)
275 report_fatal_error(reason: "Expect operand(s) of the decoration");
276 ConstantInt *DecorationId =
277 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 0));
278 if (!DecorationId)
279 report_fatal_error(reason: "Expect SPIR-V <Decoration> operand to be the first "
280 "element of the decoration");
281
282 // The goal of `spirv.Decorations` metadata is to provide a way to
283 // represent SPIR-V entities that do not map to LLVM in an obvious way.
284 // FP flags do have obvious matches between LLVM IR and SPIR-V.
285 // Additionally, we have no guarantee at this point that the flags passed
286 // through the decoration are not violated already in the optimizer passes.
287 // Therefore, we simply ignore FP flags, including NoContraction, and
288 // FPFastMathMode.
289 if (DecorationId->getZExtValue() ==
290 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||
291 DecorationId->getZExtValue() ==
292 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {
293 continue; // Ignored.
294 }
295 uint32_t Dec = static_cast<uint32_t>(DecorationId->getZExtValue());
296 if (Dec == static_cast<uint32_t>(SPIRV::Decoration::UniformId) ||
297 Dec == static_cast<uint32_t>(SPIRV::Decoration::AlignmentId) ||
298 Dec == static_cast<uint32_t>(SPIRV::Decoration::MaxByteOffsetId)) {
299 ConstantInt *IdV =
300 OpMD->getNumOperands() == 2
301 ? mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 1))
302 : nullptr;
303 if (!IdV || !isUInt<32>(x: IdV->getZExtValue()))
304 report_fatal_error(reason: "Expect a single integer <id> operand of the "
305 "decoration");
306 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
307 SPIRVTypeInst SpvTypeInt32 =
308 GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
309 Register IdReg = GR->buildConstantInt(Val: IdV->getZExtValue(), MIRBuilder,
310 SpvType: SpvTypeInt32, /*EmitIR=*/false);
311 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorateId)
312 .addUse(RegNo: Reg)
313 .addImm(Val: Dec)
314 .addUse(RegNo: IdReg);
315 continue;
316 }
317 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate).addUse(RegNo: Reg).addImm(Val: Dec);
318 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
319 if (ConstantInt *OpV =
320 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: OpI)))
321 MIB.addImm(Val: static_cast<uint32_t>(OpV->getZExtValue()));
322 else if (MDString *OpV = dyn_cast<MDString>(Val: OpMD->getOperand(I: OpI)))
323 addStringImm(Str: OpV->getString(), MIB);
324 else
325 report_fatal_error(reason: "Unexpected operand of the decoration");
326 }
327 }
328}
329
330MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF) {
331 MachineBasicBlock &MBB = MF.front();
332 // Find the position to insert the OpVariable instruction.
333 // We will insert it after the last OpFunctionParameter, if any, or
334 // after OpFunction otherwise.
335 auto IsPreamble = [](const MachineInstr &MI) {
336 switch (MI.getOpcode()) {
337 case SPIRV::OpFunction:
338 case SPIRV::OpFunctionParameter:
339 case SPIRV::OpLabel:
340 case SPIRV::ASSIGN_TYPE:
341 return true;
342 default:
343 return false;
344 }
345 };
346 MachineBasicBlock::iterator VarPos = MBB.SkipPHIsAndLabels(I: MBB.begin());
347 while (VarPos != MBB.end() && VarPos->getOpcode() != SPIRV::OpFunction)
348 ++VarPos;
349 // Advance past the preamble.
350 while (VarPos != MBB.end() && IsPreamble(*VarPos))
351 ++VarPos;
352 return VarPos;
353}
354
355MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB) {
356 MachineBasicBlock::iterator I = MBB->end();
357 if (I == MBB->begin())
358 return I;
359 --I;
360 while (I->isTerminator() || I->isDebugValue()) {
361 if (I == MBB->begin())
362 break;
363 --I;
364 }
365 return I;
366}
367
368SPIRV::StorageClass::StorageClass
369addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
370 switch (AddrSpace) {
371 case SPIRVAS::Function:
372 return SPIRV::StorageClass::Function;
373 case SPIRVAS::CrossWorkgroup:
374 return SPIRV::StorageClass::CrossWorkgroup;
375 case SPIRVAS::UniformConstant:
376 return SPIRV::StorageClass::UniformConstant;
377 case SPIRVAS::Workgroup:
378 return SPIRV::StorageClass::Workgroup;
379 case SPIRVAS::Generic:
380 return SPIRV::StorageClass::Generic;
381 case SPIRVAS::DeviceOnlyINTEL:
382 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
383 ? SPIRV::StorageClass::DeviceOnlyINTEL
384 : SPIRV::StorageClass::CrossWorkgroup;
385 case SPIRVAS::HostOnlyINTEL:
386 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
387 ? SPIRV::StorageClass::HostOnlyINTEL
388 : SPIRV::StorageClass::CrossWorkgroup;
389 case SPIRVAS::Input:
390 return SPIRV::StorageClass::Input;
391 case SPIRVAS::Output:
392 return SPIRV::StorageClass::Output;
393 case SPIRVAS::CodeSectionINTEL:
394 return SPIRV::StorageClass::CodeSectionINTEL;
395 case SPIRVAS::Private:
396 return SPIRV::StorageClass::Private;
397 case SPIRVAS::StorageBuffer:
398 return SPIRV::StorageClass::StorageBuffer;
399 case SPIRVAS::Uniform:
400 return SPIRV::StorageClass::Uniform;
401 case SPIRVAS::PushConstant:
402 return SPIRV::StorageClass::PushConstant;
403 default:
404 report_fatal_error(reason: "Unknown address space");
405 }
406}
407
408SPIRV::MemorySemantics::MemorySemantics
409getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) {
410 switch (SC) {
411 case SPIRV::StorageClass::StorageBuffer:
412 case SPIRV::StorageClass::Uniform:
413 return SPIRV::MemorySemantics::UniformMemory;
414 case SPIRV::StorageClass::Workgroup:
415 return SPIRV::MemorySemantics::WorkgroupMemory;
416 case SPIRV::StorageClass::CrossWorkgroup:
417 return SPIRV::MemorySemantics::CrossWorkgroupMemory;
418 case SPIRV::StorageClass::Generic:
419 return SPIRV::MemorySemantics::MemorySemantics(
420 SPIRV::MemorySemantics::WorkgroupMemory |
421 SPIRV::MemorySemantics::CrossWorkgroupMemory);
422 case SPIRV::StorageClass::AtomicCounter:
423 return SPIRV::MemorySemantics::AtomicCounterMemory;
424 case SPIRV::StorageClass::Image:
425 return SPIRV::MemorySemantics::ImageMemory;
426 default:
427 return SPIRV::MemorySemantics::None;
428 }
429}
430
431SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
432 switch (Ord) {
433 case AtomicOrdering::Acquire:
434 return SPIRV::MemorySemantics::Acquire;
435 case AtomicOrdering::Release:
436 return SPIRV::MemorySemantics::Release;
437 case AtomicOrdering::AcquireRelease:
438 return SPIRV::MemorySemantics::AcquireRelease;
439 case AtomicOrdering::SequentiallyConsistent:
440 return SPIRV::MemorySemantics::SequentiallyConsistent;
441 case AtomicOrdering::Unordered:
442 case AtomicOrdering::Monotonic:
443 case AtomicOrdering::NotAtomic:
444 return SPIRV::MemorySemantics::None;
445 }
446 llvm_unreachable(nullptr);
447}
448
449uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem,
450 uint32_t StorageClassSem) {
451 bool DropStorageClass =
452 TT.isVulkanOS() &&
453 OrderSem == static_cast<uint32_t>(SPIRV::MemorySemantics::None);
454 return OrderSem | (DropStorageClass ? 0 : StorageClassSem);
455}
456
457SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
458 SyncScope::ID Id) {
459 // Named by
460 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_scope_id.
461 // We don't need aliases for Invocation and CrossDevice, as we already have
462 // them covered by "singlethread" and "" strings respectively (see
463 // implementation of LLVMContext::LLVMContext()).
464 auto ScopeID = [&](AtomicScope Scope) {
465 return Ctx.getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(T: TT, S: Scope));
466 };
467 const llvm::SyncScope::ID SubGroup = ScopeID(AtomicScope::Wavefront);
468 const llvm::SyncScope::ID WorkGroup = ScopeID(AtomicScope::Workgroup);
469 const llvm::SyncScope::ID Device = ScopeID(AtomicScope::Device);
470
471 if (Id == llvm::SyncScope::SingleThread)
472 return SPIRV::Scope::Invocation;
473 else if (Id == llvm::SyncScope::System)
474 return SPIRV::Scope::CrossDevice;
475 else if (Id == SubGroup)
476 return SPIRV::Scope::Subgroup;
477 else if (Id == WorkGroup)
478 return SPIRV::Scope::Workgroup;
479 else if (Id == Device)
480 return SPIRV::Scope::Device;
481 return SPIRV::Scope::CrossDevice;
482}
483
484MachineInstr *getDefInstrMaybeConstant(Register &ConstReg,
485 const MachineRegisterInfo *MRI) {
486 MachineInstr *MI = MRI->getVRegDef(Reg: ConstReg);
487 MachineInstr *ConstInstr =
488 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT
489 ? MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg())
490 : MI;
491 if (auto *GI = dyn_cast<GIntrinsic>(Val: ConstInstr)) {
492 if (GI->is(ID: Intrinsic::spv_track_constant)) {
493 ConstReg = ConstInstr->getOperand(i: 2).getReg();
494 return MRI->getVRegDef(Reg: ConstReg);
495 }
496 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {
497 ConstReg = ConstInstr->getOperand(i: 1).getReg();
498 return MRI->getVRegDef(Reg: ConstReg);
499 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||
500 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {
501 ConstReg = ConstInstr->getOperand(i: 0).getReg();
502 return ConstInstr;
503 }
504 return MRI->getVRegDef(Reg: ConstReg);
505}
506
507uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) {
508 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
509 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
510 return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
511}
512
513bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
514 if (const auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
515 return GI->is(ID: IntrinsicID);
516 return false;
517}
518
519Type *getMDOperandAsType(const MDNode *N, unsigned I) {
520 Type *ElementTy = cast<ValueAsMetadata>(Val: N->getOperand(I))->getType();
521 return toTypedPointer(Ty: ElementTy);
522}
523
524ConstantInt *getMDOperandAsConstInt(const MDNode *N, unsigned I) {
525 if (N->getNumOperands() <= I)
526 return nullptr;
527 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I)))
528 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
529 return nullptr;
530}
531
532static bool isEnqueueKernelBI(StringRef MangledName) {
533 return MangledName == "__enqueue_kernel_basic" ||
534 MangledName == "__enqueue_kernel_basic_events" ||
535 MangledName == "__enqueue_kernel_varargs" ||
536 MangledName == "__enqueue_kernel_events_varargs";
537}
538
539static bool isKernelQueryBI(StringRef MangledName) {
540 return MangledName == "__get_kernel_work_group_size_impl" ||
541 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
542 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
543 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
544}
545
546static bool isNonMangledOCLBuiltin(StringRef Name) {
547 if (!Name.starts_with(Prefix: "__"))
548 return false;
549
550 return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) ||
551 SPIRV::isPipeOrAddressSpaceCastBuiltin(Name) ||
552 Name == "__translate_sampler_initializer";
553}
554
555std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
556 bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
557 bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_");
558 bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_");
559 bool IsMangled = Name.starts_with(Prefix: "_Z");
560
561 // Otherwise use simple demangling to return the function name.
562 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
563 return Name.str();
564
565 // Try to use the itanium demangler.
566 if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) {
567 std::string Result = DemangledName;
568 free(ptr: DemangledName);
569 return Result;
570 }
571
572 // Autocheck C++, maybe need to do explicit check of the source language.
573 // OpenCL C++ built-ins are declared in cl namespace.
574 // TODO: consider using 'St' abbriviation for cl namespace mangling.
575 // Similar to ::std:: in C++.
576 size_t Start, Len = 0;
577 size_t DemangledNameLenStart = 2;
578 if (Name.starts_with(Prefix: "_ZN")) {
579 // Skip CV and ref qualifiers.
580 size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO", From: 3);
581 // All built-ins are in the ::cl:: namespace.
582 if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv")
583 return std::string();
584 DemangledNameLenStart = NameSpaceStart + 11;
585 }
586 Start = Name.find_first_not_of(Chars: "0123456789", From: DemangledNameLenStart);
587 bool Error = Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart)
588 .getAsInteger(Radix: 10, Result&: Len);
589 if (Error)
590 return std::string();
591 return Name.substr(Start, N: Len).str();
592}
593
594bool hasBuiltinTypePrefix(StringRef Name) {
595 if (Name.starts_with(Prefix: "opencl.") || Name.starts_with(Prefix: "ocl_") ||
596 Name.starts_with(Prefix: "spirv."))
597 return true;
598 return false;
599}
600
601bool isSpecialOpaqueType(const Type *Ty) {
602 if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Val: Ty))
603 return isTypedPointerWrapper(ExtTy)
604 ? false
605 : hasBuiltinTypePrefix(Name: ExtTy->getName());
606
607 return false;
608}
609
610bool isEntryPoint(const Function &F) {
611 // OpenCL handling: any function with the SPIR_KERNEL
612 // calling convention will be a potential entry point.
613 if (F.getCallingConv() == CallingConv::SPIR_KERNEL)
614 return true;
615
616 // HLSL handling: special attribute are emitted from the
617 // front-end.
618 if (F.getFnAttribute(Kind: "hlsl.shader").isValid())
619 return true;
620
621 return false;
622}
623
624Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
625 TypeName.consume_front(Prefix: "atomic_");
626 if (TypeName.consume_front(Prefix: "void"))
627 return Type::getVoidTy(C&: Ctx);
628 else if (TypeName.consume_front(Prefix: "bool") || TypeName.consume_front(Prefix: "_Bool"))
629 return Type::getIntNTy(C&: Ctx, N: 1);
630 else if (TypeName.consume_front(Prefix: "char") ||
631 TypeName.consume_front(Prefix: "signed char") ||
632 TypeName.consume_front(Prefix: "unsigned char") ||
633 TypeName.consume_front(Prefix: "uchar"))
634 return Type::getInt8Ty(C&: Ctx);
635 else if (TypeName.consume_front(Prefix: "short") ||
636 TypeName.consume_front(Prefix: "signed short") ||
637 TypeName.consume_front(Prefix: "unsigned short") ||
638 TypeName.consume_front(Prefix: "ushort"))
639 return Type::getInt16Ty(C&: Ctx);
640 else if (TypeName.consume_front(Prefix: "int") ||
641 TypeName.consume_front(Prefix: "signed int") ||
642 TypeName.consume_front(Prefix: "unsigned int") ||
643 TypeName.consume_front(Prefix: "uint"))
644 return Type::getInt32Ty(C&: Ctx);
645 else if (TypeName.consume_front(Prefix: "long") ||
646 TypeName.consume_front(Prefix: "signed long") ||
647 TypeName.consume_front(Prefix: "unsigned long") ||
648 TypeName.consume_front(Prefix: "ulong"))
649 return Type::getInt64Ty(C&: Ctx);
650 else if (TypeName.consume_front(Prefix: "half") ||
651 TypeName.consume_front(Prefix: "_Float16") ||
652 TypeName.consume_front(Prefix: "__fp16"))
653 return Type::getHalfTy(C&: Ctx);
654 else if (TypeName.consume_front(Prefix: "float"))
655 return Type::getFloatTy(C&: Ctx);
656 else if (TypeName.consume_front(Prefix: "double"))
657 return Type::getDoubleTy(C&: Ctx);
658
659 // Unable to recognize SPIRV type name
660 return nullptr;
661}
662
663SmallPtrSet<BasicBlock *, 0>
664PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
665 std::queue<BasicBlock *> ToVisit;
666 ToVisit.push(x: Start);
667
668 SmallPtrSet<BasicBlock *, 0> Output;
669 while (ToVisit.size() != 0) {
670 BasicBlock *BB = ToVisit.front();
671 ToVisit.pop();
672
673 if (Output.count(Ptr: BB) != 0)
674 continue;
675 Output.insert(Ptr: BB);
676
677 for (BasicBlock *Successor : successors(BB)) {
678 if (DT.dominates(A: Successor, B: BB))
679 continue;
680 ToVisit.push(x: Successor);
681 }
682 }
683
684 return Output;
685}
686
687bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
688 for (BasicBlock *P : predecessors(BB)) {
689 // Ignore back-edges.
690 if (DT.dominates(A: BB, B: P))
691 continue;
692
693 // One of the predecessor hasn't been visited. Not ready yet.
694 if (BlockToOrder.count(Val: P) == 0)
695 return false;
696
697 // If the block is a loop exit, the loop must be finished before
698 // we can continue.
699 Loop *L = LI.getLoopFor(BB: P);
700 if (L == nullptr || L->contains(BB))
701 continue;
702
703 // SPIR-V requires a single back-edge. And the backend first
704 // step transforms loops into the simplified format. If we have
705 // more than 1 back-edge, something is wrong.
706 assert(L->getNumBackEdges() <= 1);
707
708 // If the loop has no latch, loop's rank won't matter, so we can
709 // proceed.
710 BasicBlock *Latch = L->getLoopLatch();
711 assert(Latch);
712 if (Latch == nullptr)
713 continue;
714
715 // The latch is not ready yet, let's wait.
716 if (BlockToOrder.count(Val: Latch) == 0)
717 return false;
718 }
719
720 return true;
721}
722
723size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
724 auto It = BlockToOrder.find(Val: BB);
725 if (It != BlockToOrder.end())
726 return It->second.Rank;
727
728 size_t result = 0;
729 for (BasicBlock *P : predecessors(BB)) {
730 // Ignore back-edges.
731 if (DT.dominates(A: BB, B: P))
732 continue;
733
734 auto Iterator = BlockToOrder.end();
735 Loop *L = LI.getLoopFor(BB: P);
736 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
737
738 // If the predecessor is either outside a loop, or part of
739 // the same loop, simply take its rank + 1.
740 if (L == nullptr || L->contains(BB) || Latch == nullptr) {
741 Iterator = BlockToOrder.find(Val: P);
742 } else {
743 // Otherwise, take the loop's rank (highest rank in the loop) as base.
744 // Since loops have a single latch, highest rank is easy to find.
745 // If the loop has no latch, then it doesn't matter.
746 Iterator = BlockToOrder.find(Val: Latch);
747 }
748
749 assert(Iterator != BlockToOrder.end());
750 result = std::max(a: result, b: Iterator->second.Rank + 1);
751 }
752
753 return result;
754}
755
756size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
757 ToVisit.push(x: BB);
758 Queued.insert(Ptr: BB);
759
760 size_t QueueIndex = 0;
761 while (ToVisit.size() != 0) {
762 BasicBlock *BB = ToVisit.front();
763 ToVisit.pop();
764
765 if (!CanBeVisited(BB)) {
766 ToVisit.push(x: BB);
767 if (QueueIndex >= ToVisit.size())
768 llvm::report_fatal_error(
769 reason: "No valid candidate in the queue. Is the graph reducible?");
770 QueueIndex++;
771 continue;
772 }
773
774 QueueIndex = 0;
775 size_t Rank = GetNodeRank(BB);
776 OrderInfo Info = {.Rank: Rank, .TraversalIndex: BlockToOrder.size()};
777 BlockToOrder.try_emplace(Key: BB, Args&: Info);
778
779 for (BasicBlock *S : successors(BB)) {
780 if (Queued.count(Ptr: S) != 0)
781 continue;
782 ToVisit.push(x: S);
783 Queued.insert(Ptr: S);
784 }
785 }
786
787 return 0;
788}
789
790PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
791 DT.recalculate(Func&: F);
792 LI = LoopInfo(DT);
793
794 visit(BB: &*F.begin(), Unused: 0);
795
796 Order.reserve(n: F.size());
797 for (auto &[BB, Info] : BlockToOrder)
798 Order.emplace_back(args&: BB);
799
800 llvm::sort(C&: Order, Comp: [&](const auto &LHS, const auto &RHS) {
801 return compare(LHS, RHS);
802 });
803}
804
805bool PartialOrderingVisitor::compare(const BasicBlock *LHS,
806 const BasicBlock *RHS) const {
807 const OrderInfo &InfoLHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(LHS));
808 const OrderInfo &InfoRHS = BlockToOrder.at(Val: const_cast<BasicBlock *>(RHS));
809 if (InfoLHS.Rank != InfoRHS.Rank)
810 return InfoLHS.Rank < InfoRHS.Rank;
811 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;
812}
813
814void PartialOrderingVisitor::partialOrderVisit(
815 BasicBlock &Start, std::function<bool(BasicBlock *)> Op) {
816 SmallPtrSet<BasicBlock *, 0> Reachable = getReachableFrom(Start: &Start);
817 assert(BlockToOrder.count(&Start) != 0);
818
819 // Skipping blocks with a rank inferior to |Start|'s rank.
820 auto It = Order.begin();
821 while (It != Order.end() && *It != &Start)
822 ++It;
823
824 // This is unexpected. Worst case |Start| is the last block,
825 // so It should point to the last block, not past-end.
826 assert(It != Order.end());
827
828 // By default, there is no rank limit. Setting it to the maximum value.
829 std::optional<size_t> EndRank = std::nullopt;
830 for (; It != Order.end(); ++It) {
831 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)
832 break;
833
834 if (Reachable.count(Ptr: *It) == 0) {
835 continue;
836 }
837
838 if (!Op(*It)) {
839 EndRank = BlockToOrder[*It].Rank;
840 }
841 }
842}
843
844bool sortBlocks(Function &F) {
845 if (F.size() == 0)
846 return false;
847
848 bool Modified = false;
849 std::vector<BasicBlock *> Order;
850 Order.reserve(n: F.size());
851
852 ReversePostOrderTraversal<Function *> RPOT(&F);
853 llvm::append_range(C&: Order, R&: RPOT);
854
855 assert(&*F.begin() == Order[0]);
856 BasicBlock *LastBlock = &*F.begin();
857 for (BasicBlock *BB : Order) {
858 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {
859 Modified = true;
860 BB->moveAfter(MovePos: LastBlock);
861 }
862 LastBlock = BB;
863 }
864
865 return Modified;
866}
867
868AllocaInst *createVariable(Function &F, Type *Type) {
869 const DataLayout &DL = F.getDataLayout();
870 return new AllocaInst(Type, DL.getAllocaAddrSpace(), nullptr, "reg",
871 F.begin()->getFirstInsertionPt());
872}
873
874Value *
875createExitVariable(BasicBlock *BB,
876 const DenseMap<BasicBlock *, ConstantInt *> &TargetToValue) {
877 auto *T = BB->getTerminator();
878 if (isa<ReturnInst>(Val: T))
879 return nullptr;
880 if (auto *BI = dyn_cast<UncondBrInst>(Val: T))
881 return TargetToValue.lookup(Val: BI->getSuccessor());
882
883 IRBuilder<> Builder(BB);
884 Builder.SetInsertPoint(T);
885
886 if (auto *BI = dyn_cast<CondBrInst>(Val: T)) {
887 Value *LHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 0));
888 Value *RHS = TargetToValue.lookup(Val: BI->getSuccessor(i: 1));
889
890 if (LHS == nullptr || RHS == nullptr)
891 return LHS == nullptr ? RHS : LHS;
892 return Builder.CreateSelect(C: BI->getCondition(), True: LHS, False: RHS);
893 }
894
895 if (auto *SI = dyn_cast<SwitchInst>(Val: T)) {
896 Value *Condition = SI->getCondition();
897 // The default destination acts as the fallback value of the select chain.
898 Value *Result = TargetToValue.lookup(Val: SI->getDefaultDest());
899 for (const auto &Case : SI->cases()) {
900 Value *CaseValue = TargetToValue.lookup(Val: Case.getCaseSuccessor());
901 // Successors that are internal to the region have no exit value.
902 if (CaseValue == nullptr)
903 continue;
904 // The first known exit value becomes the base of the select chain.
905 if (Result == nullptr) {
906 Result = CaseValue;
907 continue;
908 }
909 Value *Cmp = Builder.CreateICmpEQ(LHS: Condition, RHS: Case.getCaseValue());
910 Result = Builder.CreateSelect(C: Cmp, True: CaseValue, False: Result);
911 }
912 return Result;
913 }
914
915 llvm_unreachable("Unhandled terminator type.");
916}
917
918MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
919 MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
920 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
921 MaybeDef = MRI.getVRegDef(Reg: MaybeDef->getOperand(i: 1).getReg());
922 return MaybeDef;
923}
924
925static bool getVacantFunctionName(Module &M, std::string &Name) {
926 // It's a bit of paranoia, but still we don't want to have even a chance that
927 // the loop will work for too long.
928 constexpr unsigned MaxIters = 1024;
929 for (unsigned I = 0; I < MaxIters; ++I) {
930 std::string OrdName = Name + Twine(I).str();
931 if (!M.getFunction(Name: OrdName)) {
932 Name = std::move(OrdName);
933 return true;
934 }
935 }
936 return false;
937}
938
939// Assign SPIR-V type to the register. If the register has no valid assigned
940// class, set register LLT type and class according to the SPIR-V type.
941void setRegClassType(Register Reg, SPIRVTypeInst SpvType,
942 SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
943 const MachineFunction &MF, bool Force) {
944 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
945 if (!MRI->getRegClassOrNull(Reg) || Force) {
946 MRI->setRegClass(Reg, RC: GR->getRegClass(SpvType));
947 LLT RegType = GR->getRegType(SpvType);
948 if (Force || !MRI->getType(Reg).isValid())
949 MRI->setType(VReg: Reg, Ty: RegType);
950 }
951}
952
953// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
954// no valid assigned class, set register LLT type and class according to the
955// SPIR-V type.
956void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
957 MachineIRBuilder &MIRBuilder,
958 SPIRV::AccessQualifier::AccessQualifier AccessQual,
959 bool EmitIR, bool Force) {
960 setRegClassType(Reg,
961 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR),
962 GR, MRI: MIRBuilder.getMRI(), MF: MIRBuilder.getMF(), Force);
963}
964
965// Create a virtual register and assign SPIR-V type to the register. Set
966// register LLT type and class according to the SPIR-V type.
967Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
968 MachineRegisterInfo *MRI,
969 const MachineFunction &MF) {
970 Register Reg = MRI->createVirtualRegister(RegClass: GR->getRegClass(SpvType));
971 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
972 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
973 return Reg;
974}
975
976// Create a virtual register and assign SPIR-V type to the register. Set
977// register LLT type and class according to the SPIR-V type.
978Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
979 MachineIRBuilder &MIRBuilder) {
980 return createVirtualRegister(SpvType, GR, MRI: MIRBuilder.getMRI(),
981 MF: MIRBuilder.getMF());
982}
983
984// Create a SPIR-V type, virtual register and assign SPIR-V type to the
985// register. Set register LLT type and class according to the SPIR-V type.
986Register createVirtualRegister(
987 const Type *Ty, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIRBuilder,
988 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
989 return createVirtualRegister(
990 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR), GR,
991 MIRBuilder);
992}
993
994bool isVectorType(SPIRVTypeInst SPVTy) {
995 return SPVTy->getOpcode() == SPIRV::OpTypeVector ||
996 SPVTy->getOpcode() == SPIRV::OpTypeVectorIdEXT;
997}
998
999CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
1000 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
1001 IRBuilder<> &B) {
1002 SmallVector<Value *, 4> Args;
1003 Args.push_back(Elt: Arg2);
1004 Args.push_back(Elt: buildMD(Arg));
1005 llvm::append_range(C&: Args, R&: Imms);
1006 return B.CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: {Types}, Args);
1007}
1008
1009// Return true if there is an opaque pointer type nested in the argument.
1010bool isNestedPointer(const Type *Ty) {
1011 if (Ty->isPtrOrPtrVectorTy())
1012 return true;
1013 if (const FunctionType *RefTy = dyn_cast<FunctionType>(Val: Ty)) {
1014 if (isNestedPointer(Ty: RefTy->getReturnType()))
1015 return true;
1016 for (const Type *ArgTy : RefTy->params())
1017 if (isNestedPointer(Ty: ArgTy))
1018 return true;
1019 return false;
1020 }
1021 if (const ArrayType *RefTy = dyn_cast<ArrayType>(Val: Ty))
1022 return isNestedPointer(Ty: RefTy->getElementType());
1023 return false;
1024}
1025
1026bool isSpvIntrinsic(const Value *Arg) {
1027 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Arg))
1028 if (Function *F = II->getCalledFunction())
1029 if (F->getName().starts_with(Prefix: "llvm.spv."))
1030 return true;
1031 return false;
1032}
1033
1034// Function to create continued instructions for SPV_INTEL_long_composites
1035// extension
1036SmallVector<MachineInstr *, 4>
1037createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode,
1038 unsigned MinWC, unsigned ContinuedOpcode,
1039 ArrayRef<Register> Args, Register ReturnRegister,
1040 Register TypeID) {
1041
1042 SmallVector<MachineInstr *, 4> Instructions;
1043 constexpr unsigned MaxWordCount = UINT16_MAX;
1044 const size_t NumElements = Args.size();
1045 size_t MaxNumElements = MaxWordCount - MinWC;
1046 size_t SPIRVStructNumElements = NumElements;
1047
1048 if (NumElements > MaxNumElements) {
1049 // Do adjustments for continued instructions which always had only one
1050 // minumum word count.
1051 SPIRVStructNumElements = MaxNumElements;
1052 MaxNumElements = MaxWordCount - 1;
1053 }
1054
1055 auto MIB =
1056 MIRBuilder.buildInstr(Opcode).addDef(RegNo: ReturnRegister).addUse(RegNo: TypeID);
1057
1058 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1059 MIB.addUse(RegNo: Args[I]);
1060
1061 Instructions.push_back(Elt: MIB.getInstr());
1062
1063 for (size_t I = SPIRVStructNumElements; I < NumElements;
1064 I += MaxNumElements) {
1065 auto MIB = MIRBuilder.buildInstr(Opcode: ContinuedOpcode);
1066 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
1067 MIB.addUse(RegNo: Args[J]);
1068 Instructions.push_back(Elt: MIB.getInstr());
1069 }
1070 return Instructions;
1071}
1072
1073SmallVector<unsigned, 1>
1074getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD) {
1075 unsigned LC = SPIRV::LoopControl::None;
1076 // Currently used only to store PartialCount value. Later when other
1077 // LoopControls are added - this map should be sorted before making
1078 // them loop_merge operands to satisfy 3.23. Loop Control requirements.
1079 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;
1080 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.disable")) {
1081 LC |= SPIRV::LoopControl::DontUnroll;
1082 } else {
1083 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.enable") ||
1084 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.full")) {
1085 LC |= SPIRV::LoopControl::Unroll;
1086 }
1087 if (MDNode *CountMD =
1088 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.count")) {
1089 if (auto *CI =
1090 mdconst::extract_or_null<ConstantInt>(MD: CountMD->getOperand(I: 1))) {
1091 unsigned Count = CI->getZExtValue();
1092 if (Count != 1) {
1093 LC |= SPIRV::LoopControl::PartialCount;
1094 MaskToValueMap.emplace_back(
1095 args: std::make_pair(x: SPIRV::LoopControl::PartialCount, y&: Count));
1096 }
1097 }
1098 }
1099 }
1100 SmallVector<unsigned, 1> Result = {LC};
1101 for (auto &[Mask, Val] : MaskToValueMap)
1102 Result.push_back(Elt: Val);
1103 return Result;
1104}
1105
1106SmallVector<unsigned, 1> getSpirvLoopControlOperandsFromLoopMetadata(Loop *L) {
1107 return getSpirvLoopControlOperandsFromLoopMetadata(LoopMD: L->getLoopID());
1108}
1109
1110const std::set<unsigned> &getTypeFoldingSupportedOpcodes() {
1111 // clang-format off
1112 static const std::set<unsigned> TypeFoldingSupportingOpcs = {
1113 TargetOpcode::G_ADD,
1114 TargetOpcode::G_FADD,
1115 TargetOpcode::G_STRICT_FADD,
1116 TargetOpcode::G_SUB,
1117 TargetOpcode::G_FSUB,
1118 TargetOpcode::G_STRICT_FSUB,
1119 TargetOpcode::G_MUL,
1120 TargetOpcode::G_FMUL,
1121 TargetOpcode::G_STRICT_FMUL,
1122 TargetOpcode::G_SDIV,
1123 TargetOpcode::G_UDIV,
1124 TargetOpcode::G_FDIV,
1125 TargetOpcode::G_STRICT_FDIV,
1126 TargetOpcode::G_SREM,
1127 TargetOpcode::G_UREM,
1128 TargetOpcode::G_FREM,
1129 TargetOpcode::G_STRICT_FREM,
1130 TargetOpcode::G_FNEG,
1131 TargetOpcode::G_CONSTANT,
1132 TargetOpcode::G_FCONSTANT,
1133 TargetOpcode::G_AND,
1134 TargetOpcode::G_OR,
1135 TargetOpcode::G_XOR,
1136 TargetOpcode::G_SHL,
1137 TargetOpcode::G_ASHR,
1138 TargetOpcode::G_LSHR,
1139 TargetOpcode::G_SELECT,
1140 TargetOpcode::G_EXTRACT_VECTOR_ELT,
1141 };
1142 // clang-format on
1143 return TypeFoldingSupportingOpcs;
1144}
1145
1146bool isTypeFoldingSupported(unsigned Opcode) {
1147 return getTypeFoldingSupportedOpcodes().count(x: Opcode) > 0;
1148}
1149
1150// Traversing [g]MIR accounting for pseudo-instructions.
1151MachineInstr *passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI) {
1152 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||
1153 Def->getOpcode() == TargetOpcode::COPY)
1154 ? MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())
1155 : Def;
1156}
1157
1158MachineInstr *getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1159 if (MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg()))
1160 return passCopy(Def, MRI);
1161 return nullptr;
1162}
1163
1164MachineInstr *getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1165 if (MachineInstr *Def = getDef(MO, MRI)) {
1166 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
1167 Def->getOpcode() == SPIRV::OpConstantI)
1168 return Def;
1169 }
1170 return nullptr;
1171}
1172
1173int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1174 if (MachineInstr *Def = getImm(MO, MRI)) {
1175 if (Def->getOpcode() == SPIRV::OpConstantI)
1176 return Def->getOperand(i: 2).getImm();
1177 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1178 return Def->getOperand(i: 1).getCImm()->getZExtValue();
1179 }
1180 llvm_unreachable("Unexpected integer constant pattern");
1181}
1182
1183unsigned getArrayComponentCount(const MachineRegisterInfo *MRI,
1184 const MachineInstr *ResType) {
1185 return foldImm(MO: ResType->getOperand(i: 2), MRI);
1186}
1187
1188bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType,
1189 uint64_t &TotalSize) {
1190 // An array of N padded structs is represented as {[N-1 x <{T, pad}>], T}.
1191 if (Ty->getStructNumElements() != 2)
1192 return false;
1193
1194 Type *FirstElement = Ty->getStructElementType(N: 0);
1195 Type *SecondElement = Ty->getStructElementType(N: 1);
1196
1197 if (!FirstElement->isArrayTy())
1198 return false;
1199
1200 Type *ArrayElementType = FirstElement->getArrayElementType();
1201 if (!ArrayElementType->isStructTy() ||
1202 ArrayElementType->getStructNumElements() != 2)
1203 return false;
1204
1205 Type *T_in_struct = ArrayElementType->getStructElementType(N: 0);
1206 if (T_in_struct != SecondElement)
1207 return false;
1208
1209 auto *Padding_in_struct =
1210 dyn_cast<TargetExtType>(Val: ArrayElementType->getStructElementType(N: 1));
1211 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")
1212 return false;
1213
1214 const uint64_t ArraySize = FirstElement->getArrayNumElements();
1215 TotalSize = ArraySize + 1;
1216 OriginalElementType = ArrayElementType;
1217 return true;
1218}
1219
1220Type *reconstitutePeeledArrayType(Type *Ty) {
1221 if (!Ty->isStructTy())
1222 return Ty;
1223
1224 auto *STy = cast<StructType>(Val: Ty);
1225 Type *OriginalElementType = nullptr;
1226 uint64_t TotalSize = 0;
1227 if (matchPeeledArrayPattern(Ty: STy, OriginalElementType, TotalSize)) {
1228 Type *ResultTy = ArrayType::get(
1229 ElementType: reconstitutePeeledArrayType(Ty: OriginalElementType), NumElements: TotalSize);
1230 return ResultTy;
1231 }
1232
1233 SmallVector<Type *, 4> NewElementTypes;
1234 bool Changed = false;
1235 for (Type *ElementTy : STy->elements()) {
1236 Type *NewElementTy = reconstitutePeeledArrayType(Ty: ElementTy);
1237 if (NewElementTy != ElementTy)
1238 Changed = true;
1239 NewElementTypes.push_back(Elt: NewElementTy);
1240 }
1241
1242 if (!Changed)
1243 return Ty;
1244
1245 Type *ResultTy;
1246 if (STy->isLiteral()) {
1247 ResultTy =
1248 StructType::get(Context&: STy->getContext(), Elements: NewElementTypes, isPacked: STy->isPacked());
1249 } else {
1250 ResultTy = StructType::create(Context&: STy->getContext(), Elements: NewElementTypes,
1251 Name: STy->getName(), isPacked: STy->isPacked());
1252 }
1253 return ResultTy;
1254}
1255
1256std::optional<SPIRV::LinkageType::LinkageType>
1257getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV) {
1258 if (GV.hasLocalLinkage())
1259 return std::nullopt;
1260
1261 if (GV.isDeclarationForLinker()) {
1262 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: &GV)) {
1263 auto SC = addressSpaceToStorageClass(AddrSpace: GVar->getAddressSpace(), STI: ST);
1264 // Interface variables must not get Import linkage.
1265 if (SC == SPIRV::StorageClass::Input ||
1266 SC == SPIRV::StorageClass::Output ||
1267 SC == SPIRV::StorageClass::PushConstant)
1268 return std::nullopt;
1269 // Shaders have no linker, so module-internal storage
1270 // (e.g. HLSL groupshared) can't be imported
1271 if (ST.isShader() && (SC == SPIRV::StorageClass::Workgroup ||
1272 SC == SPIRV::StorageClass::Private))
1273 return std::nullopt;
1274 }
1275 return SPIRV::LinkageType::Import;
1276 }
1277
1278 if (GV.hasHiddenVisibility())
1279 return std::nullopt;
1280
1281 if (GV.hasLinkOnceODRLinkage() &&
1282 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr))
1283 return SPIRV::LinkageType::LinkOnceODR;
1284
1285 if (GV.hasWeakLinkage() &&
1286 ST.canUseExtension(E: SPIRV::Extension::SPV_AMD_weak_linkage))
1287 return SPIRV::LinkageType::WeakAMD;
1288
1289 return SPIRV::LinkageType::Export;
1290}
1291
1292Function *getOrCreateBackendServiceFunction(Module &M) {
1293 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
1294 if (!getVacantFunctionName(M, Name&: ServiceFunName))
1295 report_fatal_error(
1296 reason: "cannot allocate a name for the internal service function");
1297 if (Function *SF = M.getFunction(Name: ServiceFunName)) {
1298 if (SF->getInstructionCount() > 0)
1299 report_fatal_error(
1300 reason: "Unexpected combination of global variables and function pointers");
1301 return SF;
1302 }
1303 Function *SF = Function::Create(
1304 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: {}, isVarArg: false),
1305 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
1306 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
1307 return SF;
1308}
1309
1310} // namespace llvm
1311