1//===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVUtils.h"
14#include "MCTargetDesc/SPIRVBaseInfo.h"
15#include "SPIRV.h"
16#include "SPIRVGlobalRegistry.h"
17#include "SPIRVInstrInfo.h"
18#include "SPIRVSubtarget.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/Demangle/Demangle.h"
25#include "llvm/IR/IntrinsicInst.h"
26#include "llvm/IR/IntrinsicsSPIRV.h"
27#include <queue>
28#include <vector>
29
30namespace llvm {
31namespace SPIRV {
32// This code restores function args/retvalue types for composite cases
33// because the final types should still be aggregate whereas they're i32
34// during the translation to cope with aggregate flattening etc.
35// TODO: should these just return nullptr when there's no metadata?
36static FunctionType *extractFunctionTypeFromMetadata(NamedMDNode *NMD,
37 FunctionType *FTy,
38 StringRef Name) {
39 if (!NMD)
40 return FTy;
41
42 constexpr auto getConstInt = [](MDNode *MD, unsigned OpId) -> ConstantInt * {
43 if (MD->getNumOperands() <= OpId)
44 return nullptr;
45 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: OpId)))
46 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
47 return nullptr;
48 };
49
50 auto It = find_if(Range: NMD->operands(), P: [Name](MDNode *N) {
51 if (auto *MDS = dyn_cast_or_null<MDString>(Val: N->getOperand(I: 0)))
52 return MDS->getString() == Name;
53 return false;
54 });
55
56 if (It == NMD->op_end())
57 return FTy;
58
59 Type *RetTy = FTy->getReturnType();
60 SmallVector<Type *, 4> PTys(FTy->params());
61
62 for (unsigned I = 1; I != (*It)->getNumOperands(); ++I) {
63 MDNode *MD = dyn_cast<MDNode>(Val: (*It)->getOperand(I));
64 assert(MD && "MDNode operand is expected");
65
66 if (auto *Const = getConstInt(MD, 0)) {
67 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1));
68 assert(CMeta && "ConstantAsMetadata operand is expected");
69 assert(Const->getSExtValue() >= -1);
70 // Currently -1 indicates return value, greater values mean
71 // argument numbers.
72 if (Const->getSExtValue() == -1)
73 RetTy = CMeta->getType();
74 else
75 PTys[Const->getSExtValue()] = CMeta->getType();
76 }
77 }
78
79 return FunctionType::get(Result: RetTy, Params: PTys, isVarArg: FTy->isVarArg());
80}
81
82FunctionType *getOriginalFunctionType(const Function &F) {
83 return extractFunctionTypeFromMetadata(
84 NMD: F.getParent()->getNamedMetadata(Name: "spv.cloned_funcs"), FTy: F.getFunctionType(),
85 Name: F.getName());
86}
87
88FunctionType *getOriginalFunctionType(const CallBase &CB) {
89 return extractFunctionTypeFromMetadata(
90 NMD: CB.getModule()->getNamedMetadata(Name: "spv.mutated_callsites"),
91 FTy: CB.getFunctionType(), Name: CB.getName());
92}
93} // Namespace SPIRV
94
95// The following functions are used to add these string literals as a series of
96// 32-bit integer operands with the correct format, and unpack them if necessary
97// when making string comparisons in compiler passes.
98// SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment.
99static uint32_t convertCharsToWord(const StringRef &Str, unsigned i) {
100 uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars.
101 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {
102 unsigned StrIndex = i + WordIndex;
103 uint8_t CharToAdd = 0; // Initilize char as padding/null.
104 if (StrIndex < Str.size()) { // If it's within the string, get a real char.
105 CharToAdd = Str[StrIndex];
106 }
107 Word |= (CharToAdd << (WordIndex * 8));
108 }
109 return Word;
110}
111
112// Get length including padding and null terminator.
113static size_t getPaddedLen(const StringRef &Str) {
114 return (Str.size() + 4) & ~3;
115}
116
117void addStringImm(const StringRef &Str, MCInst &Inst) {
118 const size_t PaddedLen = getPaddedLen(Str);
119 for (unsigned i = 0; i < PaddedLen; i += 4) {
120 // Add an operand for the 32-bits of chars or padding.
121 Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i)));
122 }
123}
124
125void addStringImm(const StringRef &Str, MachineInstrBuilder &MIB) {
126 const size_t PaddedLen = getPaddedLen(Str);
127 for (unsigned i = 0; i < PaddedLen; i += 4) {
128 // Add an operand for the 32-bits of chars or padding.
129 MIB.addImm(Val: convertCharsToWord(Str, i));
130 }
131}
132
133void addStringImm(const StringRef &Str, IRBuilder<> &B,
134 std::vector<Value *> &Args) {
135 const size_t PaddedLen = getPaddedLen(Str);
136 for (unsigned i = 0; i < PaddedLen; i += 4) {
137 // Add a vector element for the 32-bits of chars or padding.
138 Args.push_back(x: B.getInt32(C: convertCharsToWord(Str, i)));
139 }
140}
141
142std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
143 return getSPIRVStringOperand(MI, StartIndex);
144}
145
146std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
147 MachineInstr *Def = getVRegDef(MRI, Reg);
148 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
149 "Expected G_GLOBAL_VALUE");
150 const GlobalValue *GV = Def->getOperand(i: 1).getGlobal();
151 Value *V = GV->getOperand(i: 0);
152 const ConstantDataArray *CDA = cast<ConstantDataArray>(Val: V);
153 return CDA->getAsCString().str();
154}
155
156void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
157 const auto Bitwidth = Imm.getBitWidth();
158 if (Bitwidth == 1)
159 return; // Already handled
160 else if (Bitwidth <= 32) {
161 MIB.addImm(Val: Imm.getZExtValue());
162 // Asm Printer needs this info to print floating-type correctly
163 if (Bitwidth == 16)
164 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16);
165 return;
166 } else if (Bitwidth <= 64) {
167 uint64_t FullImm = Imm.getZExtValue();
168 uint32_t LowBits = FullImm & 0xffffffff;
169 uint32_t HighBits = (FullImm >> 32) & 0xffffffff;
170 MIB.addImm(Val: LowBits).addImm(Val: HighBits);
171 // Asm Printer needs this info to print 64-bit operands correctly
172 MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH64);
173 return;
174 } else {
175 // Emit ceil(Bitwidth / 32) words to conform SPIR-V spec.
176 unsigned NumWords = (Bitwidth + 31) / 32;
177 for (unsigned I = 0; I < NumWords; ++I) {
178 unsigned LimbIdx = I / 2;
179 unsigned LimbShift = (I % 2) * 32;
180 uint32_t Word = (Imm.getRawData()[LimbIdx] >> LimbShift) & 0xffffffff;
181 MIB.addImm(Val: Word);
182 }
183 return;
184 }
185}
186
187void buildOpName(Register Target, const StringRef &Name,
188 MachineIRBuilder &MIRBuilder) {
189 if (!Name.empty()) {
190 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpName).addUse(RegNo: Target);
191 addStringImm(Str: Name, MIB);
192 }
193}
194
195void buildOpName(Register Target, const StringRef &Name, MachineInstr &I,
196 const SPIRVInstrInfo &TII) {
197 if (!Name.empty()) {
198 auto MIB =
199 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpName))
200 .addUse(RegNo: Target);
201 addStringImm(Str: Name, MIB);
202 }
203}
204
205static void finishBuildOpDecorate(MachineInstrBuilder &MIB,
206 const std::vector<uint32_t> &DecArgs,
207 StringRef StrImm) {
208 if (!StrImm.empty())
209 addStringImm(Str: StrImm, MIB);
210 for (const auto &DecArg : DecArgs)
211 MIB.addImm(Val: DecArg);
212}
213
214void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
215 SPIRV::Decoration::Decoration Dec,
216 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {
217 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
218 .addUse(RegNo: Reg)
219 .addImm(Val: static_cast<uint32_t>(Dec));
220 finishBuildOpDecorate(MIB, DecArgs, StrImm);
221}
222
223void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
224 SPIRV::Decoration::Decoration Dec,
225 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {
226 MachineBasicBlock &MBB = *I.getParent();
227 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpDecorate))
228 .addUse(RegNo: Reg)
229 .addImm(Val: static_cast<uint32_t>(Dec));
230 finishBuildOpDecorate(MIB, DecArgs, StrImm);
231}
232
233void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder,
234 SPIRV::Decoration::Decoration Dec, uint32_t Member,
235 const std::vector<uint32_t> &DecArgs,
236 StringRef StrImm) {
237 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpMemberDecorate)
238 .addUse(RegNo: Reg)
239 .addImm(Val: Member)
240 .addImm(Val: static_cast<uint32_t>(Dec));
241 finishBuildOpDecorate(MIB, DecArgs, StrImm);
242}
243
244void buildOpMemberDecorate(Register Reg, MachineInstr &I,
245 const SPIRVInstrInfo &TII,
246 SPIRV::Decoration::Decoration Dec, uint32_t Member,
247 const std::vector<uint32_t> &DecArgs,
248 StringRef StrImm) {
249 MachineBasicBlock &MBB = *I.getParent();
250 auto MIB = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: SPIRV::OpMemberDecorate))
251 .addUse(RegNo: Reg)
252 .addImm(Val: Member)
253 .addImm(Val: static_cast<uint32_t>(Dec));
254 finishBuildOpDecorate(MIB, DecArgs, StrImm);
255}
256
257void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
258 const MDNode *GVarMD, const SPIRVSubtarget &ST) {
259 for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
260 auto *OpMD = dyn_cast<MDNode>(Val: GVarMD->getOperand(I));
261 if (!OpMD)
262 report_fatal_error(reason: "Invalid decoration");
263 if (OpMD->getNumOperands() == 0)
264 report_fatal_error(reason: "Expect operand(s) of the decoration");
265 ConstantInt *DecorationId =
266 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: 0));
267 if (!DecorationId)
268 report_fatal_error(reason: "Expect SPIR-V <Decoration> operand to be the first "
269 "element of the decoration");
270
271 // The goal of `spirv.Decorations` metadata is to provide a way to
272 // represent SPIR-V entities that do not map to LLVM in an obvious way.
273 // FP flags do have obvious matches between LLVM IR and SPIR-V.
274 // Additionally, we have no guarantee at this point that the flags passed
275 // through the decoration are not violated already in the optimizer passes.
276 // Therefore, we simply ignore FP flags, including NoContraction, and
277 // FPFastMathMode.
278 if (DecorationId->getZExtValue() ==
279 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||
280 DecorationId->getZExtValue() ==
281 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {
282 continue; // Ignored.
283 }
284 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
285 .addUse(RegNo: Reg)
286 .addImm(Val: static_cast<uint32_t>(DecorationId->getZExtValue()));
287 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
288 if (ConstantInt *OpV =
289 mdconst::dyn_extract<ConstantInt>(MD: OpMD->getOperand(I: OpI)))
290 MIB.addImm(Val: static_cast<uint32_t>(OpV->getZExtValue()));
291 else if (MDString *OpV = dyn_cast<MDString>(Val: OpMD->getOperand(I: OpI)))
292 addStringImm(Str: OpV->getString(), MIB);
293 else
294 report_fatal_error(reason: "Unexpected operand of the decoration");
295 }
296 }
297}
298
299MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I) {
300 MachineFunction *MF = I.getParent()->getParent();
301 MachineBasicBlock *MBB = &MF->front();
302 MachineBasicBlock::iterator It = MBB->SkipPHIsAndLabels(I: MBB->begin()),
303 E = MBB->end();
304 bool IsHeader = false;
305 unsigned Opcode;
306 for (; It != E && It != I; ++It) {
307 Opcode = It->getOpcode();
308 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
309 IsHeader = true;
310 } else if (IsHeader &&
311 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
312 ++It;
313 break;
314 }
315 }
316 return It;
317}
318
319MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB) {
320 MachineBasicBlock::iterator I = MBB->end();
321 if (I == MBB->begin())
322 return I;
323 --I;
324 while (I->isTerminator() || I->isDebugValue()) {
325 if (I == MBB->begin())
326 break;
327 --I;
328 }
329 return I;
330}
331
332SPIRV::StorageClass::StorageClass
333addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
334 switch (AddrSpace) {
335 case 0:
336 return SPIRV::StorageClass::Function;
337 case 1:
338 return SPIRV::StorageClass::CrossWorkgroup;
339 case 2:
340 return SPIRV::StorageClass::UniformConstant;
341 case 3:
342 return SPIRV::StorageClass::Workgroup;
343 case 4:
344 return SPIRV::StorageClass::Generic;
345 case 5:
346 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
347 ? SPIRV::StorageClass::DeviceOnlyINTEL
348 : SPIRV::StorageClass::CrossWorkgroup;
349 case 6:
350 return STI.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)
351 ? SPIRV::StorageClass::HostOnlyINTEL
352 : SPIRV::StorageClass::CrossWorkgroup;
353 case 7:
354 return SPIRV::StorageClass::Input;
355 case 8:
356 return SPIRV::StorageClass::Output;
357 case 9:
358 return SPIRV::StorageClass::CodeSectionINTEL;
359 case 10:
360 return SPIRV::StorageClass::Private;
361 case 11:
362 return SPIRV::StorageClass::StorageBuffer;
363 case 12:
364 return SPIRV::StorageClass::Uniform;
365 case 13:
366 return SPIRV::StorageClass::PushConstant;
367 default:
368 report_fatal_error(reason: "Unknown address space");
369 }
370}
371
372SPIRV::MemorySemantics::MemorySemantics
373getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) {
374 switch (SC) {
375 case SPIRV::StorageClass::StorageBuffer:
376 case SPIRV::StorageClass::Uniform:
377 return SPIRV::MemorySemantics::UniformMemory;
378 case SPIRV::StorageClass::Workgroup:
379 return SPIRV::MemorySemantics::WorkgroupMemory;
380 case SPIRV::StorageClass::CrossWorkgroup:
381 return SPIRV::MemorySemantics::CrossWorkgroupMemory;
382 case SPIRV::StorageClass::AtomicCounter:
383 return SPIRV::MemorySemantics::AtomicCounterMemory;
384 case SPIRV::StorageClass::Image:
385 return SPIRV::MemorySemantics::ImageMemory;
386 default:
387 return SPIRV::MemorySemantics::None;
388 }
389}
390
391SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
392 switch (Ord) {
393 case AtomicOrdering::Acquire:
394 return SPIRV::MemorySemantics::Acquire;
395 case AtomicOrdering::Release:
396 return SPIRV::MemorySemantics::Release;
397 case AtomicOrdering::AcquireRelease:
398 return SPIRV::MemorySemantics::AcquireRelease;
399 case AtomicOrdering::SequentiallyConsistent:
400 return SPIRV::MemorySemantics::SequentiallyConsistent;
401 case AtomicOrdering::Unordered:
402 case AtomicOrdering::Monotonic:
403 case AtomicOrdering::NotAtomic:
404 return SPIRV::MemorySemantics::None;
405 }
406 llvm_unreachable(nullptr);
407}
408
409SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id) {
410 // Named by
411 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_scope_id.
412 // We don't need aliases for Invocation and CrossDevice, as we already have
413 // them covered by "singlethread" and "" strings respectively (see
414 // implementation of LLVMContext::LLVMContext()).
415 static const llvm::SyncScope::ID SubGroup =
416 Ctx.getOrInsertSyncScopeID(SSN: "subgroup");
417 static const llvm::SyncScope::ID WorkGroup =
418 Ctx.getOrInsertSyncScopeID(SSN: "workgroup");
419 static const llvm::SyncScope::ID Device =
420 Ctx.getOrInsertSyncScopeID(SSN: "device");
421
422 if (Id == llvm::SyncScope::SingleThread)
423 return SPIRV::Scope::Invocation;
424 else if (Id == llvm::SyncScope::System)
425 return SPIRV::Scope::CrossDevice;
426 else if (Id == SubGroup)
427 return SPIRV::Scope::Subgroup;
428 else if (Id == WorkGroup)
429 return SPIRV::Scope::Workgroup;
430 else if (Id == Device)
431 return SPIRV::Scope::Device;
432 return SPIRV::Scope::CrossDevice;
433}
434
435MachineInstr *getDefInstrMaybeConstant(Register &ConstReg,
436 const MachineRegisterInfo *MRI) {
437 MachineInstr *MI = MRI->getVRegDef(Reg: ConstReg);
438 MachineInstr *ConstInstr =
439 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT
440 ? MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg())
441 : MI;
442 if (auto *GI = dyn_cast<GIntrinsic>(Val: ConstInstr)) {
443 if (GI->is(ID: Intrinsic::spv_track_constant)) {
444 ConstReg = ConstInstr->getOperand(i: 2).getReg();
445 return MRI->getVRegDef(Reg: ConstReg);
446 }
447 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {
448 ConstReg = ConstInstr->getOperand(i: 1).getReg();
449 return MRI->getVRegDef(Reg: ConstReg);
450 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||
451 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {
452 ConstReg = ConstInstr->getOperand(i: 0).getReg();
453 return ConstInstr;
454 }
455 return MRI->getVRegDef(Reg: ConstReg);
456}
457
458uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) {
459 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
460 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
461 return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
462}
463
464int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI) {
465 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI);
466 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);
467 return MI->getOperand(i: 1).getCImm()->getSExtValue();
468}
469
470bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
471 if (const auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
472 return GI->is(ID: IntrinsicID);
473 return false;
474}
475
476Type *getMDOperandAsType(const MDNode *N, unsigned I) {
477 Type *ElementTy = cast<ValueAsMetadata>(Val: N->getOperand(I))->getType();
478 return toTypedPointer(Ty: ElementTy);
479}
480
481// The set of names is borrowed from the SPIR-V translator.
482// TODO: may be implemented in SPIRVBuiltins.td.
483static bool isPipeOrAddressSpaceCastBI(const StringRef MangledName) {
484 return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" ||
485 MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" ||
486 MangledName == "write_pipe_4" || MangledName == "read_pipe_4" ||
487 MangledName == "reserve_write_pipe" ||
488 MangledName == "reserve_read_pipe" ||
489 MangledName == "commit_write_pipe" ||
490 MangledName == "commit_read_pipe" ||
491 MangledName == "work_group_reserve_write_pipe" ||
492 MangledName == "work_group_reserve_read_pipe" ||
493 MangledName == "work_group_commit_write_pipe" ||
494 MangledName == "work_group_commit_read_pipe" ||
495 MangledName == "get_pipe_num_packets_ro" ||
496 MangledName == "get_pipe_max_packets_ro" ||
497 MangledName == "get_pipe_num_packets_wo" ||
498 MangledName == "get_pipe_max_packets_wo" ||
499 MangledName == "sub_group_reserve_write_pipe" ||
500 MangledName == "sub_group_reserve_read_pipe" ||
501 MangledName == "sub_group_commit_write_pipe" ||
502 MangledName == "sub_group_commit_read_pipe" ||
503 MangledName == "to_global" || MangledName == "to_local" ||
504 MangledName == "to_private";
505}
506
507static bool isEnqueueKernelBI(const StringRef MangledName) {
508 return MangledName == "__enqueue_kernel_basic" ||
509 MangledName == "__enqueue_kernel_basic_events" ||
510 MangledName == "__enqueue_kernel_varargs" ||
511 MangledName == "__enqueue_kernel_events_varargs";
512}
513
514static bool isKernelQueryBI(const StringRef MangledName) {
515 return MangledName == "__get_kernel_work_group_size_impl" ||
516 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
517 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
518 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
519}
520
521static bool isNonMangledOCLBuiltin(StringRef Name) {
522 if (!Name.starts_with(Prefix: "__"))
523 return false;
524
525 return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) ||
526 isPipeOrAddressSpaceCastBI(MangledName: Name.drop_front(N: 2)) ||
527 Name == "__translate_sampler_initializer";
528}
529
530std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
531 bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
532 bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_");
533 bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_");
534 bool IsMangled = Name.starts_with(Prefix: "_Z");
535
536 // Otherwise use simple demangling to return the function name.
537 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
538 return Name.str();
539
540 // Try to use the itanium demangler.
541 if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) {
542 std::string Result = DemangledName;
543 free(ptr: DemangledName);
544 return Result;
545 }
546
547 // Autocheck C++, maybe need to do explicit check of the source language.
548 // OpenCL C++ built-ins are declared in cl namespace.
549 // TODO: consider using 'St' abbriviation for cl namespace mangling.
550 // Similar to ::std:: in C++.
551 size_t Start, Len = 0;
552 size_t DemangledNameLenStart = 2;
553 if (Name.starts_with(Prefix: "_ZN")) {
554 // Skip CV and ref qualifiers.
555 size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO", From: 3);
556 // All built-ins are in the ::cl:: namespace.
557 if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv")
558 return std::string();
559 DemangledNameLenStart = NameSpaceStart + 11;
560 }
561 Start = Name.find_first_not_of(Chars: "0123456789", From: DemangledNameLenStart);
562 [[maybe_unused]] bool Error =
563 Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart)
564 .getAsInteger(Radix: 10, Result&: Len);
565 assert(!Error && "Failed to parse demangled name length");
566 return Name.substr(Start, N: Len).str();
567}
568
569bool hasBuiltinTypePrefix(StringRef Name) {
570 if (Name.starts_with(Prefix: "opencl.") || Name.starts_with(Prefix: "ocl_") ||
571 Name.starts_with(Prefix: "spirv."))
572 return true;
573 return false;
574}
575
576bool isSpecialOpaqueType(const Type *Ty) {
577 if (const TargetExtType *ExtTy = dyn_cast<TargetExtType>(Val: Ty))
578 return isTypedPointerWrapper(ExtTy)
579 ? false
580 : hasBuiltinTypePrefix(Name: ExtTy->getName());
581
582 return false;
583}
584
585bool isEntryPoint(const Function &F) {
586 // OpenCL handling: any function with the SPIR_KERNEL
587 // calling convention will be a potential entry point.
588 if (F.getCallingConv() == CallingConv::SPIR_KERNEL)
589 return true;
590
591 // HLSL handling: special attribute are emitted from the
592 // front-end.
593 if (F.getFnAttribute(Kind: "hlsl.shader").isValid())
594 return true;
595
596 return false;
597}
598
599Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
600 TypeName.consume_front(Prefix: "atomic_");
601 if (TypeName.consume_front(Prefix: "void"))
602 return Type::getVoidTy(C&: Ctx);
603 else if (TypeName.consume_front(Prefix: "bool") || TypeName.consume_front(Prefix: "_Bool"))
604 return Type::getIntNTy(C&: Ctx, N: 1);
605 else if (TypeName.consume_front(Prefix: "char") ||
606 TypeName.consume_front(Prefix: "signed char") ||
607 TypeName.consume_front(Prefix: "unsigned char") ||
608 TypeName.consume_front(Prefix: "uchar"))
609 return Type::getInt8Ty(C&: Ctx);
610 else if (TypeName.consume_front(Prefix: "short") ||
611 TypeName.consume_front(Prefix: "signed short") ||
612 TypeName.consume_front(Prefix: "unsigned short") ||
613 TypeName.consume_front(Prefix: "ushort"))
614 return Type::getInt16Ty(C&: Ctx);
615 else if (TypeName.consume_front(Prefix: "int") ||
616 TypeName.consume_front(Prefix: "signed int") ||
617 TypeName.consume_front(Prefix: "unsigned int") ||
618 TypeName.consume_front(Prefix: "uint"))
619 return Type::getInt32Ty(C&: Ctx);
620 else if (TypeName.consume_front(Prefix: "long") ||
621 TypeName.consume_front(Prefix: "signed long") ||
622 TypeName.consume_front(Prefix: "unsigned long") ||
623 TypeName.consume_front(Prefix: "ulong"))
624 return Type::getInt64Ty(C&: Ctx);
625 else if (TypeName.consume_front(Prefix: "half") ||
626 TypeName.consume_front(Prefix: "_Float16") ||
627 TypeName.consume_front(Prefix: "__fp16"))
628 return Type::getHalfTy(C&: Ctx);
629 else if (TypeName.consume_front(Prefix: "float"))
630 return Type::getFloatTy(C&: Ctx);
631 else if (TypeName.consume_front(Prefix: "double"))
632 return Type::getDoubleTy(C&: Ctx);
633
634 // Unable to recognize SPIRV type name
635 return nullptr;
636}
637
638std::unordered_set<BasicBlock *>
639PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {
640 std::queue<BasicBlock *> ToVisit;
641 ToVisit.push(x: Start);
642
643 std::unordered_set<BasicBlock *> Output;
644 while (ToVisit.size() != 0) {
645 BasicBlock *BB = ToVisit.front();
646 ToVisit.pop();
647
648 if (Output.count(x: BB) != 0)
649 continue;
650 Output.insert(x: BB);
651
652 for (BasicBlock *Successor : successors(BB)) {
653 if (DT.dominates(A: Successor, B: BB))
654 continue;
655 ToVisit.push(x: Successor);
656 }
657 }
658
659 return Output;
660}
661
662bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {
663 for (BasicBlock *P : predecessors(BB)) {
664 // Ignore back-edges.
665 if (DT.dominates(A: BB, B: P))
666 continue;
667
668 // One of the predecessor hasn't been visited. Not ready yet.
669 if (BlockToOrder.count(x: P) == 0)
670 return false;
671
672 // If the block is a loop exit, the loop must be finished before
673 // we can continue.
674 Loop *L = LI.getLoopFor(BB: P);
675 if (L == nullptr || L->contains(BB))
676 continue;
677
678 // SPIR-V requires a single back-edge. And the backend first
679 // step transforms loops into the simplified format. If we have
680 // more than 1 back-edge, something is wrong.
681 assert(L->getNumBackEdges() <= 1);
682
683 // If the loop has no latch, loop's rank won't matter, so we can
684 // proceed.
685 BasicBlock *Latch = L->getLoopLatch();
686 assert(Latch);
687 if (Latch == nullptr)
688 continue;
689
690 // The latch is not ready yet, let's wait.
691 if (BlockToOrder.count(x: Latch) == 0)
692 return false;
693 }
694
695 return true;
696}
697
698size_t PartialOrderingVisitor::GetNodeRank(BasicBlock *BB) const {
699 auto It = BlockToOrder.find(x: BB);
700 if (It != BlockToOrder.end())
701 return It->second.Rank;
702
703 size_t result = 0;
704 for (BasicBlock *P : predecessors(BB)) {
705 // Ignore back-edges.
706 if (DT.dominates(A: BB, B: P))
707 continue;
708
709 auto Iterator = BlockToOrder.end();
710 Loop *L = LI.getLoopFor(BB: P);
711 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;
712
713 // If the predecessor is either outside a loop, or part of
714 // the same loop, simply take its rank + 1.
715 if (L == nullptr || L->contains(BB) || Latch == nullptr) {
716 Iterator = BlockToOrder.find(x: P);
717 } else {
718 // Otherwise, take the loop's rank (highest rank in the loop) as base.
719 // Since loops have a single latch, highest rank is easy to find.
720 // If the loop has no latch, then it doesn't matter.
721 Iterator = BlockToOrder.find(x: Latch);
722 }
723
724 assert(Iterator != BlockToOrder.end());
725 result = std::max(a: result, b: Iterator->second.Rank + 1);
726 }
727
728 return result;
729}
730
731size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {
732 ToVisit.push(x: BB);
733 Queued.insert(x: BB);
734
735 size_t QueueIndex = 0;
736 while (ToVisit.size() != 0) {
737 BasicBlock *BB = ToVisit.front();
738 ToVisit.pop();
739
740 if (!CanBeVisited(BB)) {
741 ToVisit.push(x: BB);
742 if (QueueIndex >= ToVisit.size())
743 llvm::report_fatal_error(
744 reason: "No valid candidate in the queue. Is the graph reducible?");
745 QueueIndex++;
746 continue;
747 }
748
749 QueueIndex = 0;
750 size_t Rank = GetNodeRank(BB);
751 OrderInfo Info = {.Rank: Rank, .TraversalIndex: BlockToOrder.size()};
752 BlockToOrder.emplace(args&: BB, args&: Info);
753
754 for (BasicBlock *S : successors(BB)) {
755 if (Queued.count(x: S) != 0)
756 continue;
757 ToVisit.push(x: S);
758 Queued.insert(x: S);
759 }
760 }
761
762 return 0;
763}
764
765PartialOrderingVisitor::PartialOrderingVisitor(Function &F) {
766 DT.recalculate(Func&: F);
767 LI = LoopInfo(DT);
768
769 visit(BB: &*F.begin(), Unused: 0);
770
771 Order.reserve(n: F.size());
772 for (auto &[BB, Info] : BlockToOrder)
773 Order.emplace_back(args: BB);
774
775 std::sort(first: Order.begin(), last: Order.end(), comp: [&](const auto &LHS, const auto &RHS) {
776 return compare(LHS, RHS);
777 });
778}
779
780bool PartialOrderingVisitor::compare(const BasicBlock *LHS,
781 const BasicBlock *RHS) const {
782 const OrderInfo &InfoLHS = BlockToOrder.at(k: const_cast<BasicBlock *>(LHS));
783 const OrderInfo &InfoRHS = BlockToOrder.at(k: const_cast<BasicBlock *>(RHS));
784 if (InfoLHS.Rank != InfoRHS.Rank)
785 return InfoLHS.Rank < InfoRHS.Rank;
786 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;
787}
788
789void PartialOrderingVisitor::partialOrderVisit(
790 BasicBlock &Start, std::function<bool(BasicBlock *)> Op) {
791 std::unordered_set<BasicBlock *> Reachable = getReachableFrom(Start: &Start);
792 assert(BlockToOrder.count(&Start) != 0);
793
794 // Skipping blocks with a rank inferior to |Start|'s rank.
795 auto It = Order.begin();
796 while (It != Order.end() && *It != &Start)
797 ++It;
798
799 // This is unexpected. Worst case |Start| is the last block,
800 // so It should point to the last block, not past-end.
801 assert(It != Order.end());
802
803 // By default, there is no rank limit. Setting it to the maximum value.
804 std::optional<size_t> EndRank = std::nullopt;
805 for (; It != Order.end(); ++It) {
806 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)
807 break;
808
809 if (Reachable.count(x: *It) == 0) {
810 continue;
811 }
812
813 if (!Op(*It)) {
814 EndRank = BlockToOrder[*It].Rank;
815 }
816 }
817}
818
819bool sortBlocks(Function &F) {
820 if (F.size() == 0)
821 return false;
822
823 bool Modified = false;
824 std::vector<BasicBlock *> Order;
825 Order.reserve(n: F.size());
826
827 ReversePostOrderTraversal<Function *> RPOT(&F);
828 llvm::append_range(C&: Order, R&: RPOT);
829
830 assert(&*F.begin() == Order[0]);
831 BasicBlock *LastBlock = &*F.begin();
832 for (BasicBlock *BB : Order) {
833 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {
834 Modified = true;
835 BB->moveAfter(MovePos: LastBlock);
836 }
837 LastBlock = BB;
838 }
839
840 return Modified;
841}
842
843MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
844 MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
845 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
846 MaybeDef = MRI.getVRegDef(Reg: MaybeDef->getOperand(i: 1).getReg());
847 return MaybeDef;
848}
849
850bool getVacantFunctionName(Module &M, std::string &Name) {
851 // It's a bit of paranoia, but still we don't want to have even a chance that
852 // the loop will work for too long.
853 constexpr unsigned MaxIters = 1024;
854 for (unsigned I = 0; I < MaxIters; ++I) {
855 std::string OrdName = Name + Twine(I).str();
856 if (!M.getFunction(Name: OrdName)) {
857 Name = std::move(OrdName);
858 return true;
859 }
860 }
861 return false;
862}
863
864// Assign SPIR-V type to the register. If the register has no valid assigned
865// class, set register LLT type and class according to the SPIR-V type.
866void setRegClassType(Register Reg, SPIRVTypeInst SpvType,
867 SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI,
868 const MachineFunction &MF, bool Force) {
869 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
870 if (!MRI->getRegClassOrNull(Reg) || Force) {
871 MRI->setRegClass(Reg, RC: GR->getRegClass(SpvType));
872 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
873 }
874}
875
876// Create a SPIR-V type, assign SPIR-V type to the register. If the register has
877// no valid assigned class, set register LLT type and class according to the
878// SPIR-V type.
879void setRegClassType(Register Reg, const Type *Ty, SPIRVGlobalRegistry *GR,
880 MachineIRBuilder &MIRBuilder,
881 SPIRV::AccessQualifier::AccessQualifier AccessQual,
882 bool EmitIR, bool Force) {
883 setRegClassType(Reg,
884 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR),
885 GR, MRI: MIRBuilder.getMRI(), MF: MIRBuilder.getMF(), Force);
886}
887
888// Create a virtual register and assign SPIR-V type to the register. Set
889// register LLT type and class according to the SPIR-V type.
890Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
891 MachineRegisterInfo *MRI,
892 const MachineFunction &MF) {
893 Register Reg = MRI->createVirtualRegister(RegClass: GR->getRegClass(SpvType));
894 MRI->setType(VReg: Reg, Ty: GR->getRegType(SpvType));
895 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF);
896 return Reg;
897}
898
899// Create a virtual register and assign SPIR-V type to the register. Set
900// register LLT type and class according to the SPIR-V type.
901Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR,
902 MachineIRBuilder &MIRBuilder) {
903 return createVirtualRegister(SpvType, GR, MRI: MIRBuilder.getMRI(),
904 MF: MIRBuilder.getMF());
905}
906
907// Create a SPIR-V type, virtual register and assign SPIR-V type to the
908// register. Set register LLT type and class according to the SPIR-V type.
909Register createVirtualRegister(
910 const Type *Ty, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIRBuilder,
911 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
912 return createVirtualRegister(
913 SpvType: GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder, AQ: AccessQual, EmitIR), GR,
914 MIRBuilder);
915}
916
917CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
918 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
919 IRBuilder<> &B) {
920 SmallVector<Value *, 4> Args;
921 Args.push_back(Elt: Arg2);
922 Args.push_back(Elt: buildMD(Arg));
923 llvm::append_range(C&: Args, R&: Imms);
924 return B.CreateIntrinsic(ID: IntrID, Types: {Types}, Args);
925}
926
927// Return true if there is an opaque pointer type nested in the argument.
928bool isNestedPointer(const Type *Ty) {
929 if (Ty->isPtrOrPtrVectorTy())
930 return true;
931 if (const FunctionType *RefTy = dyn_cast<FunctionType>(Val: Ty)) {
932 if (isNestedPointer(Ty: RefTy->getReturnType()))
933 return true;
934 for (const Type *ArgTy : RefTy->params())
935 if (isNestedPointer(Ty: ArgTy))
936 return true;
937 return false;
938 }
939 if (const ArrayType *RefTy = dyn_cast<ArrayType>(Val: Ty))
940 return isNestedPointer(Ty: RefTy->getElementType());
941 return false;
942}
943
944bool isSpvIntrinsic(const Value *Arg) {
945 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Arg))
946 if (Function *F = II->getCalledFunction())
947 if (F->getName().starts_with(Prefix: "llvm.spv."))
948 return true;
949 return false;
950}
951
952// Function to create continued instructions for SPV_INTEL_long_composites
953// extension
954SmallVector<MachineInstr *, 4>
955createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode,
956 unsigned MinWC, unsigned ContinuedOpcode,
957 ArrayRef<Register> Args, Register ReturnRegister,
958 Register TypeID) {
959
960 SmallVector<MachineInstr *, 4> Instructions;
961 constexpr unsigned MaxWordCount = UINT16_MAX;
962 const size_t NumElements = Args.size();
963 size_t MaxNumElements = MaxWordCount - MinWC;
964 size_t SPIRVStructNumElements = NumElements;
965
966 if (NumElements > MaxNumElements) {
967 // Do adjustments for continued instructions which always had only one
968 // minumum word count.
969 SPIRVStructNumElements = MaxNumElements;
970 MaxNumElements = MaxWordCount - 1;
971 }
972
973 auto MIB =
974 MIRBuilder.buildInstr(Opcode).addDef(RegNo: ReturnRegister).addUse(RegNo: TypeID);
975
976 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
977 MIB.addUse(RegNo: Args[I]);
978
979 Instructions.push_back(Elt: MIB.getInstr());
980
981 for (size_t I = SPIRVStructNumElements; I < NumElements;
982 I += MaxNumElements) {
983 auto MIB = MIRBuilder.buildInstr(Opcode: ContinuedOpcode);
984 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
985 MIB.addUse(RegNo: Args[J]);
986 Instructions.push_back(Elt: MIB.getInstr());
987 }
988 return Instructions;
989}
990
991SmallVector<unsigned, 1>
992getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD) {
993 unsigned LC = SPIRV::LoopControl::None;
994 // Currently used only to store PartialCount value. Later when other
995 // LoopControls are added - this map should be sorted before making
996 // them loop_merge operands to satisfy 3.23. Loop Control requirements.
997 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;
998 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.disable")) {
999 LC |= SPIRV::LoopControl::DontUnroll;
1000 } else {
1001 if (findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.enable") ||
1002 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.full")) {
1003 LC |= SPIRV::LoopControl::Unroll;
1004 }
1005 if (MDNode *CountMD =
1006 findOptionMDForLoopID(LoopID: LoopMD, Name: "llvm.loop.unroll.count")) {
1007 if (auto *CI =
1008 mdconst::extract_or_null<ConstantInt>(MD: CountMD->getOperand(I: 1))) {
1009 unsigned Count = CI->getZExtValue();
1010 if (Count != 1) {
1011 LC |= SPIRV::LoopControl::PartialCount;
1012 MaskToValueMap.emplace_back(
1013 args: std::make_pair(x: SPIRV::LoopControl::PartialCount, y&: Count));
1014 }
1015 }
1016 }
1017 }
1018 SmallVector<unsigned, 1> Result = {LC};
1019 for (auto &[Mask, Val] : MaskToValueMap)
1020 Result.push_back(Elt: Val);
1021 return Result;
1022}
1023
1024SmallVector<unsigned, 1> getSpirvLoopControlOperandsFromLoopMetadata(Loop *L) {
1025 return getSpirvLoopControlOperandsFromLoopMetadata(LoopMD: L->getLoopID());
1026}
1027
1028const std::set<unsigned> &getTypeFoldingSupportedOpcodes() {
1029 // clang-format off
1030 static const std::set<unsigned> TypeFoldingSupportingOpcs = {
1031 TargetOpcode::G_ADD,
1032 TargetOpcode::G_FADD,
1033 TargetOpcode::G_STRICT_FADD,
1034 TargetOpcode::G_SUB,
1035 TargetOpcode::G_FSUB,
1036 TargetOpcode::G_STRICT_FSUB,
1037 TargetOpcode::G_MUL,
1038 TargetOpcode::G_FMUL,
1039 TargetOpcode::G_STRICT_FMUL,
1040 TargetOpcode::G_SDIV,
1041 TargetOpcode::G_UDIV,
1042 TargetOpcode::G_FDIV,
1043 TargetOpcode::G_STRICT_FDIV,
1044 TargetOpcode::G_SREM,
1045 TargetOpcode::G_UREM,
1046 TargetOpcode::G_FREM,
1047 TargetOpcode::G_STRICT_FREM,
1048 TargetOpcode::G_FNEG,
1049 TargetOpcode::G_CONSTANT,
1050 TargetOpcode::G_FCONSTANT,
1051 TargetOpcode::G_AND,
1052 TargetOpcode::G_OR,
1053 TargetOpcode::G_XOR,
1054 TargetOpcode::G_SHL,
1055 TargetOpcode::G_ASHR,
1056 TargetOpcode::G_LSHR,
1057 TargetOpcode::G_SELECT,
1058 TargetOpcode::G_EXTRACT_VECTOR_ELT,
1059 };
1060 // clang-format on
1061 return TypeFoldingSupportingOpcs;
1062}
1063
1064bool isTypeFoldingSupported(unsigned Opcode) {
1065 return getTypeFoldingSupportedOpcodes().count(x: Opcode) > 0;
1066}
1067
1068// Traversing [g]MIR accounting for pseudo-instructions.
1069MachineInstr *passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI) {
1070 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||
1071 Def->getOpcode() == TargetOpcode::COPY)
1072 ? MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())
1073 : Def;
1074}
1075
1076MachineInstr *getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1077 if (MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg()))
1078 return passCopy(Def, MRI);
1079 return nullptr;
1080}
1081
1082MachineInstr *getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1083 if (MachineInstr *Def = getDef(MO, MRI)) {
1084 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
1085 Def->getOpcode() == SPIRV::OpConstantI)
1086 return Def;
1087 }
1088 return nullptr;
1089}
1090
1091int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
1092 if (MachineInstr *Def = getImm(MO, MRI)) {
1093 if (Def->getOpcode() == SPIRV::OpConstantI)
1094 return Def->getOperand(i: 2).getImm();
1095 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
1096 return Def->getOperand(i: 1).getCImm()->getZExtValue();
1097 }
1098 llvm_unreachable("Unexpected integer constant pattern");
1099}
1100
1101unsigned getArrayComponentCount(const MachineRegisterInfo *MRI,
1102 const MachineInstr *ResType) {
1103 return foldImm(MO: ResType->getOperand(i: 2), MRI);
1104}
1105
1106MachineBasicBlock::iterator
1107getFirstValidInstructionInsertPoint(MachineBasicBlock &BB) {
1108 // Find the position to insert the OpVariable instruction.
1109 // We will insert it after the last OpFunctionParameter, if any, or
1110 // after OpFunction otherwise.
1111 MachineBasicBlock::iterator VarPos = BB.begin();
1112 while (VarPos != BB.end() && VarPos->getOpcode() != SPIRV::OpFunction) {
1113 ++VarPos;
1114 }
1115 // Advance VarPos to the next instruction after OpFunction, it will either
1116 // be an OpFunctionParameter, so that we can start the next loop, or the
1117 // position to insert the OpVariable instruction.
1118 ++VarPos;
1119 while (VarPos != BB.end() &&
1120 VarPos->getOpcode() == SPIRV::OpFunctionParameter) {
1121 ++VarPos;
1122 }
1123 // VarPos is now pointing at after the last OpFunctionParameter, if any,
1124 // or after OpFunction, if no parameters.
1125 return VarPos != BB.end() && VarPos->getOpcode() == SPIRV::OpLabel ? ++VarPos
1126 : VarPos;
1127}
1128
1129bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType,
1130 uint64_t &TotalSize) {
1131 // An array of N padded structs is represented as {[N-1 x <{T, pad}>], T}.
1132 if (Ty->getStructNumElements() != 2)
1133 return false;
1134
1135 Type *FirstElement = Ty->getStructElementType(N: 0);
1136 Type *SecondElement = Ty->getStructElementType(N: 1);
1137
1138 if (!FirstElement->isArrayTy())
1139 return false;
1140
1141 Type *ArrayElementType = FirstElement->getArrayElementType();
1142 if (!ArrayElementType->isStructTy() ||
1143 ArrayElementType->getStructNumElements() != 2)
1144 return false;
1145
1146 Type *T_in_struct = ArrayElementType->getStructElementType(N: 0);
1147 if (T_in_struct != SecondElement)
1148 return false;
1149
1150 auto *Padding_in_struct =
1151 dyn_cast<TargetExtType>(Val: ArrayElementType->getStructElementType(N: 1));
1152 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")
1153 return false;
1154
1155 const uint64_t ArraySize = FirstElement->getArrayNumElements();
1156 TotalSize = ArraySize + 1;
1157 OriginalElementType = ArrayElementType;
1158 return true;
1159}
1160
1161Type *reconstitutePeeledArrayType(Type *Ty) {
1162 if (!Ty->isStructTy())
1163 return Ty;
1164
1165 auto *STy = cast<StructType>(Val: Ty);
1166 Type *OriginalElementType = nullptr;
1167 uint64_t TotalSize = 0;
1168 if (matchPeeledArrayPattern(Ty: STy, OriginalElementType, TotalSize)) {
1169 Type *ResultTy = ArrayType::get(
1170 ElementType: reconstitutePeeledArrayType(Ty: OriginalElementType), NumElements: TotalSize);
1171 return ResultTy;
1172 }
1173
1174 SmallVector<Type *, 4> NewElementTypes;
1175 bool Changed = false;
1176 for (Type *ElementTy : STy->elements()) {
1177 Type *NewElementTy = reconstitutePeeledArrayType(Ty: ElementTy);
1178 if (NewElementTy != ElementTy)
1179 Changed = true;
1180 NewElementTypes.push_back(Elt: NewElementTy);
1181 }
1182
1183 if (!Changed)
1184 return Ty;
1185
1186 Type *ResultTy;
1187 if (STy->isLiteral())
1188 ResultTy =
1189 StructType::get(Context&: STy->getContext(), Elements: NewElementTypes, isPacked: STy->isPacked());
1190 else {
1191 auto *NewTy = StructType::create(Context&: STy->getContext(), Name: STy->getName());
1192 NewTy->setBody(Elements: NewElementTypes, isPacked: STy->isPacked());
1193 ResultTy = NewTy;
1194 }
1195 return ResultTy;
1196}
1197
1198std::optional<SPIRV::LinkageType::LinkageType>
1199getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV) {
1200 if (GV.hasLocalLinkage() || GV.hasHiddenVisibility())
1201 return std::nullopt;
1202
1203 if (GV.isDeclarationForLinker())
1204 return SPIRV::LinkageType::Import;
1205
1206 if (GV.hasLinkOnceODRLinkage() &&
1207 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_linkonce_odr))
1208 return SPIRV::LinkageType::LinkOnceODR;
1209
1210 return SPIRV::LinkageType::Export;
1211}
1212
1213} // namespace llvm
1214