1//===-- NVPTXAsmPrinter.cpp - NVPTX LLVM assembly writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to NVPTX assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXAsmPrinter.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "MCTargetDesc/NVPTXInstPrinter.h"
17#include "MCTargetDesc/NVPTXTargetStreamer.h"
18#include "NVPTX.h"
19#include "NVPTXDwarfDebug.h"
20#include "NVPTXMCExpr.h"
21#include "NVPTXMachineFunctionInfo.h"
22#include "NVPTXRegisterInfo.h"
23#include "NVPTXSubtarget.h"
24#include "NVPTXTargetMachine.h"
25#include "NVPTXUtilities.h"
26#include "NVVMProperties.h"
27#include "TargetInfo/NVPTXTargetInfo.h"
28#include "cl_common_defines.h"
29#include "llvm/ADT/APFloat.h"
30#include "llvm/ADT/APInt.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/DenseSet.h"
34#include "llvm/ADT/SCCIterator.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/Sequence.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/SmallString.h"
39#include "llvm/ADT/SmallVector.h"
40#include "llvm/ADT/StringExtras.h"
41#include "llvm/ADT/StringRef.h"
42#include "llvm/ADT/Twine.h"
43#include "llvm/ADT/iterator_range.h"
44#include "llvm/Analysis/ConstantFolding.h"
45#include "llvm/CodeGen/Analysis.h"
46#include "llvm/CodeGen/AsmPrinter.h"
47#include "llvm/CodeGen/AsmPrinterAnalysis.h"
48#include "llvm/CodeGen/MachineBasicBlock.h"
49#include "llvm/CodeGen/MachineFrameInfo.h"
50#include "llvm/CodeGen/MachineFunction.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineJumpTableInfo.h"
53#include "llvm/CodeGen/MachineLoopInfo.h"
54#include "llvm/CodeGen/MachineModuleInfo.h"
55#include "llvm/CodeGen/MachineOperand.h"
56#include "llvm/CodeGen/MachineRegisterInfo.h"
57#include "llvm/CodeGen/TargetRegisterInfo.h"
58#include "llvm/CodeGen/ValueTypes.h"
59#include "llvm/CodeGenTypes/MachineValueType.h"
60#include "llvm/IR/Argument.h"
61#include "llvm/IR/Attributes.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DebugLoc.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/Function.h"
71#include "llvm/IR/GlobalAlias.h"
72#include "llvm/IR/GlobalValue.h"
73#include "llvm/IR/GlobalVariable.h"
74#include "llvm/IR/InstrTypes.h"
75#include "llvm/IR/Instruction.h"
76#include "llvm/IR/LLVMContext.h"
77#include "llvm/IR/Module.h"
78#include "llvm/IR/Operator.h"
79#include "llvm/IR/Type.h"
80#include "llvm/IR/User.h"
81#include "llvm/IR/Value.h"
82#include "llvm/MC/MCExpr.h"
83#include "llvm/MC/MCInst.h"
84#include "llvm/MC/MCInstrDesc.h"
85#include "llvm/MC/MCStreamer.h"
86#include "llvm/MC/MCSymbol.h"
87#include "llvm/MC/TargetRegistry.h"
88#include "llvm/Pass.h"
89#include "llvm/Support/Alignment.h"
90#include "llvm/Support/Casting.h"
91#include "llvm/Support/Compiler.h"
92#include "llvm/Support/Endian.h"
93#include "llvm/Support/ErrorHandling.h"
94#include "llvm/Support/NativeFormatting.h"
95#include "llvm/Support/raw_ostream.h"
96#include "llvm/Target/TargetLoweringObjectFile.h"
97#include "llvm/Target/TargetMachine.h"
98#include "llvm/Transforms/Utils/UnrollLoop.h"
99#include <algorithm>
100#include <cassert>
101#include <cstdint>
102#include <cstring>
103#include <map>
104#include <memory>
105#include <set>
106#include <string>
107#include <type_traits>
108#include <vector>
109
110using namespace llvm;
111
112#define DEPOTNAME "__local_depot"
113
114// The ptx syntax and format is very different from that usually seem in a .s
115// file,
116// therefore we are not able to use the MCAsmStreamer interface here.
117//
118// We are handcrafting the output method here.
119//
120// A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer
121// (subclass of MCStreamer).
122
123namespace {
124
125class NVPTXAsmPrinter : public AsmPrinter {
126
127 class AggBuffer {
128 // Used to buffer the emitted string for initializing global aggregates.
129 //
130 // Normally an aggregate (array, vector, or structure) is emitted as a u8[].
131 // However, if either element/field of the aggregate is a non-NULL address,
132 // and all such addresses are properly aligned, then the aggregate is
133 // emitted as u32[] or u64[]. In the case of unaligned addresses, the
134 // aggregate is emitted as u8[], and the mask() operator is used for all
135 // pointers.
136 //
137 // We first layout the aggregate in 'buffer' in bytes, except for those
138 // symbol addresses. For the i-th symbol address in the aggregate, its
139 // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s.
140 // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1]
141 // records the Value*.
142 //
143 // Once we have this AggBuffer setup, we can choose how to print it out.
144 public:
145 // number of symbol addresses
146 unsigned numSymbols() const { return Symbols.size(); }
147
148 bool allSymbolsAligned(unsigned ptrSize) const {
149 return llvm::all_of(Range: symbolPosInBuffer,
150 P: [=](unsigned pos) { return pos % ptrSize == 0; });
151 }
152
153 private:
154 const unsigned Size; // size of the buffer in bytes
155 std::vector<unsigned char> buffer; // the buffer
156 SmallVector<unsigned, 4> symbolPosInBuffer;
157 SmallVector<const Value *, 4> Symbols;
158 // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
159 // stripping pointer casts, i.e.,
160 // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
161 //
162 // We need to keep these values because AggBuffer::print decides whether to
163 // emit a "generic()" cast for Symbols[i] depending on the address space of
164 // SymbolsBeforeStripping[i].
165 SmallVector<const Value *, 4> SymbolsBeforeStripping;
166 unsigned curpos;
167 const NVPTXAsmPrinter &AP;
168 const bool EmitGeneric;
169
170 public:
171 AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP)
172 : Size(Size), buffer(Size), curpos(0), AP(AP),
173 EmitGeneric(AP.EmitGeneric) {}
174
175 unsigned getBufferSize() const { return Size; }
176
177 // Number of bytes written so far.
178 unsigned getCurpos() const { return curpos; }
179
180 // Copy Num bytes from Ptr.
181 // if Bytes > Num, zero fill up to Bytes.
182 void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
183 for (unsigned I : llvm::seq(Size: Num))
184 addByte(Byte: Ptr[I]);
185 if (Bytes > Num)
186 addZeros(Num: Bytes - Num);
187 }
188
189 void addByte(uint8_t Byte) {
190 assert(curpos < Size);
191 buffer[curpos] = Byte;
192 curpos++;
193 }
194
195 void addZeros(unsigned Num) {
196 for ([[maybe_unused]] unsigned _ : llvm::seq(Size: Num)) {
197 addByte(Byte: 0);
198 }
199 }
200
201 void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
202 symbolPosInBuffer.push_back(Elt: curpos);
203 Symbols.push_back(Elt: GVar);
204 SymbolsBeforeStripping.push_back(Elt: GVarBeforeStripping);
205 }
206
207 void printBytes(raw_ostream &os);
208 void printWords(raw_ostream &os);
209
210 private:
211 void printSymbol(unsigned nSym, raw_ostream &os);
212 };
213
214 friend class AggBuffer;
215
216public:
217 static char ID;
218
219 StringRef getPassName() const override { return "NVPTX Assembly Printer"; }
220
221private:
222 const Function *F;
223
224 NVPTXTargetStreamer *getTargetStreamer() const;
225
226 void emitStartOfAsmFile(Module &M) override;
227 void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
228 void emitFunctionEntryLabel() override;
229 void emitFunctionBodyStart() override;
230 void emitFunctionBodyEnd() override;
231 void emitImplicitDef(const MachineInstr *MI) const override;
232
233 void emitInstruction(const MachineInstr *) override;
234 void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
235 MCOperand lowerOperand(const MachineOperand &MO);
236 MCOperand GetSymbolRef(const MCSymbol *Symbol);
237 MCRegister encodeVirtualRegister(Register Reg);
238
239 /// The number \p Reg was assigned within its register class, as declared by
240 /// this function's .reg directives.
241 unsigned getVirtualRegisterNumber(Register Reg) const;
242
243 void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O,
244 const char *Modifier = nullptr);
245 void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O,
246 bool processDemoted, const NVPTXSubtarget &STI);
247 void emitGlobals(const Module &M);
248 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
249 void emitHeader(Module &M, const NVPTXSubtarget &STI);
250 void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const;
251 void emitFunctionParamList(const Function *, raw_ostream &O);
252 void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
253 void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
254 void printReturnValStr(const Function *, raw_ostream &O);
255 void printReturnValStr(const MachineFunction &MF, raw_ostream &O);
256 void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite,
257 raw_ostream &O) const;
258 void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const;
259
260 /// Should a .noreturn directive be emitted for \p V, which is either a
261 /// function or a call site?
262 template <typename T> bool shouldEmitPTXNoReturn(const T &V) const {
263 static_assert(std::is_same_v<Function, T> || std::is_base_of_v<CallBase, T>,
264 "expected a function or a call site");
265
266 const auto &NTM = static_cast<const NVPTXTargetMachine &>(TM);
267 if (!NTM.getSubtargetImpl()->hasNoReturn())
268 return false;
269
270 if (!V.doesNotReturn() || !V.getFunctionType()->getReturnType()->isVoidTy())
271 return false;
272
273 if constexpr (std::is_same_v<Function, T>)
274 return !isKernelFunction(V);
275 else
276 return true;
277 }
278
279 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
280 const char *ExtraCode, raw_ostream &) override;
281 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
282 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
283 const char *ExtraCode, raw_ostream &) override;
284
285 const MCExpr *lowerConstantForGV(const Constant *CV,
286 bool ProcessingGeneric) const;
287 void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const;
288 /// Emit a blob of inline asm to the output streamer.
289 void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
290 const MCTargetOptions &MCOptions, const MDNode *LocMDNode,
291 InlineAsm::AsmDialect Dialect,
292 const MachineInstr *MI) override;
293
294protected:
295 bool doInitialization(Module &M) override;
296 bool doFinalization(Module &M) override;
297
298 /// Create NVPTX-specific DwarfDebug handler.
299 DwarfDebug *createDwarfDebug() override;
300
301private:
302 bool GlobalsEmitted;
303
304 // This is specific per MachineFunction.
305 const MachineRegisterInfo *MRI;
306
307 // The number assigned to each virtual register within its class, populated
308 // by setAndEmitFunctionVirtualRegisters and cleared between functions.
309 using VRegMap = DenseMap<Register, unsigned>;
310 using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>;
311 VRegRCMap VRegMapping;
312
313 // List of variables demoted to a function scope.
314 std::map<const Function *, std::vector<const GlobalVariable *>> localDecls;
315
316 void emitPTXGlobalVariable(const GlobalVariable *GVar, raw_ostream &O,
317 const NVPTXSubtarget &STI);
318 void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar,
319 raw_ostream &O,
320 const NVPTXSubtarget &STI,
321 bool EmitInitializer);
322 void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const;
323 std::string getPTXFundamentalTypeStr(Type *Ty, bool = true) const;
324 void printScalarConstant(const Constant *CPV, raw_ostream &O);
325 void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const;
326 void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer);
327 void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer);
328 void bufferAggregateConstVec(const ConstantVector *CV, AggBuffer *aggBuffer);
329
330 void emitLinkageDirective(const GlobalValue *V, raw_ostream &O);
331 void emitDeclarations(const Module &, raw_ostream &O);
332 void emitDeclaration(const Function *, raw_ostream &O);
333 void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O);
334 void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O);
335 void emitDemotedVars(const Function *, raw_ostream &);
336
337 bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const;
338
339 // Used to control the need to emit .generic() in the initializer of
340 // module scope variables.
341 // Although ptx supports the hybrid mode like the following,
342 // .global .u32 a;
343 // .global .u32 b;
344 // .global .u32 addr[] = {a, generic(b)}
345 // we have difficulty representing the difference in the NVVM IR.
346 //
347 // Since the address value should always be generic in CUDA C and always
348 // be specific in OpenCL, we use this simple control here.
349 //
350 const bool EmitGeneric;
351
352public:
353 NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
354 : AsmPrinter(TM, std::move(Streamer), ID),
355 EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() ==
356 NVPTX::CUDA) {}
357
358 bool runOnMachineFunction(MachineFunction &F) override;
359
360 void getAnalysisUsage(AnalysisUsage &AU) const override {
361 AU.addRequired<MachineLoopInfoWrapperPass>();
362 AsmPrinter::getAnalysisUsage(AU);
363 }
364
365 std::string getVirtualRegisterName(Register Reg) const;
366
367 const MCSymbol *getFunctionFrameSymbol() const override;
368
369 // Make emitGlobalVariable() no-op for NVPTX.
370 // Global variables have been already emitted by the time the base AsmPrinter
371 // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()).
372 void emitGlobalVariable(const GlobalVariable *GV) override {}
373};
374
375} // end anonymous namespace
376
377static StringRef getTextureName(const Value &V) {
378 assert(V.hasName() && "Found texture variable with no name");
379 return V.getName();
380}
381
382static StringRef getSurfaceName(const Value &V) {
383 assert(V.hasName() && "Found surface variable with no name");
384 return V.getName();
385}
386
387static StringRef getSamplerName(const Value &V) {
388 assert(V.hasName() && "Found sampler variable with no name");
389 return V.getName();
390}
391
392/// Emits initial debug location directive.
393static void emitInitialRawDwarfLocDirective(const MachineFunction &MF,
394 DwarfDebug *DD,
395 MCStreamer &OutStreamer) {
396 if (!DD)
397 return;
398
399 assert(OutStreamer.hasRawTextSupport() && "Expected assembly output mode.");
400 // This is NVPTX specific and it's unclear why.
401 // PR51079: If we have code without debug information we need to give up.
402 const DISubprogram *SP = MF.getFunction().getSubprogram();
403 if (!SP)
404 return;
405 assert(SP->getUnit());
406 // NoDebug and DebugDirectivesOnly do not require emitting the initial loc
407 // directive. NoDebug does not require any debug directives and the initial
408 // loc directive is not needed for DebugDirectivesOnly as it is redundant
409 // assuming this is a non-empty function.
410 if (SP->getUnit()->isDebugDirectivesOnly() || SP->getUnit()->isNoDebug())
411 return;
412
413 (void)DD->emitInitialLocDirective(MF, /*CUID=*/0);
414}
415
416namespace {
417
418/// Return a list of GlobalVariables on which \p V depends.
419static void
420discoverDependentGlobals(const Value *V,
421 SmallVectorImpl<const GlobalVariable *> &Globals,
422 SmallPtrSetImpl<const GlobalVariable *> &Seen) {
423 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V)) {
424 if (Seen.insert(Ptr: GV).second)
425 Globals.push_back(Elt: GV);
426 return;
427 }
428
429 // Global values are emitted as symbols. Their operands do not contribute to
430 // the initializer expression that refers to that symbol.
431 if (isa<GlobalValue>(Val: V))
432 return;
433
434 // lowerConstantForGV emits a GEP as its base symbol plus a constant byte
435 // offset. Symbols used to compute an index are not part of that expression.
436 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(Val: V)) {
437 discoverDependentGlobals(V: GEP->getPointerOperand(), Globals, Seen);
438 return;
439 }
440
441 if (const User *U = dyn_cast<User>(Val: V))
442 for (const auto &O : U->operands())
443 discoverDependentGlobals(V: O, Globals, Seen);
444}
445
446struct GlobalVariableDependencyNode {
447 const GlobalVariable *GV = nullptr;
448 unsigned ModuleOrder = 0;
449 SmallVector<const GlobalVariableDependencyNode *, 4> Dependencies;
450};
451
452class GlobalVariableDependencyGraph {
453 // scc_iterator needs a single entry node. Global initializer dependencies
454 // may be disconnected, so use a synthetic root with an edge to every global.
455 GlobalVariableDependencyNode SyntheticRoot;
456 // Edges store pointers into Nodes, so node addresses must remain stable while
457 // the graph is constructed.
458 std::map<const GlobalVariable *, GlobalVariableDependencyNode> Nodes;
459
460public:
461 explicit GlobalVariableDependencyGraph(const Module &M) {
462 unsigned ModuleOrder = 0;
463 for (const GlobalVariable &GV : M.globals()) {
464 GlobalVariableDependencyNode &Node = Nodes.try_emplace(k: &GV).first->second;
465 Node.GV = &GV;
466 Node.ModuleOrder = ModuleOrder++;
467 SyntheticRoot.Dependencies.push_back(Elt: &Node);
468 }
469
470 for (auto &[GV, Node] : Nodes) {
471 SmallVector<const GlobalVariable *, 4> Dependencies;
472 SmallPtrSet<const GlobalVariable *, 4> Seen;
473 for (const Use &Operand : GV->operands())
474 discoverDependentGlobals(V: Operand, Globals&: Dependencies, Seen);
475
476 for (const GlobalVariable *Dependency : Dependencies) {
477 auto It = Nodes.find(x: Dependency);
478 if (It != Nodes.end())
479 Node.Dependencies.push_back(Elt: &It->second);
480 }
481 }
482 }
483
484 const GlobalVariableDependencyNode *getEntryNode() const {
485 return &SyntheticRoot;
486 }
487};
488
489struct GlobalVariableDependencyGraphTraits {
490 using NodeRef = const GlobalVariableDependencyNode *;
491 using ChildIteratorType =
492 SmallVectorImpl<const GlobalVariableDependencyNode *>::const_iterator;
493
494 static NodeRef getEntryNode(NodeRef Node) { return Node; }
495 static ChildIteratorType child_begin(NodeRef Node) {
496 return Node->Dependencies.begin();
497 }
498 static ChildIteratorType child_end(NodeRef Node) {
499 return Node->Dependencies.end();
500 }
501};
502
503using GlobalVariableSCCIterator =
504 scc_iterator<const GlobalVariableDependencyNode *,
505 GlobalVariableDependencyGraphTraits>;
506
507static bool shouldSkipModuleLevelGlobal(const GlobalVariable &GV) {
508 if (GV.hasSection() && GV.getSection() == "llvm.metadata")
509 return true;
510 return GV.getName().starts_with(Prefix: "llvm.") || GV.getName().starts_with(Prefix: "nvvm.");
511}
512
513static bool isForwardDeclarableGlobal(const GlobalVariable *GVar) {
514 if (shouldSkipModuleLevelGlobal(GV: *GVar) || GVar->isDeclaration() ||
515 getPTXOpaqueType(*GVar) != PTXOpaqueType::None)
516 return false;
517
518 // A PTX .extern declaration can be resolved by a later .visible, .weak, or
519 // .common definition, but not by a static definition.
520 if (GVar->hasExternalLinkage())
521 return GVar->hasInitializer();
522
523 if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
524 GVar->hasAvailableExternallyLinkage() || GVar->hasCommonLinkage())
525 return true;
526
527 return false;
528}
529
530/// Order definitions after treating references to forward-declared globals as
531/// already satisfied. A remaining cycle cannot be emitted portably because it
532/// requires an undeclared forward reference.
533static SmallVector<const GlobalVariable *, 4> orderDefinitionsInSCC(
534 ArrayRef<const GlobalVariableDependencyNode *> SCC,
535 const DenseSet<const GlobalVariableDependencyNode *> &ForwardDeclared) {
536 using Node = GlobalVariableDependencyNode;
537
538 DenseSet<const Node *> SCCSet;
539 SCCSet.insert_range(R&: SCC);
540
541 DenseMap<const Node *, unsigned> DependencyCount;
542 DenseMap<const Node *, SmallVector<const Node *, 4>> Dependents;
543 std::set<std::pair<unsigned, const Node *>> Ready;
544
545 // Dependencies outside this SCC have already been emitted. Forward-declared
546 // dependencies are also satisfied, so only count the remaining SCC edges.
547 for (const Node *N : SCC) {
548 unsigned &Count = DependencyCount[N];
549 for (const Node *Dependency : N->Dependencies) {
550 if (!SCCSet.count(V: Dependency) || ForwardDeclared.count(V: Dependency))
551 continue;
552 ++Count;
553 Dependents[Dependency].push_back(Elt: N);
554 }
555 if (Count == 0)
556 Ready.emplace(args: N->ModuleOrder, args&: N);
557 }
558
559 SmallVector<const GlobalVariable *, 4> Order;
560 while (!Ready.empty()) {
561 const Node *N = Ready.begin()->second;
562 Ready.erase(position: Ready.begin());
563 Order.push_back(Elt: N->GV);
564
565 auto It = Dependents.find(Val: N);
566 if (It == Dependents.end())
567 continue;
568 for (const Node *Dependent : It->second) {
569 assert(DependencyCount[Dependent] && "Dependency already satisfied");
570 if (--DependencyCount[Dependent] == 0)
571 Ready.emplace(args: Dependent->ModuleOrder, args&: Dependent);
572 }
573 }
574
575 if (Order.size() != SCC.size())
576 report_fatal_error(reason: "Circular dependency found in global variable set");
577 return Order;
578}
579
580} // namespace
581
582void NVPTXAsmPrinter::emitInstruction(const MachineInstr *MI) {
583 NVPTX_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(),
584 Features: getSubtargetInfo().getFeatureBits());
585
586 MCInst Inst;
587 lowerToMCInst(MI, OutMI&: Inst);
588 EmitToStreamer(S&: *OutStreamer, Inst);
589}
590
591void NVPTXAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
592 OutMI.setOpcode(MI->getOpcode());
593 for (const auto MO : MI->operands())
594 OutMI.addOperand(Op: lowerOperand(MO));
595}
596
597MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) {
598 switch (MO.getType()) {
599 default:
600 llvm_unreachable("unknown operand type");
601 case MachineOperand::MO_Register:
602 return MCOperand::createReg(Reg: encodeVirtualRegister(Reg: MO.getReg()));
603 case MachineOperand::MO_Immediate:
604 return MCOperand::createImm(Val: MO.getImm());
605 case MachineOperand::MO_MachineBasicBlock:
606 return MCOperand::createExpr(
607 Val: MCSymbolRefExpr::create(Symbol: MO.getMBB()->getSymbol(), Ctx&: OutContext));
608 case MachineOperand::MO_ExternalSymbol:
609 return GetSymbolRef(Symbol: GetExternalSymbolSymbol(Sym: MO.getSymbolName()));
610 case MachineOperand::MO_MCSymbol:
611 return GetSymbolRef(Symbol: MO.getMCSymbol());
612 case MachineOperand::MO_JumpTableIndex:
613 // The jump table index names the .branchtargets list emitted for a brx.idx
614 // (see emitJumpTable); reference it by that label.
615 return GetSymbolRef(Symbol: GetJTISymbol(JTID: MO.getIndex()));
616 case MachineOperand::MO_GlobalAddress:
617 return GetSymbolRef(Symbol: getSymbol(GV: MO.getGlobal()));
618 case MachineOperand::MO_FPImmediate: {
619 const ConstantFP *Cnt = MO.getFPImm();
620 const APFloat &Val = Cnt->getValueAPF();
621
622 switch (Cnt->getType()->getTypeID()) {
623 default:
624 report_fatal_error(reason: "Unsupported FP type");
625 break;
626 case Type::HalfTyID:
627 return MCOperand::createExpr(
628 Val: NVPTXFloatMCExpr::createConstantFPHalf(Flt: Val, Ctx&: OutContext));
629 case Type::BFloatTyID:
630 return MCOperand::createExpr(
631 Val: NVPTXFloatMCExpr::createConstantBFPHalf(Flt: Val, Ctx&: OutContext));
632 case Type::FloatTyID:
633 return MCOperand::createExpr(
634 Val: NVPTXFloatMCExpr::createConstantFPSingle(Flt: Val, Ctx&: OutContext));
635 case Type::DoubleTyID:
636 return MCOperand::createExpr(
637 Val: NVPTXFloatMCExpr::createConstantFPDouble(Flt: Val, Ctx&: OutContext));
638 }
639 break;
640 }
641 }
642}
643
644static NVPTX::VirtualRegisterKind
645getVirtualRegisterKind(const TargetRegisterClass *RC) {
646 if (RC == &NVPTX::B1RegClass)
647 return NVPTX::VirtualRegisterKind::B1;
648 if (RC == &NVPTX::B16RegClass)
649 return NVPTX::VirtualRegisterKind::B16;
650 if (RC == &NVPTX::B32RegClass)
651 return NVPTX::VirtualRegisterKind::B32;
652 if (RC == &NVPTX::B64RegClass)
653 return NVPTX::VirtualRegisterKind::B64;
654 if (RC == &NVPTX::B128RegClass)
655 return NVPTX::VirtualRegisterKind::B128;
656 llvm_unreachable("Bad register class");
657}
658
659unsigned NVPTXAsmPrinter::getVirtualRegisterNumber(Register Reg) const {
660 const auto It = VRegMapping.find(Val: MRI->getRegClass(Reg));
661 assert(It != VRegMapping.end() && "Bad register class");
662
663 const unsigned Num = It->second.lookup(Val: Reg);
664 assert(Num && "Bad virtual register");
665 return Num;
666}
667
668MCRegister NVPTXAsmPrinter::encodeVirtualRegister(Register Reg) {
669 if (Reg.isVirtual()) {
670 // Pack the register class into the upper bits so that
671 // NVPTXInstPrinter::printRegName can recover the declared name.
672 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
673 const unsigned Num = getVirtualRegisterNumber(Reg);
674 assert(Num <= NVPTX::VirtualRegisterNumMask &&
675 "Too many virtual registers");
676 return (static_cast<unsigned>(Kind) << NVPTX::VirtualRegisterKindShift) |
677 Num;
678 }
679
680 // Some special-use registers are actually physical registers.
681 // Encode this as the register class ID of 0 and the real register ID.
682 assert(Reg.id() <= NVPTX::VirtualRegisterNumMask &&
683 "Physical register would decode as a virtual register");
684 return Reg.asMCReg();
685}
686
687MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) {
688 const MCExpr *Expr;
689 Expr = MCSymbolRefExpr::create(Symbol, Ctx&: OutContext);
690 return MCOperand::createExpr(Val: Expr);
691}
692
693void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
694 const DataLayout &DL = getDataLayout();
695 const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F);
696 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
697
698 Type *Ty = F->getReturnType();
699 // A void or zero-sized return type (e.g. an empty struct) produces no return
700 // parameter.
701 if (Ty->isVoidTy() || Ty->isEmptyTy())
702 return;
703 O << " (";
704
705 auto PrintScalarRetVal = [&](unsigned Size) {
706 O << ".param .b" << promoteScalarArgumentSize(size: Size) << " func_retval0";
707 };
708 if (shouldPassAsArray(Ty)) {
709 const unsigned TotalSize = DL.getTypeAllocSize(Ty);
710 const Align RetAlignment =
711 getPTXParamAlign(F, Ty, AttrIdx: AttributeList::ReturnIndex, DL);
712 O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
713 << TotalSize << "]";
714 } else if (Ty->isFloatingPointTy()) {
715 PrintScalarRetVal(Ty->getPrimitiveSizeInBits());
716 } else if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
717 PrintScalarRetVal(ITy->getBitWidth());
718 } else if (isa<PointerType>(Val: Ty)) {
719 PrintScalarRetVal(TLI->getPointerTy(DL).getSizeInBits());
720 } else
721 llvm_unreachable("Unknown return type");
722 O << ") ";
723}
724
725void NVPTXAsmPrinter::printReturnValStr(const MachineFunction &MF,
726 raw_ostream &O) {
727 const Function &F = MF.getFunction();
728 printReturnValStr(F: &F, O);
729}
730
731void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB,
732 unsigned UniqueCallSite,
733 raw_ostream &O) const {
734 const DataLayout &DL = getDataLayout();
735 const NVPTXSubtarget &STI = MF->getSubtarget<NVPTXSubtarget>();
736 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
737 const auto PtrVT = TLI->getPointerTy(DL);
738 Type *RetTy = CB.getFunctionType()->getReturnType();
739
740 O << "prototype_" << UniqueCallSite << " : .callprototype ";
741
742 if (RetTy->isVoidTy() || RetTy->isEmptyTy()) {
743 O << "()";
744 } else {
745 O << "(";
746 if (shouldPassAsArray(Ty: RetTy)) {
747 const Align RetAlign =
748 getPTXParamAlign(CB: &CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
749 O << ".param .align " << RetAlign.value() << " .b8 _["
750 << DL.getTypeAllocSize(Ty: RetTy) << "]";
751 } else if (RetTy->isFloatingPointTy() || RetTy->isIntegerTy()) {
752 unsigned size = 0;
753 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
754 size = ITy->getBitWidth();
755 } else {
756 assert(RetTy->isFloatingPointTy() &&
757 "Floating point type expected here");
758 size = RetTy->getPrimitiveSizeInBits();
759 }
760 // PTX ABI requires all scalar return values to be at least 32
761 // bits in size. fp16 normally uses .b16 as its storage type in
762 // PTX, so its size must be adjusted here, too.
763 size = promoteScalarArgumentSize(size);
764
765 O << ".param .b" << size << " _";
766 } else if (isa<PointerType>(Val: RetTy)) {
767 O << ".param .b" << PtrVT.getSizeInBits() << " _";
768 } else {
769 llvm_unreachable("Unknown return type");
770 }
771 O << ") ";
772 }
773 O << "_ (";
774
775 auto MakeArg = [&](const unsigned I) {
776 Type *Ty = CB.getArgOperand(i: I)->getType();
777
778 if (CB.isByValArgument(ArgNo: I)) {
779 Type *ETy = CB.getParamByValType(ArgNo: I);
780 Align ParamByValAlign = getDeviceByValParamAlign(
781 CB: &CB, ArgTy: ETy, AttrIdx: I + AttributeList::FirstArgIndex, DL);
782
783 O << ".param .align " << ParamByValAlign.value() << " .b8 _["
784 << DL.getTypeAllocSize(Ty: ETy) << "]";
785 return;
786 }
787
788 if (shouldPassAsArray(Ty)) {
789 Align ParamAlign =
790 getPTXParamAlign(CB: &CB, Ty, AttrIdx: I + AttributeList::FirstArgIndex, DL);
791 O << ".param .align " << ParamAlign.value() << " .b8 _["
792 << DL.getTypeAllocSize(Ty) << "]";
793 return;
794 }
795 // scalar type
796 unsigned sz = 0;
797 if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
798 sz = promoteScalarArgumentSize(size: ITy->getBitWidth());
799 } else if (isa<PointerType>(Val: Ty)) {
800 sz = PtrVT.getSizeInBits();
801 } else {
802 sz = Ty->getPrimitiveSizeInBits();
803 }
804 O << ".param .b" << sz << " _";
805 };
806
807 const FunctionType *FTy = CB.getFunctionType();
808 const unsigned NumArgs = FTy->getNumParams();
809
810 // Zero-sized arguments (e.g. empty structs) are not passed and so do not
811 // appear in the prototype.
812 const auto NonEmptyArgs = make_filter_range(Range: seq(Size: NumArgs), Pred: [&](unsigned I) {
813 return !CB.getArgOperand(i: I)->getType()->isEmptyTy();
814 });
815
816 interleave(c: NonEmptyArgs, os&: O, each_fn: MakeArg, separator: ", ");
817
818 if (FTy->isVarArg() && CB.arg_size() > NumArgs)
819 O << (NonEmptyArgs.empty() ? "" : ",") << " .param .align "
820 << STI.getMaxRequiredAlignment() << " .b8 _[]";
821
822 O << ")";
823 if (shouldEmitPTXNoReturn(V: CB))
824 O << " .noreturn";
825 O << ";\n";
826}
827
828void NVPTXAsmPrinter::emitJumpTable(const MachineJumpTableEntry &MJT,
829 unsigned MJTI) const {
830 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: MJTI));
831
832 if (MJT.MBBs.empty())
833 return;
834
835 const auto Targets = to_vector(
836 Range: map_range(C: MJT.MBBs, F: [](const MachineBasicBlock *MBB) -> const MCSymbol * {
837 return MBB->getSymbol();
838 }));
839 getTargetStreamer()->emitBranchTargetsDirective(Targets);
840}
841
842// Return true if MBB is the header of a loop marked with
843// llvm.loop.unroll.disable or llvm.loop.unroll.count=1.
844bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
845 const MachineBasicBlock &MBB) const {
846 const MachineLoopInfo *LI = GetMLI(*MF);
847 assert(LI && "NVPTXAsmPrinter requires MachineLoopInfo");
848 // We insert .pragma "nounroll" only to the loop header.
849 if (!LI->isLoopHeader(BB: &MBB))
850 return false;
851
852 // llvm.loop.unroll.disable is marked on the back edges of a loop. Therefore,
853 // we iterate through each back edge of the loop with header MBB, and check
854 // whether its metadata contains llvm.loop.unroll.disable.
855 for (const MachineBasicBlock *PMBB : MBB.predecessors()) {
856 if (LI->getLoopFor(BB: PMBB) != LI->getLoopFor(BB: &MBB)) {
857 // Edges from other loops to MBB are not back edges.
858 continue;
859 }
860 if (const BasicBlock *PBB = PMBB->getBasicBlock()) {
861 if (MDNode *LoopID =
862 PBB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop)) {
863 if (GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.disable"))
864 return true;
865 if (MDNode *UnrollCountMD =
866 GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.count")) {
867 if (mdconst::extract<ConstantInt>(MD: UnrollCountMD->getOperand(I: 1))
868 ->isOne())
869 return true;
870 }
871 }
872 }
873 }
874 return false;
875}
876
877void NVPTXAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
878 AsmPrinter::emitBasicBlockStart(MBB);
879 if (isLoopHeaderOfNoUnroll(MBB))
880 getTargetStreamer()->emitPragmaDirective(Pragma: "nounroll");
881}
882
883void NVPTXAsmPrinter::emitFunctionEntryLabel() {
884 SmallString<128> Str;
885 raw_svector_ostream O(Str);
886
887 if (!GlobalsEmitted) {
888 emitGlobals(M: *MF->getFunction().getParent());
889 GlobalsEmitted = true;
890 }
891
892 // Set up
893 MRI = &MF->getRegInfo();
894 F = &MF->getFunction();
895 emitLinkageDirective(V: F, O);
896 if (isKernelFunction(F: *F))
897 O << ".entry ";
898 else {
899 O << ".func ";
900 printReturnValStr(MF: *MF, O);
901 }
902
903 CurrentFnSym->print(OS&: O, MAI);
904
905 emitFunctionParamList(F, O);
906 O << "\n";
907
908 if (isKernelFunction(F: *F))
909 emitKernelFunctionDirectives(F: *F, O);
910
911 if (shouldEmitPTXNoReturn(V: *F))
912 O << ".noreturn";
913
914 OutStreamer->emitRawText(String: O.str());
915
916 VRegMapping.clear();
917 // Emit open brace for function body.
918 OutStreamer->emitRawText(String: StringRef("{\n"));
919 setAndEmitFunctionVirtualRegisters(*MF);
920 encodeDebugInfoRegisterNumbers(MF: *MF);
921 // Emit initial .loc debug directive for correct relocation symbol data.
922 emitInitialRawDwarfLocDirective(MF: *MF, DD: getDwarfDebug(), OutStreamer&: *OutStreamer);
923}
924
925bool NVPTXAsmPrinter::runOnMachineFunction(MachineFunction &F) {
926 bool Result = AsmPrinter::runOnMachineFunction(MF&: F);
927 // Emit closing brace for the body of function F.
928 // The closing brace must be emitted here because we need to emit additional
929 // debug labels/data after the last basic block.
930 // We need to emit the closing brace here because we don't have function that
931 // finished emission of the function body.
932 OutStreamer->emitRawText(String: StringRef("}\n"));
933 return Result;
934}
935
936void NVPTXAsmPrinter::emitFunctionBodyStart() {
937 SmallString<128> Str;
938 raw_svector_ostream O(Str);
939 emitDemotedVars(&MF->getFunction(), O);
940
941 const auto *MFI = MF->getInfo<NVPTXMachineFunctionInfo>();
942 for (const auto &[Id, CB] : MFI->getCallPrototypes())
943 emitCallPrototype(CB: *CB, UniqueCallSite: Id, O);
944
945 OutStreamer->emitRawText(String: O.str());
946
947 if (const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo())
948 for (const auto &[Idx, JT] : enumerate(First: MJTI->getJumpTables()))
949 emitJumpTable(MJT: JT, MJTI: Idx);
950}
951
952void NVPTXAsmPrinter::emitFunctionBodyEnd() {
953 VRegMapping.clear();
954}
955
956const MCSymbol *NVPTXAsmPrinter::getFunctionFrameSymbol() const {
957 return OutContext.getOrCreateSymbol(DEPOTNAME + Twine(getFunctionNumber()));
958}
959
960void NVPTXAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
961 Register RegNo = MI->getOperand(i: 0).getReg();
962 if (RegNo.isVirtual())
963 OutStreamer->AddComment(T: Twine("implicit-def: ") +
964 getVirtualRegisterName(Reg: RegNo));
965 else
966 OutStreamer->AddComment(T: Twine("implicit-def: ") +
967 NVPTXInstPrinter::getRegisterName(Reg: RegNo));
968 OutStreamer->addBlankLine();
969}
970
971void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
972 raw_ostream &O) const {
973 // If the NVVM IR has some of reqntid* specified, then output
974 // the reqntid directive, and set the unspecified ones to 1.
975 // If none of Reqntid* is specified, don't output reqntid directive.
976 const auto ReqNTID = getReqNTID(F);
977 if (!ReqNTID.empty())
978 O << formatv(Fmt: ".reqntid {0:$[, ]}\n",
979 Vals: make_range(x: ReqNTID.begin(), y: ReqNTID.end()));
980
981 const auto MaxNTID = getMaxNTID(F);
982 if (!MaxNTID.empty())
983 O << formatv(Fmt: ".maxntid {0:$[, ]}\n",
984 Vals: make_range(x: MaxNTID.begin(), y: MaxNTID.end()));
985
986 if (const auto Mincta = getMinCTASm(F))
987 O << ".minnctapersm " << *Mincta << "\n";
988
989 if (const auto Maxnreg = getMaxNReg(F))
990 O << ".maxnreg " << *Maxnreg << "\n";
991
992 // .maxclusterrank directive requires SM_90 or higher, make sure that we
993 // filter it out for lower SM versions, as it causes a hard ptxas crash.
994 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
995 const NVPTXSubtarget *STI = &NTM.getSubtarget<NVPTXSubtarget>(F);
996
997 if (STI->hasFeature(Feature: NVPTX::SM90)) {
998 const auto ClusterDim = getClusterDim(F);
999 const bool BlocksAreClusters = hasBlocksAreClusters(F);
1000
1001 if (!ClusterDim.empty()) {
1002
1003 if (!BlocksAreClusters)
1004 O << ".explicitcluster\n";
1005
1006 if (ClusterDim[0] != 0) {
1007 assert(llvm::all_of(ClusterDim, not_equal_to(0)) &&
1008 "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z "
1009 "should be non-zero as well");
1010
1011 O << formatv(Fmt: ".reqnctapercluster {0:$[, ]}\n",
1012 Vals: make_range(x: ClusterDim.begin(), y: ClusterDim.end()));
1013 } else {
1014 assert(llvm::all_of(ClusterDim, equal_to(0)) &&
1015 "cluster_dim_x == 0 implies cluster_dim_y and cluster_dim_z "
1016 "should be 0 as well");
1017 }
1018 }
1019
1020 if (BlocksAreClusters) {
1021 LLVMContext &Ctx = F.getContext();
1022 if (ReqNTID.empty() || ClusterDim.empty())
1023 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1024 F, "blocksareclusters requires reqntid and cluster_dim attributes",
1025 F.getSubprogram()));
1026 else if (!STI->hasFeature(Feature: NVPTX::PTX90))
1027 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1028 F, "blocksareclusters requires PTX version >= 9.0",
1029 F.getSubprogram()));
1030 else
1031 O << ".blocksareclusters\n";
1032 }
1033
1034 if (const auto Maxclusterrank = getMaxClusterRank(F))
1035 O << ".maxclusterrank " << *Maxclusterrank << "\n";
1036 }
1037}
1038
1039std::string NVPTXAsmPrinter::getVirtualRegisterName(Register Reg) const {
1040 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
1041
1042 std::string Name;
1043 raw_string_ostream(Name) << NVPTX::getVirtualRegisterPrefix(Kind)
1044 << getVirtualRegisterNumber(Reg);
1045 return Name;
1046}
1047
1048void NVPTXAsmPrinter::emitAliasDeclaration(const GlobalAlias *GA,
1049 raw_ostream &O) {
1050 const Function *F = dyn_cast_or_null<Function>(Val: GA->getAliaseeObject());
1051 if (!F || isKernelFunction(F: *F) || F->isDeclaration())
1052 report_fatal_error(
1053 reason: "NVPTX aliasee must be a non-kernel function definition");
1054
1055 if (GA->hasLinkOnceLinkage() || GA->hasWeakLinkage() ||
1056 GA->hasAvailableExternallyLinkage() || GA->hasCommonLinkage())
1057 report_fatal_error(reason: "NVPTX aliasee must not be '.weak'");
1058
1059 emitDeclarationWithName(F, getSymbol(GV: GA), O);
1060}
1061
1062void NVPTXAsmPrinter::emitDeclaration(const Function *F, raw_ostream &O) {
1063 emitDeclarationWithName(F, getSymbol(GV: F), O);
1064}
1065
1066void NVPTXAsmPrinter::emitDeclarationWithName(const Function *F, MCSymbol *S,
1067 raw_ostream &O) {
1068 emitLinkageDirective(V: F, O);
1069 if (isKernelFunction(F: *F))
1070 O << ".entry ";
1071 else
1072 O << ".func ";
1073 printReturnValStr(F, O);
1074 S->print(OS&: O, MAI);
1075 O << "\n";
1076 emitFunctionParamList(F, O);
1077 O << "\n";
1078 if (shouldEmitPTXNoReturn(V: *F))
1079 O << ".noreturn";
1080 O << ";\n";
1081}
1082
1083static bool usedInGlobalVarDef(const Constant *C) {
1084 if (!C)
1085 return false;
1086
1087 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C))
1088 return GV->getName() != "llvm.used";
1089
1090 for (const User *U : C->users())
1091 if (const Constant *C = dyn_cast<Constant>(Val: U))
1092 if (usedInGlobalVarDef(C))
1093 return true;
1094
1095 return false;
1096}
1097
1098static bool usedInOneFunc(const User *U, Function const *&OneFunc) {
1099 if (const GlobalVariable *OtherGV = dyn_cast<GlobalVariable>(Val: U))
1100 if (OtherGV->getName() == "llvm.used")
1101 return true;
1102
1103 if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1104 if (const Function *CurFunc = I->getFunction()) {
1105 if (OneFunc && (CurFunc != OneFunc))
1106 return false;
1107 OneFunc = CurFunc;
1108 return true;
1109 }
1110 return false;
1111 }
1112
1113 for (const User *UU : U->users())
1114 if (!usedInOneFunc(U: UU, OneFunc))
1115 return false;
1116
1117 return true;
1118}
1119
1120/* Find out if a global variable can be demoted to local scope.
1121 * Currently, this is valid for CUDA shared variables, which have local
1122 * scope and global lifetime. So the conditions to check are :
1123 * 1. Is the global variable in shared address space?
1124 * 2. Does it have local linkage?
1125 * 3. Is the global variable referenced only in one function?
1126 */
1127static bool canDemoteGlobalVar(const GlobalVariable *GV, Function const *&f) {
1128 if (!GV->hasLocalLinkage())
1129 return false;
1130 if (GV->getAddressSpace() != ADDRESS_SPACE_SHARED)
1131 return false;
1132
1133 const Function *oneFunc = nullptr;
1134
1135 bool flag = usedInOneFunc(U: GV, OneFunc&: oneFunc);
1136 if (!flag)
1137 return false;
1138 if (!oneFunc)
1139 return false;
1140 f = oneFunc;
1141 return true;
1142}
1143
1144static bool useFuncSeen(const Constant *C,
1145 const SmallPtrSetImpl<const Function *> &SeenSet) {
1146 for (const User *U : C->users()) {
1147 if (const Constant *cu = dyn_cast<Constant>(Val: U)) {
1148 if (useFuncSeen(C: cu, SeenSet))
1149 return true;
1150 } else if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1151 if (const Function *Caller = I->getFunction())
1152 if (SeenSet.contains(Ptr: Caller))
1153 return true;
1154 }
1155 }
1156 return false;
1157}
1158
1159void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) {
1160 SmallPtrSet<const Function *, 32> SeenSet;
1161 for (const Function &F : M) {
1162 if (F.getAttributes().hasFnAttr(Kind: "nvptx-libcall-callee")) {
1163 emitDeclaration(F: &F, O);
1164 continue;
1165 }
1166
1167 if (F.isDeclaration()) {
1168 if (F.use_empty())
1169 continue;
1170 if (F.getIntrinsicID())
1171 continue;
1172 // An unrecognized intrinsic would produce an invalid PTX declaration. Let
1173 // the user know that, and skip it.
1174 if (F.isIntrinsic()) {
1175 LLVMContext &Ctx = F.getContext();
1176 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1177 F, "unknown intrinsic '" + F.getName() +
1178 "' cannot be lowered by the NVPTX backend"));
1179 continue;
1180 }
1181 emitDeclaration(F: &F, O);
1182 continue;
1183 }
1184 for (const User *U : F.users()) {
1185 if (const Constant *C = dyn_cast<Constant>(Val: U)) {
1186 if (usedInGlobalVarDef(C)) {
1187 // The use is in the initialization of a global variable
1188 // that is a function pointer, so print a declaration
1189 // for the original function
1190 emitDeclaration(F: &F, O);
1191 break;
1192 }
1193 // Emit a declaration of this function if the function that
1194 // uses this constant expr has already been seen.
1195 if (useFuncSeen(C, SeenSet)) {
1196 emitDeclaration(F: &F, O);
1197 break;
1198 }
1199 }
1200
1201 if (!isa<Instruction>(Val: U))
1202 continue;
1203 const Function *Caller = cast<Instruction>(Val: U)->getFunction();
1204 if (!Caller)
1205 continue;
1206
1207 // If a caller has already been seen, then the caller is
1208 // appearing in the module before the callee. so print out
1209 // a declaration for the callee.
1210 if (SeenSet.contains(Ptr: Caller)) {
1211 emitDeclaration(F: &F, O);
1212 break;
1213 }
1214 }
1215 SeenSet.insert(Ptr: &F);
1216 }
1217 for (const GlobalAlias &GA : M.aliases())
1218 emitAliasDeclaration(GA: &GA, O);
1219}
1220
1221void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) {
1222 // Construct a default subtarget off of the TargetMachine defaults. The
1223 // rest of NVPTX isn't friendly to change subtargets per function and
1224 // so the default TargetMachine will have all of the options.
1225 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1226 const NVPTXSubtarget *STI = NTM.getSubtargetImpl();
1227
1228 // Emit header before any dwarf directives are emitted below.
1229 emitHeader(M, STI: *STI);
1230}
1231
1232/// Create NVPTX-specific DwarfDebug handler.
1233DwarfDebug *NVPTXAsmPrinter::createDwarfDebug() {
1234 return new NVPTXDwarfDebug(this);
1235}
1236
1237bool NVPTXAsmPrinter::doInitialization(Module &M) {
1238 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1239 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1240 if (M.alias_size() &&
1241 (!STI.hasFeature(Feature: NVPTX::PTX63) || !STI.hasFeature(Feature: NVPTX::SM30)))
1242 report_fatal_error(reason: ".alias requires PTX version >= 6.3 and sm_30");
1243
1244 // We need to call the parent's one explicitly.
1245 bool Result = AsmPrinter::doInitialization(M);
1246
1247 GlobalsEmitted = false;
1248
1249 return Result;
1250}
1251
1252void NVPTXAsmPrinter::emitGlobals(const Module &M) {
1253 SmallString<128> Str2;
1254 raw_svector_ostream OS2(Str2);
1255
1256 emitDeclarations(M, O&: OS2);
1257
1258 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1259 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1260
1261 // ptxas requires global symbols referenced by initializers to be known
1262 // before use. Acyclic dependencies can be handled by dependency-first
1263 // emission. Cyclic SCCs need compatible .extern declarations first.
1264 // Edges point from each global to the globals used by its initializer.
1265 // Reverse-topological SCC iteration therefore emits dependencies first.
1266 GlobalVariableDependencyGraph DependencyGraph(M);
1267 for (GlobalVariableSCCIterator I =
1268 GlobalVariableSCCIterator::begin(G: DependencyGraph.getEntryNode());
1269 !I.isAtEnd(); ++I) {
1270 SmallVector<const GlobalVariableDependencyNode *, 4> SCC(I->begin(),
1271 I->end());
1272
1273 // Nothing points to the synthetic root, so it is always in its own SCC.
1274 if (!SCC.front()->GV) {
1275 assert(SCC.size() == 1 && "Synthetic root must be in its own SCC");
1276 continue;
1277 }
1278
1279 llvm::sort(C&: SCC, Comp: [](const auto *LHS, const auto *RHS) {
1280 return LHS->ModuleOrder < RHS->ModuleOrder;
1281 });
1282
1283 const bool IsCyclic = I.hasCycle();
1284 DenseSet<const GlobalVariableDependencyNode *> ForwardDeclared;
1285 if (IsCyclic)
1286 for (const auto *Node : SCC)
1287 if (isForwardDeclarableGlobal(GVar: Node->GV))
1288 ForwardDeclared.insert(V: Node);
1289
1290 // Check that declarations break every cycle before writing any output.
1291 SmallVector<const GlobalVariable *, 4> OrderedGlobals =
1292 IsCyclic ? orderDefinitionsInSCC(SCC, ForwardDeclared)
1293 : SmallVector<const GlobalVariable *, 4>{SCC.front()->GV};
1294
1295 for (const auto *Node : SCC) {
1296 if (!ForwardDeclared.count(V: Node))
1297 continue;
1298 OS2 << ".extern ";
1299 emitPTXGlobalVariableDefinition(GVar: Node->GV, O&: OS2, STI,
1300 /*EmitInitializer=*/false);
1301 OS2 << ";\n";
1302 }
1303
1304 for (const GlobalVariable *GV : OrderedGlobals)
1305 printModuleLevelGV(GVar: GV, O&: OS2, /*ProcessDemoted=*/processDemoted: false, STI);
1306 }
1307
1308 OS2 << '\n';
1309
1310 OutStreamer->emitRawText(String: OS2.str());
1311}
1312
1313void NVPTXAsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) {
1314 getTargetStreamer()->emitAliasDirective(Name: getSymbol(GV: &GA),
1315 Aliasee: getSymbol(GV: GA.getAliaseeObject()));
1316}
1317
1318NVPTXTargetStreamer *NVPTXAsmPrinter::getTargetStreamer() const {
1319 return static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1320}
1321
1322static bool hasFullDebugInfo(Module &M) {
1323 for (DICompileUnit *CU : M.debug_compile_units()) {
1324 switch(CU->getEmissionKind()) {
1325 case DICompileUnit::NoDebug:
1326 case DICompileUnit::DebugDirectivesOnly:
1327 break;
1328 case DICompileUnit::LineTablesOnly:
1329 case DICompileUnit::FullDebug:
1330 return true;
1331 }
1332 }
1333
1334 return false;
1335}
1336
1337void NVPTXAsmPrinter::emitHeader(Module &M, const NVPTXSubtarget &STI) {
1338 auto *TS = getTargetStreamer();
1339
1340 TS->emitBanner();
1341
1342 const unsigned PTXVersion = STI.getPTXVersion();
1343 TS->emitVersionDirective(PTXVersion);
1344
1345 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1346 bool TexModeIndependent = NTM.getDrvInterface() == NVPTX::NVCL;
1347
1348 TS->emitTargetDirective(Target: STI.getTargetName(), TexModeIndependent,
1349 HasDebug: hasFullDebugInfo(M));
1350 TS->emitAddressSizeDirective(AddrSize: M.getDataLayout().getPointerSizeInBits());
1351}
1352
1353bool NVPTXAsmPrinter::doFinalization(Module &M) {
1354 // If we did not emit any functions, then the global declarations have not
1355 // yet been emitted.
1356 if (!GlobalsEmitted) {
1357 emitGlobals(M);
1358 GlobalsEmitted = true;
1359 }
1360
1361 // call doFinalization
1362 bool ret = AsmPrinter::doFinalization(M);
1363
1364 clearAnnotationCache(&M);
1365
1366 auto *TS =
1367 static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1368 // Close the last emitted section
1369 if (hasDebugInfo()) {
1370 TS->closeLastSection();
1371 // Emit empty .debug_macinfo section for better support of the empty files.
1372 TS->emitEmptySectionDirective(Name: ".debug_macinfo");
1373 }
1374
1375 // Output last DWARF .file directives, if any.
1376 TS->outputDwarfFileDirectives();
1377
1378 return ret;
1379}
1380
1381// This function emits appropriate linkage directives for
1382// functions and global variables.
1383//
1384// extern function declaration -> .extern
1385// extern function definition -> .visible
1386// external global variable with init -> .visible
1387// external without init -> .extern
1388// appending -> not allowed, assert.
1389// for any linkage other than
1390// internal, private, linker_private,
1391// linker_private_weak, linker_private_weak_def_auto,
1392// we emit -> .weak.
1393
1394void NVPTXAsmPrinter::emitLinkageDirective(const GlobalValue *V,
1395 raw_ostream &O) {
1396 if (static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() == NVPTX::CUDA) {
1397 if (V->hasExternalLinkage()) {
1398 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: V))
1399 O << (GVar->hasInitializer() ? ".visible " : ".extern ");
1400 else if (V->isDeclaration())
1401 O << ".extern ";
1402 else
1403 O << ".visible ";
1404 } else if (V->hasAppendingLinkage()) {
1405 report_fatal_error(reason: "Symbol '" + (V->hasName() ? V->getName() : "") +
1406 "' has unsupported appending linkage type");
1407 } else if (!V->hasInternalLinkage() && !V->hasPrivateLinkage()) {
1408 O << ".weak ";
1409 }
1410 }
1411}
1412
1413void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
1414 raw_ostream &O, bool ProcessDemoted,
1415 const NVPTXSubtarget &STI) {
1416 // Skip metadata and LLVM intrinsic global variables.
1417 if (shouldSkipModuleLevelGlobal(GV: *GVar))
1418 return;
1419
1420 if (GVar->hasExternalLinkage()) {
1421 if (GVar->hasInitializer())
1422 O << ".visible ";
1423 else
1424 O << ".extern ";
1425 } else if (STI.hasFeature(Feature: NVPTX::PTX50) && GVar->hasCommonLinkage() &&
1426 GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) {
1427 O << ".common ";
1428 } else if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
1429 GVar->hasAvailableExternallyLinkage() ||
1430 GVar->hasCommonLinkage()) {
1431 O << ".weak ";
1432 }
1433
1434 const PTXOpaqueType OpaqueType = getPTXOpaqueType(*GVar);
1435
1436 if (OpaqueType == PTXOpaqueType::Texture) {
1437 O << ".global .texref " << getTextureName(V: *GVar) << ";\n";
1438 return;
1439 }
1440
1441 if (OpaqueType == PTXOpaqueType::Surface) {
1442 O << ".global .surfref " << getSurfaceName(V: *GVar) << ";\n";
1443 return;
1444 }
1445
1446 if (GVar->isDeclaration()) {
1447 // (extern) declarations, no definition or initializer
1448 // Currently the only known declaration is for an automatic __local
1449 // (.shared) promoted to global.
1450 emitPTXGlobalVariable(GVar, O, STI);
1451 O << ";\n";
1452 return;
1453 }
1454
1455 if (OpaqueType == PTXOpaqueType::Sampler) {
1456 O << ".global .samplerref " << getSamplerName(V: *GVar);
1457
1458 const Constant *Initializer = nullptr;
1459 if (GVar->hasInitializer())
1460 Initializer = GVar->getInitializer();
1461 const ConstantInt *CI = nullptr;
1462 if (Initializer)
1463 CI = dyn_cast<ConstantInt>(Val: Initializer);
1464 if (CI) {
1465 unsigned sample = CI->getZExtValue();
1466
1467 O << " = { ";
1468
1469 for (int i = 0,
1470 addr = ((sample & __CLK_ADDRESS_MASK) >> __CLK_ADDRESS_BASE);
1471 i < 3; i++) {
1472 O << "addr_mode_" << i << " = ";
1473 switch (addr) {
1474 case 0:
1475 O << "wrap";
1476 break;
1477 case 1:
1478 O << "clamp_to_border";
1479 break;
1480 case 2:
1481 O << "clamp_to_edge";
1482 break;
1483 case 3:
1484 O << "wrap";
1485 break;
1486 case 4:
1487 O << "mirror";
1488 break;
1489 }
1490 O << ", ";
1491 }
1492 O << "filter_mode = ";
1493 switch ((sample & __CLK_FILTER_MASK) >> __CLK_FILTER_BASE) {
1494 case 0:
1495 O << "nearest";
1496 break;
1497 case 1:
1498 O << "linear";
1499 break;
1500 case 2:
1501 llvm_unreachable("Anisotropic filtering is not supported");
1502 default:
1503 O << "nearest";
1504 break;
1505 }
1506 if (!((sample & __CLK_NORMALIZED_MASK) >> __CLK_NORMALIZED_BASE)) {
1507 O << ", force_unnormalized_coords = 1";
1508 }
1509 O << " }";
1510 }
1511
1512 O << ";\n";
1513 return;
1514 }
1515
1516 if (GVar->hasPrivateLinkage()) {
1517 if (GVar->getName().starts_with(Prefix: "unrollpragma"))
1518 return;
1519
1520 // FIXME - need better way (e.g. Metadata) to avoid generating this global
1521 if (GVar->getName().starts_with(Prefix: "filename"))
1522 return;
1523 if (GVar->use_empty())
1524 return;
1525 }
1526
1527 const Function *DemotedFunc = nullptr;
1528 if (!ProcessDemoted && canDemoteGlobalVar(GV: GVar, f&: DemotedFunc)) {
1529 O << "// " << GVar->getName() << " has been demoted\n";
1530 localDecls[DemotedFunc].push_back(x: GVar);
1531 return;
1532 }
1533
1534 emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/true);
1535 O << ";\n";
1536}
1537
1538void NVPTXAsmPrinter::emitPTXGlobalVariableDefinition(
1539 const GlobalVariable *GVar, raw_ostream &O, const NVPTXSubtarget &STI,
1540 bool EmitInitializer) {
1541 const DataLayout &DL = getDataLayout();
1542
1543 Type *ETy = GVar->getValueType();
1544
1545 O << ".";
1546 emitPTXAddressSpace(AddressSpace: GVar->getAddressSpace(), O);
1547
1548 if (isManaged(*GVar)) {
1549 if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30))
1550 report_fatal_error(
1551 reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1552 O << " .attribute(.managed)";
1553 }
1554
1555 O << " .align "
1556 << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value();
1557
1558 if (ETy->isPointerTy() || ((ETy->isIntegerTy() || ETy->isFloatingPointTy()) &&
1559 ETy->getScalarSizeInBits() <= 64)) {
1560 O << " .";
1561 // Special case: ABI requires that we use .u8 for predicates
1562 if (ETy->isIntegerTy(BitWidth: 1))
1563 O << "u8";
1564 else
1565 O << getPTXFundamentalTypeStr(Ty: ETy, false);
1566 O << " ";
1567 getSymbol(GV: GVar)->print(OS&: O, MAI);
1568
1569 // Ptx allows variable initilization only for constant and global state
1570 // spaces.
1571 if (EmitInitializer && GVar->hasInitializer()) {
1572 if ((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) ||
1573 (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) {
1574 const Constant *Initializer = GVar->getInitializer();
1575 // 'undef' is treated as there is no value specified.
1576 if (!Initializer->isNullValue() && !isa<UndefValue>(Val: Initializer)) {
1577 O << " = ";
1578 printScalarConstant(CPV: Initializer, O);
1579 }
1580 } else {
1581 // The frontend adds zero-initializer to device and constant variables
1582 // that don't have an initial value, and UndefValue to shared
1583 // variables, so skip warning for this case.
1584 if (!GVar->getInitializer()->isNullValue() &&
1585 !isa<UndefValue>(Val: GVar->getInitializer())) {
1586 report_fatal_error(reason: "initial value of '" + GVar->getName() +
1587 "' is not allowed in addrspace(" +
1588 Twine(GVar->getAddressSpace()) + ")");
1589 }
1590 }
1591 }
1592 } else {
1593 // Although PTX has direct support for struct type and array type and
1594 // LLVM IR is very similar to PTX, the LLVM CodeGen does not support for
1595 // targets that support these high level field accesses. Structs, arrays
1596 // and vectors are lowered into arrays of bytes.
1597 switch (ETy->getTypeID()) {
1598 case Type::IntegerTyID: // Integers larger than 64 bits
1599 case Type::FP128TyID:
1600 case Type::StructTyID:
1601 case Type::ArrayTyID:
1602 case Type::FixedVectorTyID: {
1603 const uint64_t ElementSize = DL.getTypeStoreSize(Ty: ETy);
1604 // Ptx allows variable initilization only for constant and
1605 // global state spaces.
1606 if (((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) ||
1607 (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) &&
1608 GVar->hasInitializer()) {
1609 const Constant *Initializer = GVar->getInitializer();
1610 if (!isa<UndefValue>(Val: Initializer) && !Initializer->isNullValue()) {
1611 AggBuffer aggBuffer(ElementSize, *this);
1612 bufferAggregateConstant(CV: Initializer, aggBuffer: &aggBuffer);
1613 if (aggBuffer.numSymbols()) {
1614 const unsigned int ptrSize = MAI.getCodePointerSize();
1615 if (ElementSize % ptrSize ||
1616 !aggBuffer.allSymbolsAligned(ptrSize)) {
1617 // Print in bytes and use the mask() operator for pointers.
1618 if (!STI.hasMaskOperator())
1619 report_fatal_error(
1620 reason: "initialized packed aggregate with pointers '" +
1621 GVar->getName() +
1622 "' requires at least PTX ISA version 7.1");
1623 O << " .u8 ";
1624 getSymbol(GV: GVar)->print(OS&: O, MAI);
1625 O << "[" << ElementSize << "]";
1626 if (EmitInitializer) {
1627 O << " = {";
1628 aggBuffer.printBytes(os&: O);
1629 O << "}";
1630 }
1631 } else {
1632 O << " .u" << ptrSize * 8 << " ";
1633 getSymbol(GV: GVar)->print(OS&: O, MAI);
1634 O << "[" << ElementSize / ptrSize << "]";
1635 if (EmitInitializer) {
1636 O << " = {";
1637 aggBuffer.printWords(os&: O);
1638 O << "}";
1639 }
1640 }
1641 } else {
1642 O << " .b8 ";
1643 getSymbol(GV: GVar)->print(OS&: O, MAI);
1644 O << "[" << ElementSize << "]";
1645 if (EmitInitializer) {
1646 O << " = {";
1647 aggBuffer.printBytes(os&: O);
1648 O << "}";
1649 }
1650 }
1651 } else {
1652 O << " .b8 ";
1653 getSymbol(GV: GVar)->print(OS&: O, MAI);
1654 if (ElementSize)
1655 O << "[" << ElementSize << "]";
1656 }
1657 } else {
1658 O << " .b8 ";
1659 getSymbol(GV: GVar)->print(OS&: O, MAI);
1660 if (ElementSize)
1661 O << "[" << ElementSize << "]";
1662 }
1663 break;
1664 }
1665 default:
1666 llvm_unreachable("type not supported yet");
1667 }
1668 }
1669}
1670
1671void NVPTXAsmPrinter::AggBuffer::printSymbol(unsigned nSym, raw_ostream &os) {
1672 const Value *v = Symbols[nSym];
1673 const Value *v0 = SymbolsBeforeStripping[nSym];
1674 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: v)) {
1675 MCSymbol *Name = AP.getSymbol(GV: GVar);
1676 PointerType *PTy = dyn_cast<PointerType>(Val: v0->getType());
1677 // Is v0 a generic pointer?
1678 bool isGenericPointer = PTy && PTy->getAddressSpace() == 0;
1679 if (EmitGeneric && isGenericPointer && !isa<Function>(Val: v)) {
1680 os << "generic(";
1681 Name->print(OS&: os, MAI: AP.MAI);
1682 os << ")";
1683 } else {
1684 Name->print(OS&: os, MAI: AP.MAI);
1685 }
1686 } else if (const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(Val: v0)) {
1687 const MCExpr *Expr = AP.lowerConstantForGV(CV: CExpr, ProcessingGeneric: false);
1688 AP.printMCExpr(Expr: *Expr, OS&: os);
1689 } else
1690 llvm_unreachable("symbol type unknown");
1691}
1692
1693void NVPTXAsmPrinter::AggBuffer::printBytes(raw_ostream &os) {
1694 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1695 // Do not emit trailing zero initializers. They will be zero-initialized by
1696 // ptxas. This saves on both space requirements for the generated PTX and on
1697 // memory use by ptxas. (See:
1698 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#global-state-space)
1699 unsigned int InitializerCount = Size;
1700 // TODO: symbols make this harder, but it would still be good to trim trailing
1701 // 0s for aggs with symbols as well.
1702 if (numSymbols() == 0)
1703 while (InitializerCount >= 1 && !buffer[InitializerCount - 1])
1704 InitializerCount--;
1705
1706 symbolPosInBuffer.push_back(Elt: InitializerCount);
1707 unsigned int nSym = 0;
1708 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1709 for (unsigned int pos = 0; pos < InitializerCount;) {
1710 if (pos)
1711 os << ", ";
1712 if (pos != nextSymbolPos) {
1713 os << (unsigned int)buffer[pos];
1714 ++pos;
1715 continue;
1716 }
1717 // Generate a per-byte mask() operator for the symbol, which looks like:
1718 // .global .u8 addr[] = {0xFF(foo), 0xFF00(foo), 0xFF0000(foo), ...};
1719 // See https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#initializers
1720 std::string symText;
1721 llvm::raw_string_ostream oss(symText);
1722 printSymbol(nSym, os&: oss);
1723 for (unsigned i = 0; i < ptrSize; ++i) {
1724 if (i)
1725 os << ", ";
1726 llvm::write_hex(S&: os, N: 0xFFULL << i * 8, Style: HexPrintStyle::PrefixUpper);
1727 os << "(" << symText << ")";
1728 }
1729 pos += ptrSize;
1730 nextSymbolPos = symbolPosInBuffer[++nSym];
1731 assert(nextSymbolPos >= pos);
1732 }
1733}
1734
1735void NVPTXAsmPrinter::AggBuffer::printWords(raw_ostream &os) {
1736 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1737 symbolPosInBuffer.push_back(Elt: Size);
1738 unsigned int nSym = 0;
1739 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1740 assert(nextSymbolPos % ptrSize == 0);
1741 for (unsigned int pos = 0; pos < Size; pos += ptrSize) {
1742 if (pos)
1743 os << ", ";
1744 if (pos == nextSymbolPos) {
1745 printSymbol(nSym, os);
1746 nextSymbolPos = symbolPosInBuffer[++nSym];
1747 assert(nextSymbolPos % ptrSize == 0);
1748 assert(nextSymbolPos >= pos + ptrSize);
1749 } else if (ptrSize == 4)
1750 os << support::endian::read32le(P: &buffer[pos]);
1751 else
1752 os << support::endian::read64le(P: &buffer[pos]);
1753 }
1754}
1755
1756void NVPTXAsmPrinter::emitDemotedVars(const Function *F, raw_ostream &O) {
1757 auto It = localDecls.find(x: F);
1758 if (It == localDecls.end())
1759 return;
1760
1761 ArrayRef<const GlobalVariable *> GVars = It->second;
1762
1763 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1764 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1765
1766 for (const GlobalVariable *GV : GVars) {
1767 O << "\t// demoted variable\n\t";
1768 printModuleLevelGV(GVar: GV, O, /*processDemoted=*/ProcessDemoted: true, STI);
1769 }
1770}
1771
1772void NVPTXAsmPrinter::emitPTXAddressSpace(unsigned int AddressSpace,
1773 raw_ostream &O) const {
1774 switch (AddressSpace) {
1775 case ADDRESS_SPACE_LOCAL:
1776 O << "local";
1777 break;
1778 case ADDRESS_SPACE_GLOBAL:
1779 O << "global";
1780 break;
1781 case ADDRESS_SPACE_CONST:
1782 O << "const";
1783 break;
1784 case ADDRESS_SPACE_SHARED:
1785 O << "shared";
1786 break;
1787 default:
1788 report_fatal_error(reason: "Bad address space found while emitting PTX: " +
1789 llvm::Twine(AddressSpace));
1790 break;
1791 }
1792}
1793
1794std::string
1795NVPTXAsmPrinter::getPTXFundamentalTypeStr(Type *Ty, bool useB4PTR) const {
1796 switch (Ty->getTypeID()) {
1797 case Type::IntegerTyID: {
1798 unsigned NumBits = cast<IntegerType>(Val: Ty)->getBitWidth();
1799 if (NumBits == 1)
1800 return "pred";
1801 if (NumBits <= 64) {
1802 std::string name = "u";
1803 return name + utostr(X: NumBits);
1804 }
1805 llvm_unreachable("Integer too large");
1806 break;
1807 }
1808 case Type::BFloatTyID:
1809 case Type::HalfTyID:
1810 // fp16 and bf16 are stored as .b16 for compatibility with pre-sm_53
1811 // PTX assembly.
1812 return "b16";
1813 case Type::FloatTyID:
1814 return "f32";
1815 case Type::DoubleTyID:
1816 return "f64";
1817 case Type::PointerTyID: {
1818 unsigned PtrSize = TM.getPointerSizeInBits(AS: Ty->getPointerAddressSpace());
1819 assert((PtrSize == 64 || PtrSize == 32) && "Unexpected pointer size");
1820
1821 if (PtrSize == 64)
1822 if (useB4PTR)
1823 return "b64";
1824 else
1825 return "u64";
1826 else if (useB4PTR)
1827 return "b32";
1828 else
1829 return "u32";
1830 }
1831 default:
1832 break;
1833 }
1834 llvm_unreachable("unexpected type");
1835}
1836
1837void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
1838 raw_ostream &O,
1839 const NVPTXSubtarget &STI) {
1840 const DataLayout &DL = getDataLayout();
1841
1842 // GlobalVariables are always constant pointers themselves.
1843 Type *ETy = GVar->getValueType();
1844
1845 O << ".";
1846 emitPTXAddressSpace(AddressSpace: GVar->getType()->getAddressSpace(), O);
1847 if (isManaged(*GVar)) {
1848 if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30))
1849 report_fatal_error(
1850 reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1851
1852 O << " .attribute(.managed)";
1853 }
1854 O << " .align "
1855 << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value();
1856
1857 // Special case for i128/fp128
1858 if (ETy->getScalarSizeInBits() == 128) {
1859 O << " .b8 ";
1860 getSymbol(GV: GVar)->print(OS&: O, MAI);
1861 O << "[16]";
1862 return;
1863 }
1864
1865 if (ETy->isFloatingPointTy() || ETy->isIntOrPtrTy()) {
1866 O << " ." << getPTXFundamentalTypeStr(Ty: ETy) << " ";
1867 getSymbol(GV: GVar)->print(OS&: O, MAI);
1868 return;
1869 }
1870
1871 int64_t ElementSize = 0;
1872
1873 // Although PTX has direct support for struct type and array type and LLVM IR
1874 // is very similar to PTX, the LLVM CodeGen does not support for targets that
1875 // support these high level field accesses. Structs and arrays are lowered
1876 // into arrays of bytes.
1877 switch (ETy->getTypeID()) {
1878 case Type::StructTyID:
1879 case Type::ArrayTyID:
1880 case Type::FixedVectorTyID:
1881 ElementSize = DL.getTypeStoreSize(Ty: ETy);
1882 O << " .b8 ";
1883 getSymbol(GV: GVar)->print(OS&: O, MAI);
1884 O << "[";
1885 if (ElementSize) {
1886 O << ElementSize;
1887 }
1888 O << "]";
1889 break;
1890 default:
1891 llvm_unreachable("type not supported yet");
1892 }
1893}
1894
1895void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
1896 const DataLayout &DL = getDataLayout();
1897 const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F);
1898 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
1899 const NVPTXMachineFunctionInfo *MFI =
1900 MF ? MF->getInfo<NVPTXMachineFunctionInfo>() : nullptr;
1901
1902 bool IsFirst = true;
1903 const bool IsKernelFunc = isKernelFunction(F: *F);
1904
1905 // Zero-sized arguments (e.g. empty structs) do not produce a parameter.
1906 // Number the emitted parameters contiguously, skipping the zero-sized ones,
1907 // so that the names match those used in LowerFormalArguments and the
1908 // contiguous numbering used by callers (see LowerCall).
1909 const auto NonEmptyArgs =
1910 make_filter_range(Range: F->args(), Pred: [](const Argument &Arg) {
1911 return !Arg.getType()->isEmptyTy();
1912 });
1913
1914 if (NonEmptyArgs.empty() && !F->isVarArg()) {
1915 O << "()";
1916 return;
1917 }
1918
1919 O << "(\n";
1920
1921 for (const auto &[ParamIndex, Arg] : enumerate(First: NonEmptyArgs)) {
1922 Type *Ty = Arg.getType();
1923 MCSymbol *const ParamSym = TLI->getParamSymbol(Ctx&: OutContext, F, Idx: ParamIndex);
1924
1925 if (!IsFirst)
1926 O << ",\n";
1927
1928 IsFirst = false;
1929
1930 // Handle image/sampler parameters
1931 if (IsKernelFunc) {
1932 const PTXOpaqueType ArgOpaqueType = getPTXOpaqueType(Arg);
1933 if (ArgOpaqueType != PTXOpaqueType::None) {
1934 const bool EmitImgPtr = !MFI || !MFI->checkImageHandleSymbol(Symbol: ParamSym);
1935 O << "\t.param ";
1936 if (EmitImgPtr)
1937 O << ".u64 .ptr ";
1938
1939 switch (ArgOpaqueType) {
1940 case PTXOpaqueType::Sampler:
1941 O << ".samplerref ";
1942 break;
1943 case PTXOpaqueType::Texture:
1944 O << ".texref ";
1945 break;
1946 case PTXOpaqueType::Surface:
1947 O << ".surfref ";
1948 break;
1949 case PTXOpaqueType::None:
1950 llvm_unreachable("handled above");
1951 }
1952 O << *ParamSym;
1953 continue;
1954 }
1955 }
1956
1957 if (Arg.hasByValAttr()) {
1958 // param has byVal attribute.
1959 Type *ETy = Arg.getParamByValType();
1960 assert(ETy && "Param should have byval type");
1961
1962 // Print .param .align <a> .b8 .param[size];
1963 // <a> = optimal alignment for the element type; always multiple of
1964 // PAL.getParamAlignment
1965 // size = typeallocsize of element type
1966 const unsigned ParamIdx = Arg.getArgNo() + AttributeList::FirstArgIndex;
1967 const Align OptimalAlign =
1968 IsKernelFunc ? getPTXParamAlign(F, Ty: ETy, AttrIdx: ParamIdx, DL)
1969 : getDeviceByValParamAlign(F, ArgTy: ETy, AttrIdx: ParamIdx, DL);
1970
1971 O << "\t.param .align " << OptimalAlign.value() << " .b8 " << *ParamSym
1972 << "[" << DL.getTypeAllocSize(Ty: ETy) << "]";
1973 continue;
1974 }
1975
1976 if (shouldPassAsArray(Ty)) {
1977 // Just print .param .align <a> .b8 .param[size];
1978 // <a> = optimal alignment for the element type; always multiple of
1979 // PAL.getParamAlignment
1980 // size = typeallocsize of element type
1981 Align OptimalAlign = getPTXParamAlign(
1982 F, Ty, AttrIdx: Arg.getArgNo() + AttributeList::FirstArgIndex, DL);
1983
1984 O << "\t.param .align " << OptimalAlign.value() << " .b8 " << *ParamSym
1985 << "[" << DL.getTypeAllocSize(Ty) << "]";
1986
1987 continue;
1988 }
1989 // Just a scalar
1990 auto *PTy = dyn_cast<PointerType>(Val: Ty);
1991 unsigned PTySizeInBits = 0;
1992 if (PTy) {
1993 PTySizeInBits =
1994 TLI->getPointerTy(DL, AS: PTy->getAddressSpace()).getSizeInBits();
1995 assert(PTySizeInBits && "Invalid pointer size");
1996 }
1997
1998 if (IsKernelFunc) {
1999 if (PTy) {
2000 O << "\t.param .u" << PTySizeInBits << " .ptr";
2001
2002 switch (PTy->getAddressSpace()) {
2003 default:
2004 break;
2005 case ADDRESS_SPACE_GLOBAL:
2006 O << " .global";
2007 break;
2008 case ADDRESS_SPACE_SHARED:
2009 O << " .shared";
2010 break;
2011 case ADDRESS_SPACE_CONST:
2012 O << " .const";
2013 break;
2014 case ADDRESS_SPACE_LOCAL:
2015 O << " .local";
2016 break;
2017 }
2018
2019 O << " .align " << Arg.getParamAlign().valueOrOne().value() << " "
2020 << *ParamSym;
2021 continue;
2022 }
2023
2024 // non-pointer scalar to kernel func
2025 O << "\t.param .";
2026 // Special case: predicate operands become .u8 types
2027 if (Ty->isIntegerTy(BitWidth: 1))
2028 O << "u8";
2029 else
2030 O << getPTXFundamentalTypeStr(Ty);
2031 O << " " << *ParamSym;
2032 continue;
2033 }
2034 // Non-kernel function, just print .param .b<size> for ABI
2035 // and .reg .b<size> for non-ABI
2036 unsigned Size;
2037 if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
2038 Size = promoteScalarArgumentSize(size: ITy->getBitWidth());
2039 } else if (PTy) {
2040 assert(PTySizeInBits && "Invalid pointer size");
2041 Size = PTySizeInBits;
2042 } else
2043 Size = Ty->getPrimitiveSizeInBits();
2044 O << "\t.param .b" << Size << " " << *ParamSym;
2045 }
2046
2047 if (F->isVarArg()) {
2048 if (!IsFirst)
2049 O << ",\n";
2050 O << "\t.param .align " << STI.getMaxRequiredAlignment() << " .b8 "
2051 << *TLI->getParamSymbol(Ctx&: OutContext, F, /* vararg */ Idx: -1) << "[]";
2052 }
2053
2054 O << "\n)";
2055}
2056
2057void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
2058 const MachineFunction &MF) {
2059 auto *TS = getTargetStreamer();
2060
2061 // Emit the Fake Stack Object
2062 const MachineFrameInfo &MFI = MF.getFrameInfo();
2063 if (const int64_t NumBytes = MFI.getStackSize()) {
2064 TS->emitLocalDirective(Alignment: MFI.getMaxAlign(), Name: getFunctionFrameSymbol(),
2065 Size: NumBytes);
2066
2067 // Declare the frame pointers that NVPTXFrameLowering's prologue defines.
2068 const NVPTXRegisterInfo *NRI =
2069 MF.getSubtarget<NVPTXSubtarget>().getRegisterInfo();
2070 for (const Register FrameReg :
2071 {NRI->getFrameRegister(MF), NRI->getFrameLocalRegister(MF)})
2072 TS->emitRegDirective(
2073 SizeInBits: NRI->getRegSizeInBits(Reg: FrameReg, MRI: *MRI).getFixedValue(),
2074 Name: NVPTXInstPrinter::getRegisterName(Reg: FrameReg));
2075 }
2076
2077 // Go through all virtual registers to establish the mapping between the
2078 // global virtual
2079 // register number and the per class virtual register number.
2080 // We use the per class virtual register number in the ptx output.
2081 for (unsigned I : llvm::seq(Size: MRI->getNumVirtRegs())) {
2082 Register VR = Register::index2VirtReg(Index: I);
2083 if (MRI->use_empty(RegNo: VR) && MRI->def_empty(RegNo: VR))
2084 continue;
2085 auto &RCRegMap = VRegMapping[MRI->getRegClass(Reg: VR)];
2086 RCRegMap[VR] = RCRegMap.size() + 1;
2087 }
2088
2089 // Emit declaration of the virtual registers or 'physical' registers for
2090 // each register class
2091 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2092 for (const TargetRegisterClass &RC : TRI->regclasses()) {
2093 // Only declare those registers that may be used.
2094 const auto It = VRegMapping.find(Val: &RC);
2095 if (It == VRegMapping.end() || It->second.empty())
2096 continue;
2097
2098 TS->emitRegDirective(
2099 SizeInBits: TRI->getRegSizeInBits(RC).getFixedValue(),
2100 Name: NVPTX::getVirtualRegisterPrefix(Kind: getVirtualRegisterKind(RC: &RC)),
2101 Count: It->second.size() + 1);
2102 }
2103}
2104
2105/// Translate virtual register numbers in DebugInfo locations to their printed
2106/// encodings, as used by CUDA-GDB.
2107void NVPTXAsmPrinter::encodeDebugInfoRegisterNumbers(
2108 const MachineFunction &MF) {
2109 const NVPTXSubtarget &STI = MF.getSubtarget<NVPTXSubtarget>();
2110 const NVPTXRegisterInfo *NRI = STI.getRegisterInfo();
2111
2112 // Clear the old mapping, and add the new one. This mapping is used after the
2113 // printing of the current function is complete, but before the next function
2114 // is printed.
2115 NRI->clearDebugRegisterMap();
2116
2117 for (const VRegMap &RegMap : make_second_range(c&: VRegMapping))
2118 for (const Register Reg : make_first_range(c: RegMap))
2119 NRI->addToDebugRegisterMap(VirtReg: Reg, RegisterName: getVirtualRegisterName(Reg));
2120}
2121
2122void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp,
2123 raw_ostream &O) const {
2124 APFloat APF = APFloat(Fp->getValueAPF()); // make a copy
2125 bool ignored;
2126 unsigned int numHex;
2127 const char *lead;
2128
2129 if (Fp->getType()->getTypeID() == Type::FloatTyID) {
2130 numHex = 8;
2131 lead = "0f";
2132 APF.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &ignored);
2133 } else if (Fp->getType()->getTypeID() == Type::DoubleTyID) {
2134 numHex = 16;
2135 lead = "0d";
2136 APF.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven, losesInfo: &ignored);
2137 } else
2138 llvm_unreachable("unsupported fp type");
2139
2140 APInt API = APF.bitcastToAPInt();
2141 O << lead << format_hex_no_prefix(N: API.getZExtValue(), Width: numHex, /*Upper=*/true);
2142}
2143
2144void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) {
2145 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2146 O << CI->getValue();
2147 return;
2148 }
2149 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
2150 printFPConstant(Fp: CFP, O);
2151 return;
2152 }
2153 if (isa<ConstantPointerNull>(Val: CPV)) {
2154 O << "0";
2155 return;
2156 }
2157 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
2158 const bool IsNonGenericPointer = GVar->getAddressSpace() != 0;
2159 if (EmitGeneric && !isa<Function>(Val: CPV) && !IsNonGenericPointer) {
2160 O << "generic(";
2161 getSymbol(GV: GVar)->print(OS&: O, MAI);
2162 O << ")";
2163 } else {
2164 getSymbol(GV: GVar)->print(OS&: O, MAI);
2165 }
2166 return;
2167 }
2168 if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2169 const MCExpr *E = lowerConstantForGV(CV: cast<Constant>(Val: Cexpr), ProcessingGeneric: false);
2170 printMCExpr(Expr: *E, OS&: O);
2171 return;
2172 }
2173 llvm_unreachable("Not scalar type found in printScalarConstant()");
2174}
2175
2176void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
2177 AggBuffer *AggBuffer) {
2178 const DataLayout &DL = getDataLayout();
2179 int AllocSize = DL.getTypeAllocSize(Ty: CPV->getType());
2180 if (isa<UndefValue>(Val: CPV) || CPV->isNullValue()) {
2181 // Non-zero Bytes indicates that we need to zero-fill everything. Otherwise,
2182 // only the space allocated by CPV.
2183 AggBuffer->addZeros(Num: Bytes ? Bytes : AllocSize);
2184 return;
2185 }
2186
2187 // Helper for filling AggBuffer with APInts.
2188 auto AddIntToBuffer = [AggBuffer, Bytes](const APInt &Val) {
2189 size_t NumBytes = (Val.getBitWidth() + 7) / 8;
2190 SmallVector<unsigned char, 16> Buf(NumBytes);
2191 // `extractBitsAsZExtValue` does not allow the extraction of bits beyond the
2192 // input's bit width, and i1 arrays may not have a length that is a multuple
2193 // of 8. We handle the last byte separately, so we never request out of
2194 // bounds bits.
2195 for (unsigned I = 0; I < NumBytes - 1; ++I) {
2196 Buf[I] = Val.extractBitsAsZExtValue(numBits: 8, bitPosition: I * 8);
2197 }
2198 size_t LastBytePosition = (NumBytes - 1) * 8;
2199 size_t LastByteBits = Val.getBitWidth() - LastBytePosition;
2200 Buf[NumBytes - 1] =
2201 Val.extractBitsAsZExtValue(numBits: LastByteBits, bitPosition: LastBytePosition);
2202 AggBuffer->addBytes(Ptr: Buf.data(), Num: NumBytes, Bytes);
2203 };
2204
2205 switch (CPV->getType()->getTypeID()) {
2206 case Type::IntegerTyID:
2207 if (const auto *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2208 AddIntToBuffer(CI->getValue());
2209 break;
2210 }
2211 if (const auto *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2212 if (const auto *CI =
2213 dyn_cast<ConstantInt>(Val: ConstantFoldConstant(C: Cexpr, DL))) {
2214 AddIntToBuffer(CI->getValue());
2215 break;
2216 }
2217 if (Cexpr->getOpcode() == Instruction::PtrToInt) {
2218 Value *V = Cexpr->getOperand(i_nocapture: 0)->stripPointerCasts();
2219 AggBuffer->addSymbol(GVar: V, GVarBeforeStripping: Cexpr->getOperand(i_nocapture: 0));
2220 AggBuffer->addZeros(Num: AllocSize);
2221 break;
2222 }
2223 // A symbol-relative integer whose offset is applied outside the
2224 // ptrtoint, e.g. add(ptrtoint(@g), C). It can't fold to a ConstantInt
2225 // because it references a symbol; emit it through lowerConstantForGV, the
2226 // same path scalar symbol-relative integer globals use.
2227 AggBuffer->addSymbol(GVar: Cexpr, GVarBeforeStripping: Cexpr);
2228 AggBuffer->addZeros(Num: AllocSize);
2229 break;
2230 }
2231 llvm_unreachable("unsupported integer const type");
2232 break;
2233
2234 case Type::HalfTyID:
2235 case Type::BFloatTyID:
2236 case Type::FloatTyID:
2237 case Type::DoubleTyID:
2238 case Type::FP128TyID:
2239 AddIntToBuffer(cast<ConstantFP>(Val: CPV)->getValueAPF().bitcastToAPInt());
2240 break;
2241
2242 case Type::PointerTyID: {
2243 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
2244 AggBuffer->addSymbol(GVar, GVarBeforeStripping: GVar);
2245 } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2246 const Value *v = Cexpr->stripPointerCasts();
2247 AggBuffer->addSymbol(GVar: v, GVarBeforeStripping: Cexpr);
2248 }
2249 AggBuffer->addZeros(Num: AllocSize);
2250 break;
2251 }
2252
2253 case Type::ArrayTyID:
2254 case Type::FixedVectorTyID:
2255 case Type::StructTyID: {
2256 if (isa<ConstantAggregate>(Val: CPV) || isa<ConstantDataSequential>(Val: CPV)) {
2257 // bufferAggregateConstant doesn't emit tail-padding, i.e. it writes
2258 // `store_size` bytes, not `alloc_size` bytes. Do it ourselves here.
2259 unsigned StartPos = AggBuffer->getCurpos();
2260 bufferAggregateConstant(CV: CPV, aggBuffer: AggBuffer);
2261 unsigned Written = AggBuffer->getCurpos() - StartPos;
2262 unsigned SlotSize = std::max<int>(a: Bytes, b: AllocSize);
2263 if (SlotSize > Written)
2264 AggBuffer->addZeros(Num: SlotSize - Written);
2265 } else if (isa<ConstantAggregateZero>(Val: CPV))
2266 AggBuffer->addZeros(Num: Bytes);
2267 else
2268 llvm_unreachable("Unexpected Constant type");
2269 break;
2270 }
2271
2272 default:
2273 llvm_unreachable("unsupported type");
2274 }
2275}
2276
2277void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
2278 AggBuffer *aggBuffer) {
2279 const DataLayout &DL = getDataLayout();
2280
2281 auto ExtendBuffer = [](APInt Val, AggBuffer *Buffer) {
2282 unsigned NumBytes = divideCeil(Numerator: Val.getBitWidth(), Denominator: 8);
2283 for (unsigned I : llvm::seq(Size: NumBytes)) {
2284 unsigned NumBits = std::min(a: 8u, b: Val.getBitWidth() - I * 8);
2285 Buffer->addByte(Byte: Val.extractBitsAsZExtValue(numBits: NumBits, bitPosition: I * 8));
2286 }
2287 };
2288
2289 // Integer or floating point vector splats.
2290 if (isa<ConstantInt, ConstantFP>(Val: CPV)) {
2291 if (auto *VTy = dyn_cast<FixedVectorType>(Val: CPV->getType())) {
2292 for (unsigned I : llvm::seq(Size: VTy->getNumElements()))
2293 bufferLEByte(CPV: CPV->getAggregateElement(Elt: I), Bytes: 0, AggBuffer: aggBuffer);
2294 return;
2295 }
2296 }
2297
2298 // Integers of arbitrary width
2299 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2300 assert(CI->getType()->isIntegerTy() && "Expected integer constant!");
2301 ExtendBuffer(CI->getValue(), aggBuffer);
2302 return;
2303 }
2304
2305 // f128
2306 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
2307 assert(CFP->getType()->isFloatingPointTy() && "Expected fp constant!");
2308 if (CFP->getType()->isFP128Ty()) {
2309 ExtendBuffer(CFP->getValueAPF().bitcastToAPInt(), aggBuffer);
2310 return;
2311 }
2312 }
2313
2314 // Buffer arrays one element at a time.
2315 if (isa<ConstantArray>(Val: CPV)) {
2316 for (const auto &Op : CPV->operands())
2317 bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer);
2318 return;
2319 }
2320
2321 // Constant vectors
2322 if (const auto *CVec = dyn_cast<ConstantVector>(Val: CPV)) {
2323 bufferAggregateConstVec(CV: CVec, aggBuffer);
2324 return;
2325 }
2326
2327 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: CPV)) {
2328 for (unsigned I : llvm::seq(Size: CDS->getNumElements()))
2329 bufferLEByte(CPV: cast<Constant>(Val: CDS->getElementAsConstant(i: I)), Bytes: 0, AggBuffer: aggBuffer);
2330 return;
2331 }
2332
2333 if (isa<ConstantStruct>(Val: CPV)) {
2334 if (CPV->getNumOperands()) {
2335 StructType *ST = cast<StructType>(Val: CPV->getType());
2336 for (unsigned I : llvm::seq(Size: CPV->getNumOperands())) {
2337 int EndOffset = (I + 1 == CPV->getNumOperands())
2338 ? DL.getStructLayout(Ty: ST)->getElementOffset(Idx: 0) +
2339 DL.getTypeAllocSize(Ty: ST)
2340 : DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I + 1);
2341 int Bytes = EndOffset - DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I);
2342 bufferLEByte(CPV: cast<Constant>(Val: CPV->getOperand(i: I)), Bytes, AggBuffer: aggBuffer);
2343 }
2344 }
2345 return;
2346 }
2347 llvm_unreachable("unsupported constant type in printAggregateConstant()");
2348}
2349
2350void NVPTXAsmPrinter::bufferAggregateConstVec(const ConstantVector *CV,
2351 AggBuffer *aggBuffer) {
2352 unsigned NumElems = CV->getType()->getNumElements();
2353 const unsigned BuffSize = aggBuffer->getBufferSize();
2354
2355 // Buffer one element at a time if we have allocated enough buffer space.
2356 if (BuffSize >= NumElems) {
2357 for (const auto &Op : CV->operands())
2358 bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer);
2359 return;
2360 }
2361
2362 // Sub-byte datatypes will have more elements than bytes allocated for the
2363 // buffer. Merge consecutive elements to form a full byte. We expect that 8 %
2364 // sub-byte-elem-size should be 0 and current expected usage is for i4 (for
2365 // e2m1-fp4 types).
2366 Type *ElemTy = CV->getType()->getElementType();
2367 assert(ElemTy->isIntegerTy() && "Expected integer data type.");
2368 unsigned ElemTySize = ElemTy->getPrimitiveSizeInBits();
2369 assert(ElemTySize < 8 && "Expected sub-byte data type.");
2370 assert(8 % ElemTySize == 0 && "Element type size must evenly divide a byte.");
2371 // Number of elements to merge to form a full byte.
2372 unsigned NumElemsPerByte = 8 / ElemTySize;
2373 unsigned NumCompleteBytes = NumElems / NumElemsPerByte;
2374 unsigned NumTailElems = NumElems % NumElemsPerByte;
2375
2376 // Helper lambda to constant-fold sub-vector of sub-byte type elements into
2377 // i8. Start and end indices of the sub-vector is provided, along with number
2378 // of padding zeros if required.
2379 auto ConvertSubCVtoInt8 = [this, &ElemTy](const ConstantVector *CV,
2380 unsigned Start, unsigned End,
2381 unsigned NumPaddingZeros = 0) {
2382 // Collect elements to create sub-vector.
2383 SmallVector<Constant *, 8> SubCVElems;
2384 for (unsigned I : llvm::seq(Begin: Start, End))
2385 SubCVElems.push_back(Elt: CV->getAggregateElement(Elt: I));
2386
2387 // Optionally pad with zeros.
2388 if (NumPaddingZeros)
2389 SubCVElems.append(NumInputs: NumPaddingZeros, Elt: ConstantInt::getNullValue(Ty: ElemTy));
2390
2391 auto SubCV = ConstantVector::get(V: SubCVElems);
2392 Type *Int8Ty = IntegerType::get(C&: SubCV->getContext(), NumBits: 8);
2393
2394 // Merge elements of the sub-vector using ConstantFolding.
2395 ConstantInt *MergedElem =
2396 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldConstant(
2397 C: ConstantExpr::getBitCast(C: const_cast<Constant *>(SubCV), Ty: Int8Ty),
2398 DL: getDataLayout()));
2399
2400 if (!MergedElem)
2401 report_fatal_error(
2402 reason: "Cannot lower vector global with unusual element type");
2403
2404 return MergedElem;
2405 };
2406
2407 // Iterate through elements of vector one chunk at a time and buffer that
2408 // chunk.
2409 for (unsigned ByteIdx : llvm::seq(Size: NumCompleteBytes))
2410 bufferLEByte(CPV: ConvertSubCVtoInt8(CV, ByteIdx * NumElemsPerByte,
2411 (ByteIdx + 1) * NumElemsPerByte),
2412 Bytes: 0, AggBuffer: aggBuffer);
2413
2414 // For unevenly sized vectors add tail padding zeros.
2415 if (NumTailElems > 0)
2416 bufferLEByte(CPV: ConvertSubCVtoInt8(CV, NumElems - NumTailElems, NumElems,
2417 NumElemsPerByte - NumTailElems),
2418 Bytes: 0, AggBuffer: aggBuffer);
2419}
2420
2421/// lowerConstantForGV - Return an MCExpr for the given Constant. This is mostly
2422/// a copy from AsmPrinter::lowerConstant, except customized to only handle
2423/// expressions that are representable in PTX and create
2424/// NVPTXGenericMCSymbolRefExpr nodes for addrspacecast instructions.
2425const MCExpr *
2426NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV,
2427 bool ProcessingGeneric) const {
2428 MCContext &Ctx = OutContext;
2429
2430 if (CV->isNullValue() || isa<UndefValue>(Val: CV))
2431 return MCConstantExpr::create(Value: 0, Ctx);
2432
2433 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CV))
2434 return MCConstantExpr::create(Value: CI->getZExtValue(), Ctx);
2435
2436 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
2437 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Symbol: getSymbol(GV), Ctx);
2438 if (ProcessingGeneric)
2439 return NVPTXGenericMCSymbolRefExpr::create(SymExpr: Expr, Ctx);
2440 return Expr;
2441 }
2442
2443 const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: CV);
2444 if (!CE) {
2445 llvm_unreachable("Unknown constant value to lower!");
2446 }
2447
2448 switch (CE->getOpcode()) {
2449 default:
2450 break; // Error
2451
2452 case Instruction::AddrSpaceCast: {
2453 // Strip the addrspacecast and pass along the operand
2454 PointerType *DstTy = cast<PointerType>(Val: CE->getType());
2455 if (DstTy->getAddressSpace() == 0)
2456 return lowerConstantForGV(CV: cast<const Constant>(Val: CE->getOperand(i_nocapture: 0)), ProcessingGeneric: true);
2457
2458 break; // Error
2459 }
2460
2461 case Instruction::GetElementPtr: {
2462 const DataLayout &DL = getDataLayout();
2463
2464 // Generate a symbolic expression for the byte address
2465 APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0);
2466 cast<GEPOperator>(Val: CE)->accumulateConstantOffset(DL, Offset&: OffsetAI);
2467
2468 const MCExpr *Base = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0),
2469 ProcessingGeneric);
2470 if (!OffsetAI)
2471 return Base;
2472
2473 int64_t Offset = OffsetAI.getSExtValue();
2474 return MCBinaryExpr::createAdd(LHS: Base, RHS: MCConstantExpr::create(Value: Offset, Ctx),
2475 Ctx);
2476 }
2477
2478 case Instruction::Trunc:
2479 // We emit the value and depend on the assembler to truncate the generated
2480 // expression properly. This is important for differences between
2481 // blockaddress labels. Since the two labels are in the same function, it
2482 // is reasonable to treat their delta as a 32-bit value.
2483 [[fallthrough]];
2484 case Instruction::BitCast:
2485 return lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2486
2487 case Instruction::IntToPtr: {
2488 const DataLayout &DL = getDataLayout();
2489
2490 // Handle casts to pointers by changing them into casts to the appropriate
2491 // integer type. This promotes constant folding and simplifies this code.
2492 Constant *Op = CE->getOperand(i_nocapture: 0);
2493 Op = ConstantFoldIntegerCast(C: Op, DestTy: DL.getIntPtrType(CV->getType()),
2494 /*IsSigned*/ false, DL);
2495 if (Op)
2496 return lowerConstantForGV(CV: Op, ProcessingGeneric);
2497
2498 break; // Error
2499 }
2500
2501 case Instruction::PtrToInt: {
2502 const DataLayout &DL = getDataLayout();
2503
2504 // Support only foldable casts to/from pointers that can be eliminated by
2505 // changing the pointer to the appropriately sized integer type.
2506 Constant *Op = CE->getOperand(i_nocapture: 0);
2507 Type *Ty = CE->getType();
2508
2509 const MCExpr *OpExpr = lowerConstantForGV(CV: Op, ProcessingGeneric);
2510
2511 // We can emit the pointer value into this slot if the slot is an
2512 // integer slot equal to the size of the pointer.
2513 if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Ty: Op->getType()))
2514 return OpExpr;
2515
2516 // Otherwise the pointer is smaller than the resultant integer, mask off
2517 // the high bits so we are sure to get a proper truncation if the input is
2518 // a constant expr.
2519 unsigned InBits = DL.getTypeAllocSizeInBits(Ty: Op->getType());
2520 const MCExpr *MaskExpr = MCConstantExpr::create(Value: ~0ULL >> (64-InBits), Ctx);
2521 return MCBinaryExpr::createAnd(LHS: OpExpr, RHS: MaskExpr, Ctx);
2522 }
2523
2524 // The MC library also has a right-shift operator, but it isn't consistently
2525 // signed or unsigned between different targets.
2526 case Instruction::Add: {
2527 const MCExpr *LHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2528 const MCExpr *RHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 1), ProcessingGeneric);
2529 switch (CE->getOpcode()) {
2530 default: llvm_unreachable("Unknown binary operator constant cast expr");
2531 case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx);
2532 }
2533 }
2534 }
2535
2536 // If the code isn't optimized, there may be outstanding folding
2537 // opportunities. Attempt to fold the expression using DataLayout as a
2538 // last resort before giving up.
2539 Constant *C = ConstantFoldConstant(C: CE, DL: getDataLayout());
2540 if (C != CE)
2541 return lowerConstantForGV(CV: C, ProcessingGeneric);
2542
2543 // Otherwise report the problem to the user.
2544 std::string S;
2545 raw_string_ostream OS(S);
2546 OS << "Unsupported expression in static initializer: ";
2547 CE->printAsOperand(O&: OS, /*PrintType=*/false,
2548 M: !MF ? nullptr : MF->getFunction().getParent());
2549 report_fatal_error(reason: Twine(OS.str()));
2550}
2551
2552void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) const {
2553 OutContext.getAsmInfo().printExpr(OS, Expr);
2554}
2555
2556/// PrintAsmOperand - Print out an operand for an inline asm expression.
2557///
2558bool NVPTXAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
2559 const char *ExtraCode, raw_ostream &O) {
2560 if (ExtraCode && ExtraCode[0]) {
2561 if (ExtraCode[1] != 0)
2562 return true; // Unknown modifier.
2563
2564 switch (ExtraCode[0]) {
2565 default:
2566 // See if this is a generic print operand
2567 return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O);
2568 case 'r':
2569 break;
2570 }
2571 }
2572
2573 printOperand(MI, OpNum: OpNo, O);
2574
2575 return false;
2576}
2577
2578bool NVPTXAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
2579 unsigned OpNo,
2580 const char *ExtraCode,
2581 raw_ostream &O) {
2582 if (ExtraCode && ExtraCode[0])
2583 return true; // Unknown modifier
2584
2585 O << '[';
2586 printMemOperand(MI, OpNum: OpNo, O);
2587 O << ']';
2588
2589 return false;
2590}
2591
2592void NVPTXAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
2593 raw_ostream &O) {
2594 const MachineOperand &MO = MI->getOperand(i: OpNum);
2595 switch (MO.getType()) {
2596 case MachineOperand::MO_Register:
2597 if (MO.getReg().isPhysical()) {
2598 if (MO.getReg() == NVPTX::VRDepot)
2599 getFunctionFrameSymbol()->print(OS&: O, MAI);
2600 else
2601 O << NVPTXInstPrinter::getRegisterName(Reg: MO.getReg());
2602 } else {
2603 O << getVirtualRegisterName(Reg: MO.getReg());
2604 }
2605 break;
2606
2607 case MachineOperand::MO_Immediate:
2608 O << MO.getImm();
2609 break;
2610
2611 case MachineOperand::MO_FPImmediate:
2612 printFPConstant(Fp: MO.getFPImm(), O);
2613 break;
2614
2615 case MachineOperand::MO_GlobalAddress:
2616 PrintSymbolOperand(MO, OS&: O);
2617 break;
2618
2619 case MachineOperand::MO_MCSymbol:
2620 MO.getMCSymbol()->print(OS&: O, MAI);
2621 break;
2622
2623 case MachineOperand::MO_MachineBasicBlock:
2624 MO.getMBB()->getSymbol()->print(OS&: O, MAI);
2625 break;
2626
2627 default:
2628 llvm_unreachable("Operand type not supported.");
2629 }
2630}
2631
2632void NVPTXAsmPrinter::printMemOperand(const MachineInstr *MI, unsigned OpNum,
2633 raw_ostream &O, const char *Modifier) {
2634 printOperand(MI, OpNum, O);
2635
2636 if (Modifier && strcmp(s1: Modifier, s2: "add") == 0) {
2637 O << ", ";
2638 printOperand(MI, OpNum: OpNum + 1, O);
2639 } else {
2640 if (MI->getOperand(i: OpNum + 1).isImm() &&
2641 MI->getOperand(i: OpNum + 1).getImm() == 0)
2642 return; // don't print ',0' or '+0'
2643 O << "+";
2644 printOperand(MI, OpNum: OpNum + 1, O);
2645 }
2646}
2647
2648/// Returns true if \p Line begins with an alphabetic character or underscore,
2649/// indicating it is a PTX instruction that should receive a .loc directive.
2650static bool isPTXInstruction(StringRef Line) {
2651 StringRef Trimmed = Line.ltrim();
2652 return !Trimmed.empty() &&
2653 (std::isalpha(static_cast<unsigned char>(Trimmed[0])) ||
2654 Trimmed[0] == '_');
2655}
2656
2657/// Returns the DILocation for an inline asm MachineInstr if debug line info
2658/// should be emitted, or nullptr otherwise.
2659static const DILocation *getInlineAsmDebugLoc(const MachineInstr *MI) {
2660 if (!MI || !MI->getDebugLoc())
2661 return nullptr;
2662 const DISubprogram *SP = MI->getMF()->getFunction().getSubprogram();
2663 if (!SP || SP->getUnit()->getEmissionKind() == DICompileUnit::NoDebug)
2664 return nullptr;
2665 const DILocation *DL = MI->getDebugLoc();
2666 if (!DL->getFile() || !DL->getLine() || DL->isImplicitCode())
2667 return nullptr;
2668 return DL;
2669}
2670
2671namespace {
2672struct InlineAsmInliningContext {
2673 MCSymbol *FuncNameSym = nullptr;
2674 unsigned FileIA = 0;
2675 unsigned LineIA = 0;
2676 unsigned ColIA = 0;
2677
2678 bool hasInlinedAt() const { return FuncNameSym != nullptr; }
2679};
2680} // namespace
2681
2682/// Resolves the enhanced-lineinfo inlining context for an inline asm debug
2683/// location. Returns a default (empty) context if inlining info is unavailable.
2684static InlineAsmInliningContext
2685getInlineAsmInliningContext(const DILocation *DL, const MachineFunction &MF,
2686 NVPTXDwarfDebug *NVDD, MCStreamer &Streamer,
2687 unsigned CUID) {
2688 InlineAsmInliningContext Ctx;
2689 const DILocation *InlinedAt = DL->getInlinedAt();
2690 if (!InlinedAt || !InlinedAt->getFile() || !NVDD ||
2691 !NVDD->isEnhancedLineinfo(MF))
2692 return Ctx;
2693 const auto *SubProg = getDISubprogram(Scope: DL->getScope());
2694 if (!SubProg)
2695 return Ctx;
2696 Ctx.FuncNameSym = NVDD->getOrCreateFuncNameSymbol(LinkageName: SubProg->getLinkageName());
2697 Ctx.FileIA = Streamer.emitDwarfFileDirective(
2698 FileNo: 0, Directory: InlinedAt->getFile()->getDirectory(),
2699 Filename: InlinedAt->getFile()->getFilename(), Checksum: std::nullopt, Source: std::nullopt, CUID);
2700 Ctx.LineIA = InlinedAt->getLine();
2701 Ctx.ColIA = InlinedAt->getColumn();
2702 return Ctx;
2703}
2704
2705void NVPTXAsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
2706 const MCTargetOptions &MCOptions,
2707 const MDNode *LocMDNode,
2708 InlineAsm::AsmDialect Dialect,
2709 const MachineInstr *MI) {
2710 assert(!Str.empty() && "Can't emit empty inline asm block");
2711 if (Str.back() == 0)
2712 Str = Str.substr(Start: 0, N: Str.size() - 1);
2713
2714 auto emitAsmStr = [&](StringRef AsmStr) {
2715 emitInlineAsmStart();
2716 OutStreamer->emitRawText(String: AsmStr);
2717 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2718 };
2719
2720 const DILocation *DL = getInlineAsmDebugLoc(MI);
2721 if (!DL) {
2722 emitAsmStr(Str);
2723 return;
2724 }
2725
2726 const DIFile *File = DL->getFile();
2727 unsigned Line = DL->getLine();
2728 const unsigned Column = DL->getColumn();
2729 const unsigned CUID = OutStreamer->getContext().getDwarfCompileUnitID();
2730 const unsigned FileNumber = OutStreamer->emitDwarfFileDirective(
2731 FileNo: 0, Directory: File->getDirectory(), Filename: File->getFilename(), Checksum: std::nullopt, Source: std::nullopt,
2732 CUID);
2733
2734 auto *NVDD = static_cast<NVPTXDwarfDebug *>(getDwarfDebug());
2735 InlineAsmInliningContext InlineCtx =
2736 getInlineAsmInliningContext(DL, MF: *MI->getMF(), NVDD, Streamer&: *OutStreamer, CUID);
2737
2738 SmallVector<StringRef, 16> Lines;
2739 Str.split(A&: Lines, Separator: '\n');
2740 emitInlineAsmStart();
2741 for (const StringRef &L : Lines) {
2742 StringRef RTrimmed = L.rtrim(Char: '\r');
2743 if (isPTXInstruction(Line: L)) {
2744 if (InlineCtx.hasInlinedAt()) {
2745 OutStreamer->emitDwarfLocDirectiveWithInlinedAt(
2746 FileNo: FileNumber, Line, Column, FileIA: InlineCtx.FileIA, LineIA: InlineCtx.LineIA,
2747 ColumnIA: InlineCtx.ColIA, Sym: InlineCtx.FuncNameSym, DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2748 FileName: File->getFilename());
2749 } else {
2750 OutStreamer->emitDwarfLocDirective(FileNo: FileNumber, Line, Column,
2751 DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2752 FileName: File->getFilename());
2753 }
2754 }
2755 OutStreamer->emitRawText(String: RTrimmed);
2756 ++Line;
2757 }
2758 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2759}
2760
2761char NVPTXAsmPrinter::ID = 0;
2762
2763INITIALIZE_PASS(NVPTXAsmPrinter, "nvptx-asm-printer", "NVPTX Assembly Printer",
2764 false, false)
2765
2766// Force static initialization.
2767extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
2768LLVMInitializeNVPTXAsmPrinter() {
2769 RegisterAsmPrinter<NVPTXAsmPrinter> X(getTheNVPTXTarget32());
2770 RegisterAsmPrinter<NVPTXAsmPrinter> Y(getTheNVPTXTarget64());
2771}
2772
2773PreservedAnalyses NVPTXAsmPrinterBeginPass::run(Module &M,
2774 ModuleAnalysisManager &MAM) {
2775 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2776 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2777 Printer.doInitialization(M);
2778 return PreservedAnalyses::all();
2779}
2780
2781PreservedAnalyses
2782NVPTXAsmPrinterPass::run(MachineFunction &MF,
2783 MachineFunctionAnalysisManager &MFAM) {
2784 AsmPrinter &Printer =
2785 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2786 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2787 ->getPrinter();
2788 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter&: Printer);
2789 Printer.runOnMachineFunction(MF);
2790 return PreservedAnalyses::all();
2791}
2792
2793PreservedAnalyses NVPTXAsmPrinterEndPass::run(Module &M,
2794 ModuleAnalysisManager &MAM) {
2795 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2796 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2797 Printer.doFinalization(M);
2798 return PreservedAnalyses::all();
2799}
2800