1//===-- NVPTXAsmPrinter.cpp - NVPTX LLVM assembly writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to NVPTX assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXAsmPrinter.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "MCTargetDesc/NVPTXInstPrinter.h"
17#include "MCTargetDesc/NVPTXTargetStreamer.h"
18#include "NVPTX.h"
19#include "NVPTXDwarfDebug.h"
20#include "NVPTXMCExpr.h"
21#include "NVPTXMachineFunctionInfo.h"
22#include "NVPTXRegisterInfo.h"
23#include "NVPTXSubtarget.h"
24#include "NVPTXTargetMachine.h"
25#include "NVPTXUtilities.h"
26#include "NVVMProperties.h"
27#include "TargetInfo/NVPTXTargetInfo.h"
28#include "cl_common_defines.h"
29#include "llvm/ADT/APFloat.h"
30#include "llvm/ADT/APInt.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/DenseSet.h"
34#include "llvm/ADT/SCCIterator.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/Sequence.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/SmallString.h"
39#include "llvm/ADT/SmallVector.h"
40#include "llvm/ADT/StringExtras.h"
41#include "llvm/ADT/StringRef.h"
42#include "llvm/ADT/Twine.h"
43#include "llvm/ADT/iterator_range.h"
44#include "llvm/Analysis/ConstantFolding.h"
45#include "llvm/CodeGen/Analysis.h"
46#include "llvm/CodeGen/AsmPrinter.h"
47#include "llvm/CodeGen/AsmPrinterAnalysis.h"
48#include "llvm/CodeGen/MachineBasicBlock.h"
49#include "llvm/CodeGen/MachineFrameInfo.h"
50#include "llvm/CodeGen/MachineFunction.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineJumpTableInfo.h"
53#include "llvm/CodeGen/MachineLoopInfo.h"
54#include "llvm/CodeGen/MachineModuleInfo.h"
55#include "llvm/CodeGen/MachineOperand.h"
56#include "llvm/CodeGen/MachineRegisterInfo.h"
57#include "llvm/CodeGen/TargetRegisterInfo.h"
58#include "llvm/CodeGen/ValueTypes.h"
59#include "llvm/CodeGenTypes/MachineValueType.h"
60#include "llvm/IR/Argument.h"
61#include "llvm/IR/Attributes.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DebugLoc.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/Function.h"
71#include "llvm/IR/GlobalAlias.h"
72#include "llvm/IR/GlobalValue.h"
73#include "llvm/IR/GlobalVariable.h"
74#include "llvm/IR/InstrTypes.h"
75#include "llvm/IR/Instruction.h"
76#include "llvm/IR/LLVMContext.h"
77#include "llvm/IR/Module.h"
78#include "llvm/IR/Operator.h"
79#include "llvm/IR/Type.h"
80#include "llvm/IR/User.h"
81#include "llvm/IR/Value.h"
82#include "llvm/MC/MCExpr.h"
83#include "llvm/MC/MCInst.h"
84#include "llvm/MC/MCInstrDesc.h"
85#include "llvm/MC/MCStreamer.h"
86#include "llvm/MC/MCSymbol.h"
87#include "llvm/MC/TargetRegistry.h"
88#include "llvm/Pass.h"
89#include "llvm/Support/Alignment.h"
90#include "llvm/Support/Casting.h"
91#include "llvm/Support/Compiler.h"
92#include "llvm/Support/Endian.h"
93#include "llvm/Support/ErrorHandling.h"
94#include "llvm/Support/NativeFormatting.h"
95#include "llvm/Support/raw_ostream.h"
96#include "llvm/Target/TargetLoweringObjectFile.h"
97#include "llvm/Target/TargetMachine.h"
98#include "llvm/Transforms/Utils/UnrollLoop.h"
99#include <algorithm>
100#include <cassert>
101#include <cstdint>
102#include <cstring>
103#include <map>
104#include <memory>
105#include <set>
106#include <string>
107#include <type_traits>
108#include <vector>
109
110using namespace llvm;
111
112#define DEPOTNAME "__local_depot"
113
114// The ptx syntax and format is very different from that usually seem in a .s
115// file,
116// therefore we are not able to use the MCAsmStreamer interface here.
117//
118// We are handcrafting the output method here.
119//
120// A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer
121// (subclass of MCStreamer).
122
123namespace {
124
125class NVPTXAsmPrinter : public AsmPrinter {
126
127 class AggBuffer {
128 // Used to buffer the emitted string for initializing global aggregates.
129 //
130 // Normally an aggregate (array, vector, or structure) is emitted as a u8[].
131 // However, if either element/field of the aggregate is a non-NULL address,
132 // and all such addresses are properly aligned, then the aggregate is
133 // emitted as u32[] or u64[]. In the case of unaligned addresses, the
134 // aggregate is emitted as u8[], and the mask() operator is used for all
135 // pointers.
136 //
137 // We first layout the aggregate in 'buffer' in bytes, except for those
138 // symbol addresses. For the i-th symbol address in the aggregate, its
139 // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s.
140 // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1]
141 // records the Value*.
142 //
143 // Once we have this AggBuffer setup, we can choose how to print it out.
144 public:
145 // number of symbol addresses
146 unsigned numSymbols() const { return Symbols.size(); }
147
148 bool allSymbolsAligned(unsigned ptrSize) const {
149 return llvm::all_of(Range: symbolPosInBuffer,
150 P: [=](unsigned pos) { return pos % ptrSize == 0; });
151 }
152
153 private:
154 const unsigned Size; // size of the buffer in bytes
155 std::vector<unsigned char> buffer; // the buffer
156 SmallVector<unsigned, 4> symbolPosInBuffer;
157 SmallVector<const Value *, 4> Symbols;
158 // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
159 // stripping pointer casts, i.e.,
160 // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
161 //
162 // We need to keep these values because AggBuffer::print decides whether to
163 // emit a "generic()" cast for Symbols[i] depending on the address space of
164 // SymbolsBeforeStripping[i].
165 SmallVector<const Value *, 4> SymbolsBeforeStripping;
166 unsigned curpos;
167 const NVPTXAsmPrinter &AP;
168 const bool EmitGeneric;
169
170 public:
171 AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP)
172 : Size(Size), buffer(Size), curpos(0), AP(AP),
173 EmitGeneric(AP.EmitGeneric) {}
174
175 unsigned getBufferSize() const { return Size; }
176
177 // Number of bytes written so far.
178 unsigned getCurpos() const { return curpos; }
179
180 // Copy Num bytes from Ptr.
181 // if Bytes > Num, zero fill up to Bytes.
182 void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
183 for (unsigned I : llvm::seq(Size: Num))
184 addByte(Byte: Ptr[I]);
185 if (Bytes > Num)
186 addZeros(Num: Bytes - Num);
187 }
188
189 void addByte(uint8_t Byte) {
190 assert(curpos < Size);
191 buffer[curpos] = Byte;
192 curpos++;
193 }
194
195 void addZeros(unsigned Num) {
196 for ([[maybe_unused]] unsigned _ : llvm::seq(Size: Num)) {
197 addByte(Byte: 0);
198 }
199 }
200
201 void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
202 symbolPosInBuffer.push_back(Elt: curpos);
203 Symbols.push_back(Elt: GVar);
204 SymbolsBeforeStripping.push_back(Elt: GVarBeforeStripping);
205 }
206
207 void printBytes(raw_ostream &os);
208 void printWords(raw_ostream &os);
209
210 private:
211 void printSymbol(unsigned nSym, raw_ostream &os);
212 };
213
214 friend class AggBuffer;
215
216public:
217 static char ID;
218
219 StringRef getPassName() const override { return "NVPTX Assembly Printer"; }
220
221private:
222 const Function *F;
223
224 NVPTXTargetStreamer *getTargetStreamer() const;
225
226 void emitStartOfAsmFile(Module &M) override;
227 void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
228 void emitFunctionEntryLabel() override;
229 void emitFunctionBodyStart() override;
230 void emitFunctionBodyEnd() override;
231 void emitImplicitDef(const MachineInstr *MI) const override;
232
233 void emitInstruction(const MachineInstr *) override;
234 void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
235 MCOperand lowerOperand(const MachineOperand &MO);
236 MCOperand GetSymbolRef(const MCSymbol *Symbol);
237 MCRegister encodeVirtualRegister(Register Reg);
238
239 /// The number \p Reg was assigned within its register class, as declared by
240 /// this function's .reg directives.
241 unsigned getVirtualRegisterNumber(Register Reg) const;
242
243 void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O,
244 const char *Modifier = nullptr);
245 void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O,
246 bool processDemoted, const NVPTXSubtarget &STI);
247 void emitGlobals(const Module &M);
248 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
249 void emitHeader(Module &M, const NVPTXSubtarget &STI);
250 void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const;
251 void emitFunctionParamList(const Function *, raw_ostream &O);
252 void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
253 void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
254 void printReturnValStr(const Function *, raw_ostream &O);
255 void printReturnValStr(const MachineFunction &MF, raw_ostream &O);
256 void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite,
257 raw_ostream &O) const;
258 void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const;
259
260 /// Should a .noreturn directive be emitted for \p V, which is either a
261 /// function or a call site?
262 template <typename T> bool shouldEmitPTXNoReturn(const T &V) const {
263 static_assert(std::is_same_v<Function, T> || std::is_base_of_v<CallBase, T>,
264 "expected a function or a call site");
265
266 const auto &NTM = static_cast<const NVPTXTargetMachine &>(TM);
267 if (!NTM.getSubtargetImpl()->hasNoReturn())
268 return false;
269
270 if (!V.doesNotReturn() || !V.getFunctionType()->getReturnType()->isVoidTy())
271 return false;
272
273 if constexpr (std::is_same_v<Function, T>)
274 return !isKernelFunction(V);
275 else
276 return true;
277 }
278
279 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
280 const char *ExtraCode, raw_ostream &) override;
281 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
282 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
283 const char *ExtraCode, raw_ostream &) override;
284
285 const MCExpr *lowerConstantForGV(const Constant *CV,
286 bool ProcessingGeneric) const;
287 void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const;
288 /// Emit a blob of inline asm to the output streamer.
289 void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
290 const MCTargetOptions &MCOptions, const MDNode *LocMDNode,
291 InlineAsm::AsmDialect Dialect,
292 const MachineInstr *MI) override;
293
294protected:
295 bool doInitialization(Module &M) override;
296 bool doFinalization(Module &M) override;
297
298 /// Create NVPTX-specific DwarfDebug handler.
299 DwarfDebug *createDwarfDebug() override;
300
301private:
302 bool GlobalsEmitted;
303
304 // This is specific per MachineFunction.
305 const MachineRegisterInfo *MRI;
306
307 // The number assigned to each virtual register within its class, populated
308 // by setAndEmitFunctionVirtualRegisters and cleared between functions.
309 using VRegMap = DenseMap<Register, unsigned>;
310 using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>;
311 VRegRCMap VRegMapping;
312
313 // List of variables demoted to a function scope.
314 std::map<const Function *, std::vector<const GlobalVariable *>> localDecls;
315
316 void emitPTXGlobalVariable(const GlobalVariable *GVar, raw_ostream &O,
317 const NVPTXSubtarget &STI);
318 void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar,
319 raw_ostream &O,
320 const NVPTXSubtarget &STI,
321 bool EmitInitializer);
322 void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const;
323 std::string getPTXFundamentalTypeStr(Type *Ty, bool = true) const;
324 void printScalarConstant(const Constant *CPV, raw_ostream &O);
325 void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const;
326 void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer);
327 void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer);
328 void bufferAggregateConstVec(const ConstantVector *CV, AggBuffer *aggBuffer);
329
330 void emitLinkageDirective(const GlobalValue *V, raw_ostream &O);
331 void emitDeclarations(const Module &, raw_ostream &O);
332 void emitDeclaration(const Function *, raw_ostream &O);
333 void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O);
334 void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O);
335 void emitDemotedVars(const Function *, raw_ostream &);
336
337 bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const;
338
339 // Used to control the need to emit .generic() in the initializer of
340 // module scope variables.
341 // Although ptx supports the hybrid mode like the following,
342 // .global .u32 a;
343 // .global .u32 b;
344 // .global .u32 addr[] = {a, generic(b)}
345 // we have difficulty representing the difference in the NVVM IR.
346 //
347 // Since the address value should always be generic in CUDA C and always
348 // be specific in OpenCL, we use this simple control here.
349 //
350 const bool EmitGeneric;
351
352public:
353 NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
354 : AsmPrinter(TM, std::move(Streamer), ID),
355 EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() ==
356 NVPTX::CUDA) {}
357
358 bool runOnMachineFunction(MachineFunction &F) override;
359
360 void getAnalysisUsage(AnalysisUsage &AU) const override {
361 AU.addRequired<MachineLoopInfoWrapperPass>();
362 AsmPrinter::getAnalysisUsage(AU);
363 }
364
365 std::string getVirtualRegisterName(Register Reg) const;
366
367 const MCSymbol *getFunctionFrameSymbol() const override;
368
369 // Make emitGlobalVariable() no-op for NVPTX.
370 // Global variables have been already emitted by the time the base AsmPrinter
371 // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()).
372 void emitGlobalVariable(const GlobalVariable *GV) override {}
373};
374
375} // end anonymous namespace
376
377static StringRef getTextureName(const Value &V) {
378 assert(V.hasName() && "Found texture variable with no name");
379 return V.getName();
380}
381
382static StringRef getSurfaceName(const Value &V) {
383 assert(V.hasName() && "Found surface variable with no name");
384 return V.getName();
385}
386
387static StringRef getSamplerName(const Value &V) {
388 assert(V.hasName() && "Found sampler variable with no name");
389 return V.getName();
390}
391
392/// Emits initial debug location directive.
393static void emitInitialRawDwarfLocDirective(const MachineFunction &MF,
394 DwarfDebug *DD,
395 MCStreamer &OutStreamer) {
396 if (!DD)
397 return;
398
399 assert(OutStreamer.hasRawTextSupport() && "Expected assembly output mode.");
400 // This is NVPTX specific and it's unclear why.
401 // PR51079: If we have code without debug information we need to give up.
402 const DISubprogram *SP = MF.getFunction().getSubprogram();
403 if (!SP)
404 return;
405 assert(SP->getUnit());
406 // NoDebug and DebugDirectivesOnly do not require emitting the initial loc
407 // directive. NoDebug does not require any debug directives and the initial
408 // loc directive is not needed for DebugDirectivesOnly as it is redundant
409 // assuming this is a non-empty function.
410 if (SP->getUnit()->isDebugDirectivesOnly() || SP->getUnit()->isNoDebug())
411 return;
412
413 (void)DD->emitInitialLocDirective(MF, /*CUID=*/0);
414}
415
416namespace {
417
418/// Return a list of GlobalVariables on which \p V depends.
419static void
420discoverDependentGlobals(const Value *V,
421 SmallVectorImpl<const GlobalVariable *> &Globals,
422 SmallPtrSetImpl<const GlobalVariable *> &Seen) {
423 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V)) {
424 if (Seen.insert(Ptr: GV).second)
425 Globals.push_back(Elt: GV);
426 return;
427 }
428
429 // Global values are emitted as symbols. Their operands do not contribute to
430 // the initializer expression that refers to that symbol.
431 if (isa<GlobalValue>(Val: V))
432 return;
433
434 // lowerConstantForGV emits a GEP as its base symbol plus a constant byte
435 // offset. Symbols used to compute an index are not part of that expression.
436 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(Val: V)) {
437 discoverDependentGlobals(V: GEP->getPointerOperand(), Globals, Seen);
438 return;
439 }
440
441 if (const User *U = dyn_cast<User>(Val: V))
442 for (const auto &O : U->operands())
443 discoverDependentGlobals(V: O, Globals, Seen);
444}
445
446struct GlobalVariableDependencyNode {
447 const GlobalVariable *GV = nullptr;
448 unsigned ModuleOrder = 0;
449 SmallVector<const GlobalVariableDependencyNode *, 4> Dependencies;
450};
451
452class GlobalVariableDependencyGraph {
453 // scc_iterator needs a single entry node. Global initializer dependencies
454 // may be disconnected, so use a synthetic root with an edge to every global.
455 GlobalVariableDependencyNode SyntheticRoot;
456 // Edges store pointers into Nodes, so node addresses must remain stable while
457 // the graph is constructed.
458 std::map<const GlobalVariable *, GlobalVariableDependencyNode> Nodes;
459
460public:
461 explicit GlobalVariableDependencyGraph(const Module &M) {
462 unsigned ModuleOrder = 0;
463 for (const GlobalVariable &GV : M.globals()) {
464 GlobalVariableDependencyNode &Node = Nodes.try_emplace(k: &GV).first->second;
465 Node.GV = &GV;
466 Node.ModuleOrder = ModuleOrder++;
467 SyntheticRoot.Dependencies.push_back(Elt: &Node);
468 }
469
470 for (auto &[GV, Node] : Nodes) {
471 SmallVector<const GlobalVariable *, 4> Dependencies;
472 SmallPtrSet<const GlobalVariable *, 4> Seen;
473 for (const Use &Operand : GV->operands())
474 discoverDependentGlobals(V: Operand, Globals&: Dependencies, Seen);
475
476 for (const GlobalVariable *Dependency : Dependencies) {
477 auto It = Nodes.find(x: Dependency);
478 if (It != Nodes.end())
479 Node.Dependencies.push_back(Elt: &It->second);
480 }
481 }
482 }
483
484 const GlobalVariableDependencyNode *getEntryNode() const {
485 return &SyntheticRoot;
486 }
487};
488
489struct GlobalVariableDependencyGraphTraits {
490 using NodeRef = const GlobalVariableDependencyNode *;
491 using ChildIteratorType =
492 SmallVectorImpl<const GlobalVariableDependencyNode *>::const_iterator;
493
494 static NodeRef getEntryNode(NodeRef Node) { return Node; }
495 static ChildIteratorType child_begin(NodeRef Node) {
496 return Node->Dependencies.begin();
497 }
498 static ChildIteratorType child_end(NodeRef Node) {
499 return Node->Dependencies.end();
500 }
501};
502
503using GlobalVariableSCCIterator =
504 scc_iterator<const GlobalVariableDependencyNode *,
505 GlobalVariableDependencyGraphTraits>;
506
507static bool shouldSkipModuleLevelGlobal(const GlobalVariable &GV) {
508 if (GV.hasSection() && GV.getSection() == "llvm.metadata")
509 return true;
510 return GV.getName().starts_with(Prefix: "llvm.") || GV.getName().starts_with(Prefix: "nvvm.");
511}
512
513static bool isForwardDeclarableGlobal(const GlobalVariable *GVar) {
514 if (shouldSkipModuleLevelGlobal(GV: *GVar) || GVar->isDeclaration() ||
515 getPTXOpaqueType(*GVar) != PTXOpaqueType::None)
516 return false;
517
518 // A PTX .extern declaration can be resolved by a later .visible, .weak, or
519 // .common definition, but not by a static definition.
520 if (GVar->hasExternalLinkage())
521 return GVar->hasInitializer();
522
523 if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
524 GVar->hasAvailableExternallyLinkage() || GVar->hasCommonLinkage())
525 return true;
526
527 return false;
528}
529
530/// Order definitions after treating references to forward-declared globals as
531/// already satisfied. A remaining cycle cannot be emitted portably because it
532/// requires an undeclared forward reference.
533static SmallVector<const GlobalVariable *, 4> orderDefinitionsInSCC(
534 ArrayRef<const GlobalVariableDependencyNode *> SCC,
535 const DenseSet<const GlobalVariableDependencyNode *> &ForwardDeclared) {
536 using Node = GlobalVariableDependencyNode;
537
538 DenseSet<const Node *> SCCSet;
539 SCCSet.insert_range(R&: SCC);
540
541 DenseMap<const Node *, unsigned> DependencyCount;
542 DenseMap<const Node *, SmallVector<const Node *, 4>> Dependents;
543 std::set<std::pair<unsigned, const Node *>> Ready;
544
545 // Dependencies outside this SCC have already been emitted. Forward-declared
546 // dependencies are also satisfied, so only count the remaining SCC edges.
547 for (const Node *N : SCC) {
548 unsigned &Count = DependencyCount[N];
549 for (const Node *Dependency : N->Dependencies) {
550 if (!SCCSet.count(V: Dependency) || ForwardDeclared.count(V: Dependency))
551 continue;
552 ++Count;
553 Dependents[Dependency].push_back(Elt: N);
554 }
555 if (Count == 0)
556 Ready.emplace(args: N->ModuleOrder, args&: N);
557 }
558
559 SmallVector<const GlobalVariable *, 4> Order;
560 while (!Ready.empty()) {
561 const Node *N = Ready.begin()->second;
562 Ready.erase(position: Ready.begin());
563 Order.push_back(Elt: N->GV);
564
565 auto It = Dependents.find(Val: N);
566 if (It == Dependents.end())
567 continue;
568 for (const Node *Dependent : It->second) {
569 assert(DependencyCount[Dependent] && "Dependency already satisfied");
570 if (--DependencyCount[Dependent] == 0)
571 Ready.emplace(args: Dependent->ModuleOrder, args&: Dependent);
572 }
573 }
574
575 if (Order.size() != SCC.size())
576 report_fatal_error(reason: "Circular dependency found in global variable set");
577 return Order;
578}
579
580} // namespace
581
582void NVPTXAsmPrinter::emitInstruction(const MachineInstr *MI) {
583 NVPTX_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(),
584 Features: getSubtargetInfo().getFeatureBits());
585
586 MCInst Inst;
587 lowerToMCInst(MI, OutMI&: Inst);
588 EmitToStreamer(S&: *OutStreamer, Inst);
589}
590
591void NVPTXAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
592 OutMI.setOpcode(MI->getOpcode());
593 for (const auto MO : MI->operands())
594 OutMI.addOperand(Op: lowerOperand(MO));
595}
596
597MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) {
598 switch (MO.getType()) {
599 default:
600 llvm_unreachable("unknown operand type");
601 case MachineOperand::MO_Register:
602 return MCOperand::createReg(Reg: encodeVirtualRegister(Reg: MO.getReg()));
603 case MachineOperand::MO_Immediate:
604 return MCOperand::createImm(Val: MO.getImm());
605 case MachineOperand::MO_MachineBasicBlock:
606 return MCOperand::createExpr(
607 Val: MCSymbolRefExpr::create(Symbol: MO.getMBB()->getSymbol(), Ctx&: OutContext));
608 case MachineOperand::MO_ExternalSymbol:
609 return GetSymbolRef(Symbol: GetExternalSymbolSymbol(Sym: MO.getSymbolName()));
610 case MachineOperand::MO_JumpTableIndex:
611 // The jump table index names the .branchtargets list emitted for a brx.idx
612 // (see emitJumpTable); reference it by that label.
613 return GetSymbolRef(Symbol: GetJTISymbol(JTID: MO.getIndex()));
614 case MachineOperand::MO_GlobalAddress:
615 return GetSymbolRef(Symbol: getSymbol(GV: MO.getGlobal()));
616 case MachineOperand::MO_FPImmediate: {
617 const ConstantFP *Cnt = MO.getFPImm();
618 const APFloat &Val = Cnt->getValueAPF();
619
620 switch (Cnt->getType()->getTypeID()) {
621 default:
622 report_fatal_error(reason: "Unsupported FP type");
623 break;
624 case Type::HalfTyID:
625 return MCOperand::createExpr(
626 Val: NVPTXFloatMCExpr::createConstantFPHalf(Flt: Val, Ctx&: OutContext));
627 case Type::BFloatTyID:
628 return MCOperand::createExpr(
629 Val: NVPTXFloatMCExpr::createConstantBFPHalf(Flt: Val, Ctx&: OutContext));
630 case Type::FloatTyID:
631 return MCOperand::createExpr(
632 Val: NVPTXFloatMCExpr::createConstantFPSingle(Flt: Val, Ctx&: OutContext));
633 case Type::DoubleTyID:
634 return MCOperand::createExpr(
635 Val: NVPTXFloatMCExpr::createConstantFPDouble(Flt: Val, Ctx&: OutContext));
636 }
637 break;
638 }
639 }
640}
641
642static NVPTX::VirtualRegisterKind
643getVirtualRegisterKind(const TargetRegisterClass *RC) {
644 if (RC == &NVPTX::B1RegClass)
645 return NVPTX::VirtualRegisterKind::B1;
646 if (RC == &NVPTX::B16RegClass)
647 return NVPTX::VirtualRegisterKind::B16;
648 if (RC == &NVPTX::B32RegClass)
649 return NVPTX::VirtualRegisterKind::B32;
650 if (RC == &NVPTX::B64RegClass)
651 return NVPTX::VirtualRegisterKind::B64;
652 if (RC == &NVPTX::B128RegClass)
653 return NVPTX::VirtualRegisterKind::B128;
654 llvm_unreachable("Bad register class");
655}
656
657unsigned NVPTXAsmPrinter::getVirtualRegisterNumber(Register Reg) const {
658 const auto It = VRegMapping.find(Val: MRI->getRegClass(Reg));
659 assert(It != VRegMapping.end() && "Bad register class");
660
661 const unsigned Num = It->second.lookup(Val: Reg);
662 assert(Num && "Bad virtual register");
663 return Num;
664}
665
666MCRegister NVPTXAsmPrinter::encodeVirtualRegister(Register Reg) {
667 if (Reg.isVirtual()) {
668 // Pack the register class into the upper bits so that
669 // NVPTXInstPrinter::printRegName can recover the declared name.
670 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
671 const unsigned Num = getVirtualRegisterNumber(Reg);
672 assert(Num <= NVPTX::VirtualRegisterNumMask &&
673 "Too many virtual registers");
674 return (static_cast<unsigned>(Kind) << NVPTX::VirtualRegisterKindShift) |
675 Num;
676 }
677
678 // Some special-use registers are actually physical registers.
679 // Encode this as the register class ID of 0 and the real register ID.
680 assert(Reg.id() <= NVPTX::VirtualRegisterNumMask &&
681 "Physical register would decode as a virtual register");
682 return Reg.asMCReg();
683}
684
685MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) {
686 const MCExpr *Expr;
687 Expr = MCSymbolRefExpr::create(Symbol, Ctx&: OutContext);
688 return MCOperand::createExpr(Val: Expr);
689}
690
691void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
692 const DataLayout &DL = getDataLayout();
693 const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F);
694 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
695
696 Type *Ty = F->getReturnType();
697 // A void or zero-sized return type (e.g. an empty struct) produces no return
698 // parameter.
699 if (Ty->isVoidTy() || Ty->isEmptyTy())
700 return;
701 O << " (";
702
703 auto PrintScalarRetVal = [&](unsigned Size) {
704 O << ".param .b" << promoteScalarArgumentSize(size: Size) << " func_retval0";
705 };
706 if (shouldPassAsArray(Ty)) {
707 const unsigned TotalSize = DL.getTypeAllocSize(Ty);
708 const Align RetAlignment =
709 getPTXParamAlign(F, Ty, AttrIdx: AttributeList::ReturnIndex, DL);
710 O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
711 << TotalSize << "]";
712 } else if (Ty->isFloatingPointTy()) {
713 PrintScalarRetVal(Ty->getPrimitiveSizeInBits());
714 } else if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
715 PrintScalarRetVal(ITy->getBitWidth());
716 } else if (isa<PointerType>(Val: Ty)) {
717 PrintScalarRetVal(TLI->getPointerTy(DL).getSizeInBits());
718 } else
719 llvm_unreachable("Unknown return type");
720 O << ") ";
721}
722
723void NVPTXAsmPrinter::printReturnValStr(const MachineFunction &MF,
724 raw_ostream &O) {
725 const Function &F = MF.getFunction();
726 printReturnValStr(F: &F, O);
727}
728
729void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB,
730 unsigned UniqueCallSite,
731 raw_ostream &O) const {
732 const DataLayout &DL = getDataLayout();
733 const NVPTXSubtarget &STI = MF->getSubtarget<NVPTXSubtarget>();
734 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
735 const auto PtrVT = TLI->getPointerTy(DL);
736 Type *RetTy = CB.getFunctionType()->getReturnType();
737
738 O << "prototype_" << UniqueCallSite << " : .callprototype ";
739
740 if (RetTy->isVoidTy() || RetTy->isEmptyTy()) {
741 O << "()";
742 } else {
743 O << "(";
744 if (shouldPassAsArray(Ty: RetTy)) {
745 const Align RetAlign =
746 getPTXParamAlign(CB: &CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
747 O << ".param .align " << RetAlign.value() << " .b8 _["
748 << DL.getTypeAllocSize(Ty: RetTy) << "]";
749 } else if (RetTy->isFloatingPointTy() || RetTy->isIntegerTy()) {
750 unsigned size = 0;
751 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
752 size = ITy->getBitWidth();
753 } else {
754 assert(RetTy->isFloatingPointTy() &&
755 "Floating point type expected here");
756 size = RetTy->getPrimitiveSizeInBits();
757 }
758 // PTX ABI requires all scalar return values to be at least 32
759 // bits in size. fp16 normally uses .b16 as its storage type in
760 // PTX, so its size must be adjusted here, too.
761 size = promoteScalarArgumentSize(size);
762
763 O << ".param .b" << size << " _";
764 } else if (isa<PointerType>(Val: RetTy)) {
765 O << ".param .b" << PtrVT.getSizeInBits() << " _";
766 } else {
767 llvm_unreachable("Unknown return type");
768 }
769 O << ") ";
770 }
771 O << "_ (";
772
773 auto MakeArg = [&](const unsigned I) {
774 Type *Ty = CB.getArgOperand(i: I)->getType();
775
776 if (CB.paramHasAttr(ArgNo: I, Kind: Attribute::ByVal)) {
777 Type *ETy = CB.getParamByValType(ArgNo: I);
778 Align ParamByValAlign = getDeviceByValParamAlign(
779 CB: &CB, ArgTy: ETy, AttrIdx: I + AttributeList::FirstArgIndex, DL);
780
781 O << ".param .align " << ParamByValAlign.value() << " .b8 _["
782 << DL.getTypeAllocSize(Ty: ETy) << "]";
783 return;
784 }
785
786 if (shouldPassAsArray(Ty)) {
787 Align ParamAlign =
788 getPTXParamAlign(CB: &CB, Ty, AttrIdx: I + AttributeList::FirstArgIndex, DL);
789 O << ".param .align " << ParamAlign.value() << " .b8 _["
790 << DL.getTypeAllocSize(Ty) << "]";
791 return;
792 }
793 // scalar type
794 unsigned sz = 0;
795 if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
796 sz = promoteScalarArgumentSize(size: ITy->getBitWidth());
797 } else if (isa<PointerType>(Val: Ty)) {
798 sz = PtrVT.getSizeInBits();
799 } else {
800 sz = Ty->getPrimitiveSizeInBits();
801 }
802 O << ".param .b" << sz << " _";
803 };
804
805 const FunctionType *FTy = CB.getFunctionType();
806 const unsigned NumArgs = FTy->getNumParams();
807
808 // Zero-sized arguments (e.g. empty structs) are not passed and so do not
809 // appear in the prototype.
810 const auto NonEmptyArgs = make_filter_range(Range: seq(Size: NumArgs), Pred: [&](unsigned I) {
811 return !CB.getArgOperand(i: I)->getType()->isEmptyTy();
812 });
813
814 interleave(c: NonEmptyArgs, os&: O, each_fn: MakeArg, separator: ", ");
815
816 if (FTy->isVarArg() && CB.arg_size() > NumArgs)
817 O << (NonEmptyArgs.empty() ? "" : ",") << " .param .align "
818 << STI.getMaxRequiredAlignment() << " .b8 _[]";
819
820 O << ")";
821 if (shouldEmitPTXNoReturn(V: CB))
822 O << " .noreturn";
823 O << ";\n";
824}
825
826void NVPTXAsmPrinter::emitJumpTable(const MachineJumpTableEntry &MJT,
827 unsigned MJTI) const {
828 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: MJTI));
829
830 if (MJT.MBBs.empty())
831 return;
832
833 const auto Targets = to_vector(
834 Range: map_range(C: MJT.MBBs, F: [](const MachineBasicBlock *MBB) -> const MCSymbol * {
835 return MBB->getSymbol();
836 }));
837 getTargetStreamer()->emitBranchTargetsDirective(Targets);
838}
839
840// Return true if MBB is the header of a loop marked with
841// llvm.loop.unroll.disable or llvm.loop.unroll.count=1.
842bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
843 const MachineBasicBlock &MBB) const {
844 const MachineLoopInfo *LI = GetMLI(*MF);
845 assert(LI && "NVPTXAsmPrinter requires MachineLoopInfo");
846 // We insert .pragma "nounroll" only to the loop header.
847 if (!LI->isLoopHeader(BB: &MBB))
848 return false;
849
850 // llvm.loop.unroll.disable is marked on the back edges of a loop. Therefore,
851 // we iterate through each back edge of the loop with header MBB, and check
852 // whether its metadata contains llvm.loop.unroll.disable.
853 for (const MachineBasicBlock *PMBB : MBB.predecessors()) {
854 if (LI->getLoopFor(BB: PMBB) != LI->getLoopFor(BB: &MBB)) {
855 // Edges from other loops to MBB are not back edges.
856 continue;
857 }
858 if (const BasicBlock *PBB = PMBB->getBasicBlock()) {
859 if (MDNode *LoopID =
860 PBB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop)) {
861 if (GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.disable"))
862 return true;
863 if (MDNode *UnrollCountMD =
864 GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.count")) {
865 if (mdconst::extract<ConstantInt>(MD: UnrollCountMD->getOperand(I: 1))
866 ->isOne())
867 return true;
868 }
869 }
870 }
871 }
872 return false;
873}
874
875void NVPTXAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
876 AsmPrinter::emitBasicBlockStart(MBB);
877 if (isLoopHeaderOfNoUnroll(MBB))
878 getTargetStreamer()->emitPragmaDirective(Pragma: "nounroll");
879}
880
881void NVPTXAsmPrinter::emitFunctionEntryLabel() {
882 SmallString<128> Str;
883 raw_svector_ostream O(Str);
884
885 if (!GlobalsEmitted) {
886 emitGlobals(M: *MF->getFunction().getParent());
887 GlobalsEmitted = true;
888 }
889
890 // Set up
891 MRI = &MF->getRegInfo();
892 F = &MF->getFunction();
893 emitLinkageDirective(V: F, O);
894 if (isKernelFunction(F: *F))
895 O << ".entry ";
896 else {
897 O << ".func ";
898 printReturnValStr(MF: *MF, O);
899 }
900
901 CurrentFnSym->print(OS&: O, MAI);
902
903 emitFunctionParamList(F, O);
904 O << "\n";
905
906 if (isKernelFunction(F: *F))
907 emitKernelFunctionDirectives(F: *F, O);
908
909 if (shouldEmitPTXNoReturn(V: *F))
910 O << ".noreturn";
911
912 OutStreamer->emitRawText(String: O.str());
913
914 VRegMapping.clear();
915 // Emit open brace for function body.
916 OutStreamer->emitRawText(String: StringRef("{\n"));
917 setAndEmitFunctionVirtualRegisters(*MF);
918 encodeDebugInfoRegisterNumbers(MF: *MF);
919 // Emit initial .loc debug directive for correct relocation symbol data.
920 emitInitialRawDwarfLocDirective(MF: *MF, DD: getDwarfDebug(), OutStreamer&: *OutStreamer);
921}
922
923bool NVPTXAsmPrinter::runOnMachineFunction(MachineFunction &F) {
924 bool Result = AsmPrinter::runOnMachineFunction(MF&: F);
925 // Emit closing brace for the body of function F.
926 // The closing brace must be emitted here because we need to emit additional
927 // debug labels/data after the last basic block.
928 // We need to emit the closing brace here because we don't have function that
929 // finished emission of the function body.
930 OutStreamer->emitRawText(String: StringRef("}\n"));
931 return Result;
932}
933
934void NVPTXAsmPrinter::emitFunctionBodyStart() {
935 SmallString<128> Str;
936 raw_svector_ostream O(Str);
937 emitDemotedVars(&MF->getFunction(), O);
938
939 const auto *MFI = MF->getInfo<NVPTXMachineFunctionInfo>();
940 for (const auto &[Id, CB] : MFI->getCallPrototypes())
941 emitCallPrototype(CB: *CB, UniqueCallSite: Id, O);
942
943 OutStreamer->emitRawText(String: O.str());
944
945 if (const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo())
946 for (const auto &[Idx, JT] : enumerate(First: MJTI->getJumpTables()))
947 emitJumpTable(MJT: JT, MJTI: Idx);
948}
949
950void NVPTXAsmPrinter::emitFunctionBodyEnd() {
951 VRegMapping.clear();
952}
953
954const MCSymbol *NVPTXAsmPrinter::getFunctionFrameSymbol() const {
955 return OutContext.getOrCreateSymbol(DEPOTNAME + Twine(getFunctionNumber()));
956}
957
958void NVPTXAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
959 Register RegNo = MI->getOperand(i: 0).getReg();
960 if (RegNo.isVirtual())
961 OutStreamer->AddComment(T: Twine("implicit-def: ") +
962 getVirtualRegisterName(Reg: RegNo));
963 else
964 OutStreamer->AddComment(T: Twine("implicit-def: ") +
965 NVPTXInstPrinter::getRegisterName(Reg: RegNo));
966 OutStreamer->addBlankLine();
967}
968
969void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
970 raw_ostream &O) const {
971 // If the NVVM IR has some of reqntid* specified, then output
972 // the reqntid directive, and set the unspecified ones to 1.
973 // If none of Reqntid* is specified, don't output reqntid directive.
974 const auto ReqNTID = getReqNTID(F);
975 if (!ReqNTID.empty())
976 O << formatv(Fmt: ".reqntid {0:$[, ]}\n",
977 Vals: make_range(x: ReqNTID.begin(), y: ReqNTID.end()));
978
979 const auto MaxNTID = getMaxNTID(F);
980 if (!MaxNTID.empty())
981 O << formatv(Fmt: ".maxntid {0:$[, ]}\n",
982 Vals: make_range(x: MaxNTID.begin(), y: MaxNTID.end()));
983
984 if (const auto Mincta = getMinCTASm(F))
985 O << ".minnctapersm " << *Mincta << "\n";
986
987 if (const auto Maxnreg = getMaxNReg(F))
988 O << ".maxnreg " << *Maxnreg << "\n";
989
990 // .maxclusterrank directive requires SM_90 or higher, make sure that we
991 // filter it out for lower SM versions, as it causes a hard ptxas crash.
992 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
993 const NVPTXSubtarget *STI = &NTM.getSubtarget<NVPTXSubtarget>(F);
994
995 if (STI->hasFeature(Feature: NVPTX::SM90)) {
996 const auto ClusterDim = getClusterDim(F);
997 const bool BlocksAreClusters = hasBlocksAreClusters(F);
998
999 if (!ClusterDim.empty()) {
1000
1001 if (!BlocksAreClusters)
1002 O << ".explicitcluster\n";
1003
1004 if (ClusterDim[0] != 0) {
1005 assert(llvm::all_of(ClusterDim, not_equal_to(0)) &&
1006 "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z "
1007 "should be non-zero as well");
1008
1009 O << formatv(Fmt: ".reqnctapercluster {0:$[, ]}\n",
1010 Vals: make_range(x: ClusterDim.begin(), y: ClusterDim.end()));
1011 } else {
1012 assert(llvm::all_of(ClusterDim, equal_to(0)) &&
1013 "cluster_dim_x == 0 implies cluster_dim_y and cluster_dim_z "
1014 "should be 0 as well");
1015 }
1016 }
1017
1018 if (BlocksAreClusters) {
1019 LLVMContext &Ctx = F.getContext();
1020 if (ReqNTID.empty() || ClusterDim.empty())
1021 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1022 F, "blocksareclusters requires reqntid and cluster_dim attributes",
1023 F.getSubprogram()));
1024 else if (!STI->hasFeature(Feature: NVPTX::PTX90))
1025 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1026 F, "blocksareclusters requires PTX version >= 9.0",
1027 F.getSubprogram()));
1028 else
1029 O << ".blocksareclusters\n";
1030 }
1031
1032 if (const auto Maxclusterrank = getMaxClusterRank(F))
1033 O << ".maxclusterrank " << *Maxclusterrank << "\n";
1034 }
1035}
1036
1037std::string NVPTXAsmPrinter::getVirtualRegisterName(Register Reg) const {
1038 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
1039
1040 std::string Name;
1041 raw_string_ostream(Name) << NVPTX::getVirtualRegisterPrefix(Kind)
1042 << getVirtualRegisterNumber(Reg);
1043 return Name;
1044}
1045
1046void NVPTXAsmPrinter::emitAliasDeclaration(const GlobalAlias *GA,
1047 raw_ostream &O) {
1048 const Function *F = dyn_cast_or_null<Function>(Val: GA->getAliaseeObject());
1049 if (!F || isKernelFunction(F: *F) || F->isDeclaration())
1050 report_fatal_error(
1051 reason: "NVPTX aliasee must be a non-kernel function definition");
1052
1053 if (GA->hasLinkOnceLinkage() || GA->hasWeakLinkage() ||
1054 GA->hasAvailableExternallyLinkage() || GA->hasCommonLinkage())
1055 report_fatal_error(reason: "NVPTX aliasee must not be '.weak'");
1056
1057 emitDeclarationWithName(F, getSymbol(GV: GA), O);
1058}
1059
1060void NVPTXAsmPrinter::emitDeclaration(const Function *F, raw_ostream &O) {
1061 emitDeclarationWithName(F, getSymbol(GV: F), O);
1062}
1063
1064void NVPTXAsmPrinter::emitDeclarationWithName(const Function *F, MCSymbol *S,
1065 raw_ostream &O) {
1066 emitLinkageDirective(V: F, O);
1067 if (isKernelFunction(F: *F))
1068 O << ".entry ";
1069 else
1070 O << ".func ";
1071 printReturnValStr(F, O);
1072 S->print(OS&: O, MAI);
1073 O << "\n";
1074 emitFunctionParamList(F, O);
1075 O << "\n";
1076 if (shouldEmitPTXNoReturn(V: *F))
1077 O << ".noreturn";
1078 O << ";\n";
1079}
1080
1081static bool usedInGlobalVarDef(const Constant *C) {
1082 if (!C)
1083 return false;
1084
1085 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C))
1086 return GV->getName() != "llvm.used";
1087
1088 for (const User *U : C->users())
1089 if (const Constant *C = dyn_cast<Constant>(Val: U))
1090 if (usedInGlobalVarDef(C))
1091 return true;
1092
1093 return false;
1094}
1095
1096static bool usedInOneFunc(const User *U, Function const *&OneFunc) {
1097 if (const GlobalVariable *OtherGV = dyn_cast<GlobalVariable>(Val: U))
1098 if (OtherGV->getName() == "llvm.used")
1099 return true;
1100
1101 if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1102 if (const Function *CurFunc = I->getFunction()) {
1103 if (OneFunc && (CurFunc != OneFunc))
1104 return false;
1105 OneFunc = CurFunc;
1106 return true;
1107 }
1108 return false;
1109 }
1110
1111 for (const User *UU : U->users())
1112 if (!usedInOneFunc(U: UU, OneFunc))
1113 return false;
1114
1115 return true;
1116}
1117
1118/* Find out if a global variable can be demoted to local scope.
1119 * Currently, this is valid for CUDA shared variables, which have local
1120 * scope and global lifetime. So the conditions to check are :
1121 * 1. Is the global variable in shared address space?
1122 * 2. Does it have local linkage?
1123 * 3. Is the global variable referenced only in one function?
1124 */
1125static bool canDemoteGlobalVar(const GlobalVariable *GV, Function const *&f) {
1126 if (!GV->hasLocalLinkage())
1127 return false;
1128 if (GV->getAddressSpace() != ADDRESS_SPACE_SHARED)
1129 return false;
1130
1131 const Function *oneFunc = nullptr;
1132
1133 bool flag = usedInOneFunc(U: GV, OneFunc&: oneFunc);
1134 if (!flag)
1135 return false;
1136 if (!oneFunc)
1137 return false;
1138 f = oneFunc;
1139 return true;
1140}
1141
1142static bool useFuncSeen(const Constant *C,
1143 const SmallPtrSetImpl<const Function *> &SeenSet) {
1144 for (const User *U : C->users()) {
1145 if (const Constant *cu = dyn_cast<Constant>(Val: U)) {
1146 if (useFuncSeen(C: cu, SeenSet))
1147 return true;
1148 } else if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1149 if (const Function *Caller = I->getFunction())
1150 if (SeenSet.contains(Ptr: Caller))
1151 return true;
1152 }
1153 }
1154 return false;
1155}
1156
1157void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) {
1158 SmallPtrSet<const Function *, 32> SeenSet;
1159 for (const Function &F : M) {
1160 if (F.getAttributes().hasFnAttr(Kind: "nvptx-libcall-callee")) {
1161 emitDeclaration(F: &F, O);
1162 continue;
1163 }
1164
1165 if (F.isDeclaration()) {
1166 if (F.use_empty())
1167 continue;
1168 if (F.getIntrinsicID())
1169 continue;
1170 // An unrecognized intrinsic would produce an invalid PTX declaration. Let
1171 // the user know that, and skip it.
1172 if (F.isIntrinsic()) {
1173 LLVMContext &Ctx = F.getContext();
1174 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1175 F, "unknown intrinsic '" + F.getName() +
1176 "' cannot be lowered by the NVPTX backend"));
1177 continue;
1178 }
1179 emitDeclaration(F: &F, O);
1180 continue;
1181 }
1182 for (const User *U : F.users()) {
1183 if (const Constant *C = dyn_cast<Constant>(Val: U)) {
1184 if (usedInGlobalVarDef(C)) {
1185 // The use is in the initialization of a global variable
1186 // that is a function pointer, so print a declaration
1187 // for the original function
1188 emitDeclaration(F: &F, O);
1189 break;
1190 }
1191 // Emit a declaration of this function if the function that
1192 // uses this constant expr has already been seen.
1193 if (useFuncSeen(C, SeenSet)) {
1194 emitDeclaration(F: &F, O);
1195 break;
1196 }
1197 }
1198
1199 if (!isa<Instruction>(Val: U))
1200 continue;
1201 const Function *Caller = cast<Instruction>(Val: U)->getFunction();
1202 if (!Caller)
1203 continue;
1204
1205 // If a caller has already been seen, then the caller is
1206 // appearing in the module before the callee. so print out
1207 // a declaration for the callee.
1208 if (SeenSet.contains(Ptr: Caller)) {
1209 emitDeclaration(F: &F, O);
1210 break;
1211 }
1212 }
1213 SeenSet.insert(Ptr: &F);
1214 }
1215 for (const GlobalAlias &GA : M.aliases())
1216 emitAliasDeclaration(GA: &GA, O);
1217}
1218
1219void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) {
1220 // Construct a default subtarget off of the TargetMachine defaults. The
1221 // rest of NVPTX isn't friendly to change subtargets per function and
1222 // so the default TargetMachine will have all of the options.
1223 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1224 const NVPTXSubtarget *STI = NTM.getSubtargetImpl();
1225
1226 // Emit header before any dwarf directives are emitted below.
1227 emitHeader(M, STI: *STI);
1228}
1229
1230/// Create NVPTX-specific DwarfDebug handler.
1231DwarfDebug *NVPTXAsmPrinter::createDwarfDebug() {
1232 return new NVPTXDwarfDebug(this);
1233}
1234
1235bool NVPTXAsmPrinter::doInitialization(Module &M) {
1236 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1237 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1238 if (M.alias_size() &&
1239 (!STI.hasFeature(Feature: NVPTX::PTX63) || !STI.hasFeature(Feature: NVPTX::SM30)))
1240 report_fatal_error(reason: ".alias requires PTX version >= 6.3 and sm_30");
1241
1242 // We need to call the parent's one explicitly.
1243 bool Result = AsmPrinter::doInitialization(M);
1244
1245 GlobalsEmitted = false;
1246
1247 return Result;
1248}
1249
1250void NVPTXAsmPrinter::emitGlobals(const Module &M) {
1251 SmallString<128> Str2;
1252 raw_svector_ostream OS2(Str2);
1253
1254 emitDeclarations(M, O&: OS2);
1255
1256 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1257 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1258
1259 // ptxas requires global symbols referenced by initializers to be known
1260 // before use. Acyclic dependencies can be handled by dependency-first
1261 // emission. Cyclic SCCs need compatible .extern declarations first.
1262 // Edges point from each global to the globals used by its initializer.
1263 // Reverse-topological SCC iteration therefore emits dependencies first.
1264 GlobalVariableDependencyGraph DependencyGraph(M);
1265 for (GlobalVariableSCCIterator I =
1266 GlobalVariableSCCIterator::begin(G: DependencyGraph.getEntryNode());
1267 !I.isAtEnd(); ++I) {
1268 SmallVector<const GlobalVariableDependencyNode *, 4> SCC(I->begin(),
1269 I->end());
1270
1271 // Nothing points to the synthetic root, so it is always in its own SCC.
1272 if (!SCC.front()->GV) {
1273 assert(SCC.size() == 1 && "Synthetic root must be in its own SCC");
1274 continue;
1275 }
1276
1277 llvm::sort(C&: SCC, Comp: [](const auto *LHS, const auto *RHS) {
1278 return LHS->ModuleOrder < RHS->ModuleOrder;
1279 });
1280
1281 const bool IsCyclic = I.hasCycle();
1282 DenseSet<const GlobalVariableDependencyNode *> ForwardDeclared;
1283 if (IsCyclic)
1284 for (const auto *Node : SCC)
1285 if (isForwardDeclarableGlobal(GVar: Node->GV))
1286 ForwardDeclared.insert(V: Node);
1287
1288 // Check that declarations break every cycle before writing any output.
1289 SmallVector<const GlobalVariable *, 4> OrderedGlobals =
1290 IsCyclic ? orderDefinitionsInSCC(SCC, ForwardDeclared)
1291 : SmallVector<const GlobalVariable *, 4>{SCC.front()->GV};
1292
1293 for (const auto *Node : SCC) {
1294 if (!ForwardDeclared.count(V: Node))
1295 continue;
1296 OS2 << ".extern ";
1297 emitPTXGlobalVariableDefinition(GVar: Node->GV, O&: OS2, STI,
1298 /*EmitInitializer=*/false);
1299 OS2 << ";\n";
1300 }
1301
1302 for (const GlobalVariable *GV : OrderedGlobals)
1303 printModuleLevelGV(GVar: GV, O&: OS2, /*ProcessDemoted=*/processDemoted: false, STI);
1304 }
1305
1306 OS2 << '\n';
1307
1308 OutStreamer->emitRawText(String: OS2.str());
1309}
1310
1311void NVPTXAsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) {
1312 getTargetStreamer()->emitAliasDirective(Name: getSymbol(GV: &GA),
1313 Aliasee: getSymbol(GV: GA.getAliaseeObject()));
1314}
1315
1316NVPTXTargetStreamer *NVPTXAsmPrinter::getTargetStreamer() const {
1317 return static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1318}
1319
1320static bool hasFullDebugInfo(Module &M) {
1321 for (DICompileUnit *CU : M.debug_compile_units()) {
1322 switch(CU->getEmissionKind()) {
1323 case DICompileUnit::NoDebug:
1324 case DICompileUnit::DebugDirectivesOnly:
1325 break;
1326 case DICompileUnit::LineTablesOnly:
1327 case DICompileUnit::FullDebug:
1328 return true;
1329 }
1330 }
1331
1332 return false;
1333}
1334
1335void NVPTXAsmPrinter::emitHeader(Module &M, const NVPTXSubtarget &STI) {
1336 auto *TS = getTargetStreamer();
1337
1338 TS->emitBanner();
1339
1340 const unsigned PTXVersion = STI.getPTXVersion();
1341 TS->emitVersionDirective(PTXVersion);
1342
1343 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1344 bool TexModeIndependent = NTM.getDrvInterface() == NVPTX::NVCL;
1345
1346 TS->emitTargetDirective(Target: STI.getTargetName(), TexModeIndependent,
1347 HasDebug: hasFullDebugInfo(M));
1348 TS->emitAddressSizeDirective(AddrSize: M.getDataLayout().getPointerSizeInBits());
1349}
1350
1351bool NVPTXAsmPrinter::doFinalization(Module &M) {
1352 // If we did not emit any functions, then the global declarations have not
1353 // yet been emitted.
1354 if (!GlobalsEmitted) {
1355 emitGlobals(M);
1356 GlobalsEmitted = true;
1357 }
1358
1359 // call doFinalization
1360 bool ret = AsmPrinter::doFinalization(M);
1361
1362 clearAnnotationCache(&M);
1363
1364 auto *TS =
1365 static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1366 // Close the last emitted section
1367 if (hasDebugInfo()) {
1368 TS->closeLastSection();
1369 // Emit empty .debug_macinfo section for better support of the empty files.
1370 TS->emitEmptySectionDirective(Name: ".debug_macinfo");
1371 }
1372
1373 // Output last DWARF .file directives, if any.
1374 TS->outputDwarfFileDirectives();
1375
1376 return ret;
1377}
1378
1379// This function emits appropriate linkage directives for
1380// functions and global variables.
1381//
1382// extern function declaration -> .extern
1383// extern function definition -> .visible
1384// external global variable with init -> .visible
1385// external without init -> .extern
1386// appending -> not allowed, assert.
1387// for any linkage other than
1388// internal, private, linker_private,
1389// linker_private_weak, linker_private_weak_def_auto,
1390// we emit -> .weak.
1391
1392void NVPTXAsmPrinter::emitLinkageDirective(const GlobalValue *V,
1393 raw_ostream &O) {
1394 if (static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() == NVPTX::CUDA) {
1395 if (V->hasExternalLinkage()) {
1396 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: V))
1397 O << (GVar->hasInitializer() ? ".visible " : ".extern ");
1398 else if (V->isDeclaration())
1399 O << ".extern ";
1400 else
1401 O << ".visible ";
1402 } else if (V->hasAppendingLinkage()) {
1403 report_fatal_error(reason: "Symbol '" + (V->hasName() ? V->getName() : "") +
1404 "' has unsupported appending linkage type");
1405 } else if (!V->hasInternalLinkage() && !V->hasPrivateLinkage()) {
1406 O << ".weak ";
1407 }
1408 }
1409}
1410
1411void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
1412 raw_ostream &O, bool ProcessDemoted,
1413 const NVPTXSubtarget &STI) {
1414 // Skip metadata and LLVM intrinsic global variables.
1415 if (shouldSkipModuleLevelGlobal(GV: *GVar))
1416 return;
1417
1418 if (GVar->hasExternalLinkage()) {
1419 if (GVar->hasInitializer())
1420 O << ".visible ";
1421 else
1422 O << ".extern ";
1423 } else if (STI.hasFeature(Feature: NVPTX::PTX50) && GVar->hasCommonLinkage() &&
1424 GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) {
1425 O << ".common ";
1426 } else if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
1427 GVar->hasAvailableExternallyLinkage() ||
1428 GVar->hasCommonLinkage()) {
1429 O << ".weak ";
1430 }
1431
1432 const PTXOpaqueType OpaqueType = getPTXOpaqueType(*GVar);
1433
1434 if (OpaqueType == PTXOpaqueType::Texture) {
1435 O << ".global .texref " << getTextureName(V: *GVar) << ";\n";
1436 return;
1437 }
1438
1439 if (OpaqueType == PTXOpaqueType::Surface) {
1440 O << ".global .surfref " << getSurfaceName(V: *GVar) << ";\n";
1441 return;
1442 }
1443
1444 if (GVar->isDeclaration()) {
1445 // (extern) declarations, no definition or initializer
1446 // Currently the only known declaration is for an automatic __local
1447 // (.shared) promoted to global.
1448 emitPTXGlobalVariable(GVar, O, STI);
1449 O << ";\n";
1450 return;
1451 }
1452
1453 if (OpaqueType == PTXOpaqueType::Sampler) {
1454 O << ".global .samplerref " << getSamplerName(V: *GVar);
1455
1456 const Constant *Initializer = nullptr;
1457 if (GVar->hasInitializer())
1458 Initializer = GVar->getInitializer();
1459 const ConstantInt *CI = nullptr;
1460 if (Initializer)
1461 CI = dyn_cast<ConstantInt>(Val: Initializer);
1462 if (CI) {
1463 unsigned sample = CI->getZExtValue();
1464
1465 O << " = { ";
1466
1467 for (int i = 0,
1468 addr = ((sample & __CLK_ADDRESS_MASK) >> __CLK_ADDRESS_BASE);
1469 i < 3; i++) {
1470 O << "addr_mode_" << i << " = ";
1471 switch (addr) {
1472 case 0:
1473 O << "wrap";
1474 break;
1475 case 1:
1476 O << "clamp_to_border";
1477 break;
1478 case 2:
1479 O << "clamp_to_edge";
1480 break;
1481 case 3:
1482 O << "wrap";
1483 break;
1484 case 4:
1485 O << "mirror";
1486 break;
1487 }
1488 O << ", ";
1489 }
1490 O << "filter_mode = ";
1491 switch ((sample & __CLK_FILTER_MASK) >> __CLK_FILTER_BASE) {
1492 case 0:
1493 O << "nearest";
1494 break;
1495 case 1:
1496 O << "linear";
1497 break;
1498 case 2:
1499 llvm_unreachable("Anisotropic filtering is not supported");
1500 default:
1501 O << "nearest";
1502 break;
1503 }
1504 if (!((sample & __CLK_NORMALIZED_MASK) >> __CLK_NORMALIZED_BASE)) {
1505 O << ", force_unnormalized_coords = 1";
1506 }
1507 O << " }";
1508 }
1509
1510 O << ";\n";
1511 return;
1512 }
1513
1514 if (GVar->hasPrivateLinkage()) {
1515 if (GVar->getName().starts_with(Prefix: "unrollpragma"))
1516 return;
1517
1518 // FIXME - need better way (e.g. Metadata) to avoid generating this global
1519 if (GVar->getName().starts_with(Prefix: "filename"))
1520 return;
1521 if (GVar->use_empty())
1522 return;
1523 }
1524
1525 const Function *DemotedFunc = nullptr;
1526 if (!ProcessDemoted && canDemoteGlobalVar(GV: GVar, f&: DemotedFunc)) {
1527 O << "// " << GVar->getName() << " has been demoted\n";
1528 localDecls[DemotedFunc].push_back(x: GVar);
1529 return;
1530 }
1531
1532 emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/true);
1533 O << ";\n";
1534}
1535
1536void NVPTXAsmPrinter::emitPTXGlobalVariableDefinition(
1537 const GlobalVariable *GVar, raw_ostream &O, const NVPTXSubtarget &STI,
1538 bool EmitInitializer) {
1539 const DataLayout &DL = getDataLayout();
1540
1541 Type *ETy = GVar->getValueType();
1542
1543 O << ".";
1544 emitPTXAddressSpace(AddressSpace: GVar->getAddressSpace(), O);
1545
1546 if (isManaged(*GVar)) {
1547 if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30))
1548 report_fatal_error(
1549 reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1550 O << " .attribute(.managed)";
1551 }
1552
1553 O << " .align "
1554 << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value();
1555
1556 if (ETy->isPointerTy() || ((ETy->isIntegerTy() || ETy->isFloatingPointTy()) &&
1557 ETy->getScalarSizeInBits() <= 64)) {
1558 O << " .";
1559 // Special case: ABI requires that we use .u8 for predicates
1560 if (ETy->isIntegerTy(BitWidth: 1))
1561 O << "u8";
1562 else
1563 O << getPTXFundamentalTypeStr(Ty: ETy, false);
1564 O << " ";
1565 getSymbol(GV: GVar)->print(OS&: O, MAI);
1566
1567 // Ptx allows variable initilization only for constant and global state
1568 // spaces.
1569 if (EmitInitializer && GVar->hasInitializer()) {
1570 if ((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) ||
1571 (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) {
1572 const Constant *Initializer = GVar->getInitializer();
1573 // 'undef' is treated as there is no value specified.
1574 if (!Initializer->isNullValue() && !isa<UndefValue>(Val: Initializer)) {
1575 O << " = ";
1576 printScalarConstant(CPV: Initializer, O);
1577 }
1578 } else {
1579 // The frontend adds zero-initializer to device and constant variables
1580 // that don't have an initial value, and UndefValue to shared
1581 // variables, so skip warning for this case.
1582 if (!GVar->getInitializer()->isNullValue() &&
1583 !isa<UndefValue>(Val: GVar->getInitializer())) {
1584 report_fatal_error(reason: "initial value of '" + GVar->getName() +
1585 "' is not allowed in addrspace(" +
1586 Twine(GVar->getAddressSpace()) + ")");
1587 }
1588 }
1589 }
1590 } else {
1591 // Although PTX has direct support for struct type and array type and
1592 // LLVM IR is very similar to PTX, the LLVM CodeGen does not support for
1593 // targets that support these high level field accesses. Structs, arrays
1594 // and vectors are lowered into arrays of bytes.
1595 switch (ETy->getTypeID()) {
1596 case Type::IntegerTyID: // Integers larger than 64 bits
1597 case Type::FP128TyID:
1598 case Type::StructTyID:
1599 case Type::ArrayTyID:
1600 case Type::FixedVectorTyID: {
1601 const uint64_t ElementSize = DL.getTypeStoreSize(Ty: ETy);
1602 // Ptx allows variable initilization only for constant and
1603 // global state spaces.
1604 if (((GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) ||
1605 (GVar->getAddressSpace() == ADDRESS_SPACE_CONST)) &&
1606 GVar->hasInitializer()) {
1607 const Constant *Initializer = GVar->getInitializer();
1608 if (!isa<UndefValue>(Val: Initializer) && !Initializer->isNullValue()) {
1609 AggBuffer aggBuffer(ElementSize, *this);
1610 bufferAggregateConstant(CV: Initializer, aggBuffer: &aggBuffer);
1611 if (aggBuffer.numSymbols()) {
1612 const unsigned int ptrSize = MAI.getCodePointerSize();
1613 if (ElementSize % ptrSize ||
1614 !aggBuffer.allSymbolsAligned(ptrSize)) {
1615 // Print in bytes and use the mask() operator for pointers.
1616 if (!STI.hasMaskOperator())
1617 report_fatal_error(
1618 reason: "initialized packed aggregate with pointers '" +
1619 GVar->getName() +
1620 "' requires at least PTX ISA version 7.1");
1621 O << " .u8 ";
1622 getSymbol(GV: GVar)->print(OS&: O, MAI);
1623 O << "[" << ElementSize << "]";
1624 if (EmitInitializer) {
1625 O << " = {";
1626 aggBuffer.printBytes(os&: O);
1627 O << "}";
1628 }
1629 } else {
1630 O << " .u" << ptrSize * 8 << " ";
1631 getSymbol(GV: GVar)->print(OS&: O, MAI);
1632 O << "[" << ElementSize / ptrSize << "]";
1633 if (EmitInitializer) {
1634 O << " = {";
1635 aggBuffer.printWords(os&: O);
1636 O << "}";
1637 }
1638 }
1639 } else {
1640 O << " .b8 ";
1641 getSymbol(GV: GVar)->print(OS&: O, MAI);
1642 O << "[" << ElementSize << "]";
1643 if (EmitInitializer) {
1644 O << " = {";
1645 aggBuffer.printBytes(os&: O);
1646 O << "}";
1647 }
1648 }
1649 } else {
1650 O << " .b8 ";
1651 getSymbol(GV: GVar)->print(OS&: O, MAI);
1652 if (ElementSize)
1653 O << "[" << ElementSize << "]";
1654 }
1655 } else {
1656 O << " .b8 ";
1657 getSymbol(GV: GVar)->print(OS&: O, MAI);
1658 if (ElementSize)
1659 O << "[" << ElementSize << "]";
1660 }
1661 break;
1662 }
1663 default:
1664 llvm_unreachable("type not supported yet");
1665 }
1666 }
1667}
1668
1669void NVPTXAsmPrinter::AggBuffer::printSymbol(unsigned nSym, raw_ostream &os) {
1670 const Value *v = Symbols[nSym];
1671 const Value *v0 = SymbolsBeforeStripping[nSym];
1672 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: v)) {
1673 MCSymbol *Name = AP.getSymbol(GV: GVar);
1674 PointerType *PTy = dyn_cast<PointerType>(Val: v0->getType());
1675 // Is v0 a generic pointer?
1676 bool isGenericPointer = PTy && PTy->getAddressSpace() == 0;
1677 if (EmitGeneric && isGenericPointer && !isa<Function>(Val: v)) {
1678 os << "generic(";
1679 Name->print(OS&: os, MAI: AP.MAI);
1680 os << ")";
1681 } else {
1682 Name->print(OS&: os, MAI: AP.MAI);
1683 }
1684 } else if (const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(Val: v0)) {
1685 const MCExpr *Expr = AP.lowerConstantForGV(CV: CExpr, ProcessingGeneric: false);
1686 AP.printMCExpr(Expr: *Expr, OS&: os);
1687 } else
1688 llvm_unreachable("symbol type unknown");
1689}
1690
1691void NVPTXAsmPrinter::AggBuffer::printBytes(raw_ostream &os) {
1692 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1693 // Do not emit trailing zero initializers. They will be zero-initialized by
1694 // ptxas. This saves on both space requirements for the generated PTX and on
1695 // memory use by ptxas. (See:
1696 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#global-state-space)
1697 unsigned int InitializerCount = Size;
1698 // TODO: symbols make this harder, but it would still be good to trim trailing
1699 // 0s for aggs with symbols as well.
1700 if (numSymbols() == 0)
1701 while (InitializerCount >= 1 && !buffer[InitializerCount - 1])
1702 InitializerCount--;
1703
1704 symbolPosInBuffer.push_back(Elt: InitializerCount);
1705 unsigned int nSym = 0;
1706 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1707 for (unsigned int pos = 0; pos < InitializerCount;) {
1708 if (pos)
1709 os << ", ";
1710 if (pos != nextSymbolPos) {
1711 os << (unsigned int)buffer[pos];
1712 ++pos;
1713 continue;
1714 }
1715 // Generate a per-byte mask() operator for the symbol, which looks like:
1716 // .global .u8 addr[] = {0xFF(foo), 0xFF00(foo), 0xFF0000(foo), ...};
1717 // See https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#initializers
1718 std::string symText;
1719 llvm::raw_string_ostream oss(symText);
1720 printSymbol(nSym, os&: oss);
1721 for (unsigned i = 0; i < ptrSize; ++i) {
1722 if (i)
1723 os << ", ";
1724 llvm::write_hex(S&: os, N: 0xFFULL << i * 8, Style: HexPrintStyle::PrefixUpper);
1725 os << "(" << symText << ")";
1726 }
1727 pos += ptrSize;
1728 nextSymbolPos = symbolPosInBuffer[++nSym];
1729 assert(nextSymbolPos >= pos);
1730 }
1731}
1732
1733void NVPTXAsmPrinter::AggBuffer::printWords(raw_ostream &os) {
1734 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1735 symbolPosInBuffer.push_back(Elt: Size);
1736 unsigned int nSym = 0;
1737 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1738 assert(nextSymbolPos % ptrSize == 0);
1739 for (unsigned int pos = 0; pos < Size; pos += ptrSize) {
1740 if (pos)
1741 os << ", ";
1742 if (pos == nextSymbolPos) {
1743 printSymbol(nSym, os);
1744 nextSymbolPos = symbolPosInBuffer[++nSym];
1745 assert(nextSymbolPos % ptrSize == 0);
1746 assert(nextSymbolPos >= pos + ptrSize);
1747 } else if (ptrSize == 4)
1748 os << support::endian::read32le(P: &buffer[pos]);
1749 else
1750 os << support::endian::read64le(P: &buffer[pos]);
1751 }
1752}
1753
1754void NVPTXAsmPrinter::emitDemotedVars(const Function *F, raw_ostream &O) {
1755 auto It = localDecls.find(x: F);
1756 if (It == localDecls.end())
1757 return;
1758
1759 ArrayRef<const GlobalVariable *> GVars = It->second;
1760
1761 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1762 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1763
1764 for (const GlobalVariable *GV : GVars) {
1765 O << "\t// demoted variable\n\t";
1766 printModuleLevelGV(GVar: GV, O, /*processDemoted=*/ProcessDemoted: true, STI);
1767 }
1768}
1769
1770void NVPTXAsmPrinter::emitPTXAddressSpace(unsigned int AddressSpace,
1771 raw_ostream &O) const {
1772 switch (AddressSpace) {
1773 case ADDRESS_SPACE_LOCAL:
1774 O << "local";
1775 break;
1776 case ADDRESS_SPACE_GLOBAL:
1777 O << "global";
1778 break;
1779 case ADDRESS_SPACE_CONST:
1780 O << "const";
1781 break;
1782 case ADDRESS_SPACE_SHARED:
1783 O << "shared";
1784 break;
1785 default:
1786 report_fatal_error(reason: "Bad address space found while emitting PTX: " +
1787 llvm::Twine(AddressSpace));
1788 break;
1789 }
1790}
1791
1792std::string
1793NVPTXAsmPrinter::getPTXFundamentalTypeStr(Type *Ty, bool useB4PTR) const {
1794 switch (Ty->getTypeID()) {
1795 case Type::IntegerTyID: {
1796 unsigned NumBits = cast<IntegerType>(Val: Ty)->getBitWidth();
1797 if (NumBits == 1)
1798 return "pred";
1799 if (NumBits <= 64) {
1800 std::string name = "u";
1801 return name + utostr(X: NumBits);
1802 }
1803 llvm_unreachable("Integer too large");
1804 break;
1805 }
1806 case Type::BFloatTyID:
1807 case Type::HalfTyID:
1808 // fp16 and bf16 are stored as .b16 for compatibility with pre-sm_53
1809 // PTX assembly.
1810 return "b16";
1811 case Type::FloatTyID:
1812 return "f32";
1813 case Type::DoubleTyID:
1814 return "f64";
1815 case Type::PointerTyID: {
1816 unsigned PtrSize = TM.getPointerSizeInBits(AS: Ty->getPointerAddressSpace());
1817 assert((PtrSize == 64 || PtrSize == 32) && "Unexpected pointer size");
1818
1819 if (PtrSize == 64)
1820 if (useB4PTR)
1821 return "b64";
1822 else
1823 return "u64";
1824 else if (useB4PTR)
1825 return "b32";
1826 else
1827 return "u32";
1828 }
1829 default:
1830 break;
1831 }
1832 llvm_unreachable("unexpected type");
1833}
1834
1835void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
1836 raw_ostream &O,
1837 const NVPTXSubtarget &STI) {
1838 const DataLayout &DL = getDataLayout();
1839
1840 // GlobalVariables are always constant pointers themselves.
1841 Type *ETy = GVar->getValueType();
1842
1843 O << ".";
1844 emitPTXAddressSpace(AddressSpace: GVar->getType()->getAddressSpace(), O);
1845 if (isManaged(*GVar)) {
1846 if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30))
1847 report_fatal_error(
1848 reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1849
1850 O << " .attribute(.managed)";
1851 }
1852 O << " .align "
1853 << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value();
1854
1855 // Special case for i128/fp128
1856 if (ETy->getScalarSizeInBits() == 128) {
1857 O << " .b8 ";
1858 getSymbol(GV: GVar)->print(OS&: O, MAI);
1859 O << "[16]";
1860 return;
1861 }
1862
1863 if (ETy->isFloatingPointTy() || ETy->isIntOrPtrTy()) {
1864 O << " ." << getPTXFundamentalTypeStr(Ty: ETy) << " ";
1865 getSymbol(GV: GVar)->print(OS&: O, MAI);
1866 return;
1867 }
1868
1869 int64_t ElementSize = 0;
1870
1871 // Although PTX has direct support for struct type and array type and LLVM IR
1872 // is very similar to PTX, the LLVM CodeGen does not support for targets that
1873 // support these high level field accesses. Structs and arrays are lowered
1874 // into arrays of bytes.
1875 switch (ETy->getTypeID()) {
1876 case Type::StructTyID:
1877 case Type::ArrayTyID:
1878 case Type::FixedVectorTyID:
1879 ElementSize = DL.getTypeStoreSize(Ty: ETy);
1880 O << " .b8 ";
1881 getSymbol(GV: GVar)->print(OS&: O, MAI);
1882 O << "[";
1883 if (ElementSize) {
1884 O << ElementSize;
1885 }
1886 O << "]";
1887 break;
1888 default:
1889 llvm_unreachable("type not supported yet");
1890 }
1891}
1892
1893void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
1894 const DataLayout &DL = getDataLayout();
1895 const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F);
1896 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
1897 const NVPTXMachineFunctionInfo *MFI =
1898 MF ? MF->getInfo<NVPTXMachineFunctionInfo>() : nullptr;
1899
1900 bool IsFirst = true;
1901 const bool IsKernelFunc = isKernelFunction(F: *F);
1902
1903 // Zero-sized arguments (e.g. empty structs) do not produce a parameter.
1904 // Number the emitted parameters contiguously, skipping the zero-sized ones,
1905 // so that the names match those used in LowerFormalArguments and the
1906 // contiguous numbering used by callers (see LowerCall).
1907 const auto NonEmptyArgs =
1908 make_filter_range(Range: F->args(), Pred: [](const Argument &Arg) {
1909 return !Arg.getType()->isEmptyTy();
1910 });
1911
1912 if (NonEmptyArgs.empty() && !F->isVarArg()) {
1913 O << "()";
1914 return;
1915 }
1916
1917 O << "(\n";
1918
1919 for (const auto &[ParamIndex, Arg] : enumerate(First: NonEmptyArgs)) {
1920 Type *Ty = Arg.getType();
1921 const std::string ParamSym = TLI->getParamName(F, Idx: ParamIndex);
1922
1923 if (!IsFirst)
1924 O << ",\n";
1925
1926 IsFirst = false;
1927
1928 // Handle image/sampler parameters
1929 if (IsKernelFunc) {
1930 const PTXOpaqueType ArgOpaqueType = getPTXOpaqueType(Arg);
1931 if (ArgOpaqueType != PTXOpaqueType::None) {
1932 const bool EmitImgPtr = !MFI || !MFI->checkImageHandleSymbol(Symbol: ParamSym);
1933 O << "\t.param ";
1934 if (EmitImgPtr)
1935 O << ".u64 .ptr ";
1936
1937 switch (ArgOpaqueType) {
1938 case PTXOpaqueType::Sampler:
1939 O << ".samplerref ";
1940 break;
1941 case PTXOpaqueType::Texture:
1942 O << ".texref ";
1943 break;
1944 case PTXOpaqueType::Surface:
1945 O << ".surfref ";
1946 break;
1947 case PTXOpaqueType::None:
1948 llvm_unreachable("handled above");
1949 }
1950 O << ParamSym;
1951 continue;
1952 }
1953 }
1954
1955 if (Arg.hasByValAttr()) {
1956 // param has byVal attribute.
1957 Type *ETy = Arg.getParamByValType();
1958 assert(ETy && "Param should have byval type");
1959
1960 // Print .param .align <a> .b8 .param[size];
1961 // <a> = optimal alignment for the element type; always multiple of
1962 // PAL.getParamAlignment
1963 // size = typeallocsize of element type
1964 const unsigned ParamIdx = Arg.getArgNo() + AttributeList::FirstArgIndex;
1965 const Align OptimalAlign =
1966 IsKernelFunc ? getPTXParamAlign(F, Ty: ETy, AttrIdx: ParamIdx, DL)
1967 : getDeviceByValParamAlign(F, ArgTy: ETy, AttrIdx: ParamIdx, DL);
1968
1969 O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
1970 << "[" << DL.getTypeAllocSize(Ty: ETy) << "]";
1971 continue;
1972 }
1973
1974 if (shouldPassAsArray(Ty)) {
1975 // Just print .param .align <a> .b8 .param[size];
1976 // <a> = optimal alignment for the element type; always multiple of
1977 // PAL.getParamAlignment
1978 // size = typeallocsize of element type
1979 Align OptimalAlign = getPTXParamAlign(
1980 F, Ty, AttrIdx: Arg.getArgNo() + AttributeList::FirstArgIndex, DL);
1981
1982 O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
1983 << "[" << DL.getTypeAllocSize(Ty) << "]";
1984
1985 continue;
1986 }
1987 // Just a scalar
1988 auto *PTy = dyn_cast<PointerType>(Val: Ty);
1989 unsigned PTySizeInBits = 0;
1990 if (PTy) {
1991 PTySizeInBits =
1992 TLI->getPointerTy(DL, AS: PTy->getAddressSpace()).getSizeInBits();
1993 assert(PTySizeInBits && "Invalid pointer size");
1994 }
1995
1996 if (IsKernelFunc) {
1997 if (PTy) {
1998 O << "\t.param .u" << PTySizeInBits << " .ptr";
1999
2000 switch (PTy->getAddressSpace()) {
2001 default:
2002 break;
2003 case ADDRESS_SPACE_GLOBAL:
2004 O << " .global";
2005 break;
2006 case ADDRESS_SPACE_SHARED:
2007 O << " .shared";
2008 break;
2009 case ADDRESS_SPACE_CONST:
2010 O << " .const";
2011 break;
2012 case ADDRESS_SPACE_LOCAL:
2013 O << " .local";
2014 break;
2015 }
2016
2017 O << " .align " << Arg.getParamAlign().valueOrOne().value() << " "
2018 << ParamSym;
2019 continue;
2020 }
2021
2022 // non-pointer scalar to kernel func
2023 O << "\t.param .";
2024 // Special case: predicate operands become .u8 types
2025 if (Ty->isIntegerTy(BitWidth: 1))
2026 O << "u8";
2027 else
2028 O << getPTXFundamentalTypeStr(Ty);
2029 O << " " << ParamSym;
2030 continue;
2031 }
2032 // Non-kernel function, just print .param .b<size> for ABI
2033 // and .reg .b<size> for non-ABI
2034 unsigned Size;
2035 if (auto *ITy = dyn_cast<IntegerType>(Val: Ty)) {
2036 Size = promoteScalarArgumentSize(size: ITy->getBitWidth());
2037 } else if (PTy) {
2038 assert(PTySizeInBits && "Invalid pointer size");
2039 Size = PTySizeInBits;
2040 } else
2041 Size = Ty->getPrimitiveSizeInBits();
2042 O << "\t.param .b" << Size << " " << ParamSym;
2043 }
2044
2045 if (F->isVarArg()) {
2046 if (!IsFirst)
2047 O << ",\n";
2048 O << "\t.param .align " << STI.getMaxRequiredAlignment() << " .b8 "
2049 << TLI->getParamName(F, /* vararg */ Idx: -1) << "[]";
2050 }
2051
2052 O << "\n)";
2053}
2054
2055void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
2056 const MachineFunction &MF) {
2057 auto *TS = getTargetStreamer();
2058
2059 // Emit the Fake Stack Object
2060 const MachineFrameInfo &MFI = MF.getFrameInfo();
2061 if (const int64_t NumBytes = MFI.getStackSize()) {
2062 TS->emitLocalDirective(Alignment: MFI.getMaxAlign(), Name: getFunctionFrameSymbol(),
2063 Size: NumBytes);
2064
2065 // Declare the frame pointers that NVPTXFrameLowering's prologue defines.
2066 const NVPTXRegisterInfo *NRI =
2067 MF.getSubtarget<NVPTXSubtarget>().getRegisterInfo();
2068 for (const Register FrameReg :
2069 {NRI->getFrameRegister(MF), NRI->getFrameLocalRegister(MF)})
2070 TS->emitRegDirective(
2071 SizeInBits: NRI->getRegSizeInBits(Reg: FrameReg, MRI: *MRI).getFixedValue(),
2072 Name: NVPTXInstPrinter::getRegisterName(Reg: FrameReg));
2073 }
2074
2075 // Go through all virtual registers to establish the mapping between the
2076 // global virtual
2077 // register number and the per class virtual register number.
2078 // We use the per class virtual register number in the ptx output.
2079 for (unsigned I : llvm::seq(Size: MRI->getNumVirtRegs())) {
2080 Register VR = Register::index2VirtReg(Index: I);
2081 if (MRI->use_empty(RegNo: VR) && MRI->def_empty(RegNo: VR))
2082 continue;
2083 auto &RCRegMap = VRegMapping[MRI->getRegClass(Reg: VR)];
2084 RCRegMap[VR] = RCRegMap.size() + 1;
2085 }
2086
2087 // Emit declaration of the virtual registers or 'physical' registers for
2088 // each register class
2089 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2090 for (const TargetRegisterClass &RC : TRI->regclasses()) {
2091 // Only declare those registers that may be used.
2092 const auto It = VRegMapping.find(Val: &RC);
2093 if (It == VRegMapping.end() || It->second.empty())
2094 continue;
2095
2096 TS->emitRegDirective(
2097 SizeInBits: TRI->getRegSizeInBits(RC).getFixedValue(),
2098 Name: NVPTX::getVirtualRegisterPrefix(Kind: getVirtualRegisterKind(RC: &RC)),
2099 Count: It->second.size() + 1);
2100 }
2101}
2102
2103/// Translate virtual register numbers in DebugInfo locations to their printed
2104/// encodings, as used by CUDA-GDB.
2105void NVPTXAsmPrinter::encodeDebugInfoRegisterNumbers(
2106 const MachineFunction &MF) {
2107 const NVPTXSubtarget &STI = MF.getSubtarget<NVPTXSubtarget>();
2108 const NVPTXRegisterInfo *NRI = STI.getRegisterInfo();
2109
2110 // Clear the old mapping, and add the new one. This mapping is used after the
2111 // printing of the current function is complete, but before the next function
2112 // is printed.
2113 NRI->clearDebugRegisterMap();
2114
2115 for (const VRegMap &RegMap : make_second_range(c&: VRegMapping))
2116 for (const Register Reg : make_first_range(c: RegMap))
2117 NRI->addToDebugRegisterMap(VirtReg: Reg, RegisterName: getVirtualRegisterName(Reg));
2118}
2119
2120void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp,
2121 raw_ostream &O) const {
2122 APFloat APF = APFloat(Fp->getValueAPF()); // make a copy
2123 bool ignored;
2124 unsigned int numHex;
2125 const char *lead;
2126
2127 if (Fp->getType()->getTypeID() == Type::FloatTyID) {
2128 numHex = 8;
2129 lead = "0f";
2130 APF.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &ignored);
2131 } else if (Fp->getType()->getTypeID() == Type::DoubleTyID) {
2132 numHex = 16;
2133 lead = "0d";
2134 APF.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven, losesInfo: &ignored);
2135 } else
2136 llvm_unreachable("unsupported fp type");
2137
2138 APInt API = APF.bitcastToAPInt();
2139 O << lead << format_hex_no_prefix(N: API.getZExtValue(), Width: numHex, /*Upper=*/true);
2140}
2141
2142void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) {
2143 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2144 O << CI->getValue();
2145 return;
2146 }
2147 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
2148 printFPConstant(Fp: CFP, O);
2149 return;
2150 }
2151 if (isa<ConstantPointerNull>(Val: CPV)) {
2152 O << "0";
2153 return;
2154 }
2155 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
2156 const bool IsNonGenericPointer = GVar->getAddressSpace() != 0;
2157 if (EmitGeneric && !isa<Function>(Val: CPV) && !IsNonGenericPointer) {
2158 O << "generic(";
2159 getSymbol(GV: GVar)->print(OS&: O, MAI);
2160 O << ")";
2161 } else {
2162 getSymbol(GV: GVar)->print(OS&: O, MAI);
2163 }
2164 return;
2165 }
2166 if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2167 const MCExpr *E = lowerConstantForGV(CV: cast<Constant>(Val: Cexpr), ProcessingGeneric: false);
2168 printMCExpr(Expr: *E, OS&: O);
2169 return;
2170 }
2171 llvm_unreachable("Not scalar type found in printScalarConstant()");
2172}
2173
2174void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
2175 AggBuffer *AggBuffer) {
2176 const DataLayout &DL = getDataLayout();
2177 int AllocSize = DL.getTypeAllocSize(Ty: CPV->getType());
2178 if (isa<UndefValue>(Val: CPV) || CPV->isNullValue()) {
2179 // Non-zero Bytes indicates that we need to zero-fill everything. Otherwise,
2180 // only the space allocated by CPV.
2181 AggBuffer->addZeros(Num: Bytes ? Bytes : AllocSize);
2182 return;
2183 }
2184
2185 // Helper for filling AggBuffer with APInts.
2186 auto AddIntToBuffer = [AggBuffer, Bytes](const APInt &Val) {
2187 size_t NumBytes = (Val.getBitWidth() + 7) / 8;
2188 SmallVector<unsigned char, 16> Buf(NumBytes);
2189 // `extractBitsAsZExtValue` does not allow the extraction of bits beyond the
2190 // input's bit width, and i1 arrays may not have a length that is a multuple
2191 // of 8. We handle the last byte separately, so we never request out of
2192 // bounds bits.
2193 for (unsigned I = 0; I < NumBytes - 1; ++I) {
2194 Buf[I] = Val.extractBitsAsZExtValue(numBits: 8, bitPosition: I * 8);
2195 }
2196 size_t LastBytePosition = (NumBytes - 1) * 8;
2197 size_t LastByteBits = Val.getBitWidth() - LastBytePosition;
2198 Buf[NumBytes - 1] =
2199 Val.extractBitsAsZExtValue(numBits: LastByteBits, bitPosition: LastBytePosition);
2200 AggBuffer->addBytes(Ptr: Buf.data(), Num: NumBytes, Bytes);
2201 };
2202
2203 switch (CPV->getType()->getTypeID()) {
2204 case Type::IntegerTyID:
2205 if (const auto *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2206 AddIntToBuffer(CI->getValue());
2207 break;
2208 }
2209 if (const auto *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2210 if (const auto *CI =
2211 dyn_cast<ConstantInt>(Val: ConstantFoldConstant(C: Cexpr, DL))) {
2212 AddIntToBuffer(CI->getValue());
2213 break;
2214 }
2215 if (Cexpr->getOpcode() == Instruction::PtrToInt) {
2216 Value *V = Cexpr->getOperand(i_nocapture: 0)->stripPointerCasts();
2217 AggBuffer->addSymbol(GVar: V, GVarBeforeStripping: Cexpr->getOperand(i_nocapture: 0));
2218 AggBuffer->addZeros(Num: AllocSize);
2219 break;
2220 }
2221 // A symbol-relative integer whose offset is applied outside the
2222 // ptrtoint, e.g. add(ptrtoint(@g), C). It can't fold to a ConstantInt
2223 // because it references a symbol; emit it through lowerConstantForGV, the
2224 // same path scalar symbol-relative integer globals use.
2225 AggBuffer->addSymbol(GVar: Cexpr, GVarBeforeStripping: Cexpr);
2226 AggBuffer->addZeros(Num: AllocSize);
2227 break;
2228 }
2229 llvm_unreachable("unsupported integer const type");
2230 break;
2231
2232 case Type::HalfTyID:
2233 case Type::BFloatTyID:
2234 case Type::FloatTyID:
2235 case Type::DoubleTyID:
2236 case Type::FP128TyID:
2237 AddIntToBuffer(cast<ConstantFP>(Val: CPV)->getValueAPF().bitcastToAPInt());
2238 break;
2239
2240 case Type::PointerTyID: {
2241 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
2242 AggBuffer->addSymbol(GVar, GVarBeforeStripping: GVar);
2243 } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2244 const Value *v = Cexpr->stripPointerCasts();
2245 AggBuffer->addSymbol(GVar: v, GVarBeforeStripping: Cexpr);
2246 }
2247 AggBuffer->addZeros(Num: AllocSize);
2248 break;
2249 }
2250
2251 case Type::ArrayTyID:
2252 case Type::FixedVectorTyID:
2253 case Type::StructTyID: {
2254 if (isa<ConstantAggregate>(Val: CPV) || isa<ConstantDataSequential>(Val: CPV)) {
2255 // bufferAggregateConstant doesn't emit tail-padding, i.e. it writes
2256 // `store_size` bytes, not `alloc_size` bytes. Do it ourselves here.
2257 unsigned StartPos = AggBuffer->getCurpos();
2258 bufferAggregateConstant(CV: CPV, aggBuffer: AggBuffer);
2259 unsigned Written = AggBuffer->getCurpos() - StartPos;
2260 unsigned SlotSize = std::max<int>(a: Bytes, b: AllocSize);
2261 if (SlotSize > Written)
2262 AggBuffer->addZeros(Num: SlotSize - Written);
2263 } else if (isa<ConstantAggregateZero>(Val: CPV))
2264 AggBuffer->addZeros(Num: Bytes);
2265 else
2266 llvm_unreachable("Unexpected Constant type");
2267 break;
2268 }
2269
2270 default:
2271 llvm_unreachable("unsupported type");
2272 }
2273}
2274
2275void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
2276 AggBuffer *aggBuffer) {
2277 const DataLayout &DL = getDataLayout();
2278
2279 auto ExtendBuffer = [](APInt Val, AggBuffer *Buffer) {
2280 unsigned NumBytes = divideCeil(Numerator: Val.getBitWidth(), Denominator: 8);
2281 for (unsigned I : llvm::seq(Size: NumBytes)) {
2282 unsigned NumBits = std::min(a: 8u, b: Val.getBitWidth() - I * 8);
2283 Buffer->addByte(Byte: Val.extractBitsAsZExtValue(numBits: NumBits, bitPosition: I * 8));
2284 }
2285 };
2286
2287 // Integer or floating point vector splats.
2288 if (isa<ConstantInt, ConstantFP>(Val: CPV)) {
2289 if (auto *VTy = dyn_cast<FixedVectorType>(Val: CPV->getType())) {
2290 for (unsigned I : llvm::seq(Size: VTy->getNumElements()))
2291 bufferLEByte(CPV: CPV->getAggregateElement(Elt: I), Bytes: 0, AggBuffer: aggBuffer);
2292 return;
2293 }
2294 }
2295
2296 // Integers of arbitrary width
2297 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2298 assert(CI->getType()->isIntegerTy() && "Expected integer constant!");
2299 ExtendBuffer(CI->getValue(), aggBuffer);
2300 return;
2301 }
2302
2303 // f128
2304 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
2305 assert(CFP->getType()->isFloatingPointTy() && "Expected fp constant!");
2306 if (CFP->getType()->isFP128Ty()) {
2307 ExtendBuffer(CFP->getValueAPF().bitcastToAPInt(), aggBuffer);
2308 return;
2309 }
2310 }
2311
2312 // Buffer arrays one element at a time.
2313 if (isa<ConstantArray>(Val: CPV)) {
2314 for (const auto &Op : CPV->operands())
2315 bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer);
2316 return;
2317 }
2318
2319 // Constant vectors
2320 if (const auto *CVec = dyn_cast<ConstantVector>(Val: CPV)) {
2321 bufferAggregateConstVec(CV: CVec, aggBuffer);
2322 return;
2323 }
2324
2325 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: CPV)) {
2326 for (unsigned I : llvm::seq(Size: CDS->getNumElements()))
2327 bufferLEByte(CPV: cast<Constant>(Val: CDS->getElementAsConstant(i: I)), Bytes: 0, AggBuffer: aggBuffer);
2328 return;
2329 }
2330
2331 if (isa<ConstantStruct>(Val: CPV)) {
2332 if (CPV->getNumOperands()) {
2333 StructType *ST = cast<StructType>(Val: CPV->getType());
2334 for (unsigned I : llvm::seq(Size: CPV->getNumOperands())) {
2335 int EndOffset = (I + 1 == CPV->getNumOperands())
2336 ? DL.getStructLayout(Ty: ST)->getElementOffset(Idx: 0) +
2337 DL.getTypeAllocSize(Ty: ST)
2338 : DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I + 1);
2339 int Bytes = EndOffset - DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I);
2340 bufferLEByte(CPV: cast<Constant>(Val: CPV->getOperand(i: I)), Bytes, AggBuffer: aggBuffer);
2341 }
2342 }
2343 return;
2344 }
2345 llvm_unreachable("unsupported constant type in printAggregateConstant()");
2346}
2347
2348void NVPTXAsmPrinter::bufferAggregateConstVec(const ConstantVector *CV,
2349 AggBuffer *aggBuffer) {
2350 unsigned NumElems = CV->getType()->getNumElements();
2351 const unsigned BuffSize = aggBuffer->getBufferSize();
2352
2353 // Buffer one element at a time if we have allocated enough buffer space.
2354 if (BuffSize >= NumElems) {
2355 for (const auto &Op : CV->operands())
2356 bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer);
2357 return;
2358 }
2359
2360 // Sub-byte datatypes will have more elements than bytes allocated for the
2361 // buffer. Merge consecutive elements to form a full byte. We expect that 8 %
2362 // sub-byte-elem-size should be 0 and current expected usage is for i4 (for
2363 // e2m1-fp4 types).
2364 Type *ElemTy = CV->getType()->getElementType();
2365 assert(ElemTy->isIntegerTy() && "Expected integer data type.");
2366 unsigned ElemTySize = ElemTy->getPrimitiveSizeInBits();
2367 assert(ElemTySize < 8 && "Expected sub-byte data type.");
2368 assert(8 % ElemTySize == 0 && "Element type size must evenly divide a byte.");
2369 // Number of elements to merge to form a full byte.
2370 unsigned NumElemsPerByte = 8 / ElemTySize;
2371 unsigned NumCompleteBytes = NumElems / NumElemsPerByte;
2372 unsigned NumTailElems = NumElems % NumElemsPerByte;
2373
2374 // Helper lambda to constant-fold sub-vector of sub-byte type elements into
2375 // i8. Start and end indices of the sub-vector is provided, along with number
2376 // of padding zeros if required.
2377 auto ConvertSubCVtoInt8 = [this, &ElemTy](const ConstantVector *CV,
2378 unsigned Start, unsigned End,
2379 unsigned NumPaddingZeros = 0) {
2380 // Collect elements to create sub-vector.
2381 SmallVector<Constant *, 8> SubCVElems;
2382 for (unsigned I : llvm::seq(Begin: Start, End))
2383 SubCVElems.push_back(Elt: CV->getAggregateElement(Elt: I));
2384
2385 // Optionally pad with zeros.
2386 if (NumPaddingZeros)
2387 SubCVElems.append(NumInputs: NumPaddingZeros, Elt: ConstantInt::getNullValue(Ty: ElemTy));
2388
2389 auto SubCV = ConstantVector::get(V: SubCVElems);
2390 Type *Int8Ty = IntegerType::get(C&: SubCV->getContext(), NumBits: 8);
2391
2392 // Merge elements of the sub-vector using ConstantFolding.
2393 ConstantInt *MergedElem =
2394 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldConstant(
2395 C: ConstantExpr::getBitCast(C: const_cast<Constant *>(SubCV), Ty: Int8Ty),
2396 DL: getDataLayout()));
2397
2398 if (!MergedElem)
2399 report_fatal_error(
2400 reason: "Cannot lower vector global with unusual element type");
2401
2402 return MergedElem;
2403 };
2404
2405 // Iterate through elements of vector one chunk at a time and buffer that
2406 // chunk.
2407 for (unsigned ByteIdx : llvm::seq(Size: NumCompleteBytes))
2408 bufferLEByte(CPV: ConvertSubCVtoInt8(CV, ByteIdx * NumElemsPerByte,
2409 (ByteIdx + 1) * NumElemsPerByte),
2410 Bytes: 0, AggBuffer: aggBuffer);
2411
2412 // For unevenly sized vectors add tail padding zeros.
2413 if (NumTailElems > 0)
2414 bufferLEByte(CPV: ConvertSubCVtoInt8(CV, NumElems - NumTailElems, NumElems,
2415 NumElemsPerByte - NumTailElems),
2416 Bytes: 0, AggBuffer: aggBuffer);
2417}
2418
2419/// lowerConstantForGV - Return an MCExpr for the given Constant. This is mostly
2420/// a copy from AsmPrinter::lowerConstant, except customized to only handle
2421/// expressions that are representable in PTX and create
2422/// NVPTXGenericMCSymbolRefExpr nodes for addrspacecast instructions.
2423const MCExpr *
2424NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV,
2425 bool ProcessingGeneric) const {
2426 MCContext &Ctx = OutContext;
2427
2428 if (CV->isNullValue() || isa<UndefValue>(Val: CV))
2429 return MCConstantExpr::create(Value: 0, Ctx);
2430
2431 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CV))
2432 return MCConstantExpr::create(Value: CI->getZExtValue(), Ctx);
2433
2434 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
2435 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Symbol: getSymbol(GV), Ctx);
2436 if (ProcessingGeneric)
2437 return NVPTXGenericMCSymbolRefExpr::create(SymExpr: Expr, Ctx);
2438 return Expr;
2439 }
2440
2441 const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: CV);
2442 if (!CE) {
2443 llvm_unreachable("Unknown constant value to lower!");
2444 }
2445
2446 switch (CE->getOpcode()) {
2447 default:
2448 break; // Error
2449
2450 case Instruction::AddrSpaceCast: {
2451 // Strip the addrspacecast and pass along the operand
2452 PointerType *DstTy = cast<PointerType>(Val: CE->getType());
2453 if (DstTy->getAddressSpace() == 0)
2454 return lowerConstantForGV(CV: cast<const Constant>(Val: CE->getOperand(i_nocapture: 0)), ProcessingGeneric: true);
2455
2456 break; // Error
2457 }
2458
2459 case Instruction::GetElementPtr: {
2460 const DataLayout &DL = getDataLayout();
2461
2462 // Generate a symbolic expression for the byte address
2463 APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0);
2464 cast<GEPOperator>(Val: CE)->accumulateConstantOffset(DL, Offset&: OffsetAI);
2465
2466 const MCExpr *Base = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0),
2467 ProcessingGeneric);
2468 if (!OffsetAI)
2469 return Base;
2470
2471 int64_t Offset = OffsetAI.getSExtValue();
2472 return MCBinaryExpr::createAdd(LHS: Base, RHS: MCConstantExpr::create(Value: Offset, Ctx),
2473 Ctx);
2474 }
2475
2476 case Instruction::Trunc:
2477 // We emit the value and depend on the assembler to truncate the generated
2478 // expression properly. This is important for differences between
2479 // blockaddress labels. Since the two labels are in the same function, it
2480 // is reasonable to treat their delta as a 32-bit value.
2481 [[fallthrough]];
2482 case Instruction::BitCast:
2483 return lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2484
2485 case Instruction::IntToPtr: {
2486 const DataLayout &DL = getDataLayout();
2487
2488 // Handle casts to pointers by changing them into casts to the appropriate
2489 // integer type. This promotes constant folding and simplifies this code.
2490 Constant *Op = CE->getOperand(i_nocapture: 0);
2491 Op = ConstantFoldIntegerCast(C: Op, DestTy: DL.getIntPtrType(CV->getType()),
2492 /*IsSigned*/ false, DL);
2493 if (Op)
2494 return lowerConstantForGV(CV: Op, ProcessingGeneric);
2495
2496 break; // Error
2497 }
2498
2499 case Instruction::PtrToInt: {
2500 const DataLayout &DL = getDataLayout();
2501
2502 // Support only foldable casts to/from pointers that can be eliminated by
2503 // changing the pointer to the appropriately sized integer type.
2504 Constant *Op = CE->getOperand(i_nocapture: 0);
2505 Type *Ty = CE->getType();
2506
2507 const MCExpr *OpExpr = lowerConstantForGV(CV: Op, ProcessingGeneric);
2508
2509 // We can emit the pointer value into this slot if the slot is an
2510 // integer slot equal to the size of the pointer.
2511 if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Ty: Op->getType()))
2512 return OpExpr;
2513
2514 // Otherwise the pointer is smaller than the resultant integer, mask off
2515 // the high bits so we are sure to get a proper truncation if the input is
2516 // a constant expr.
2517 unsigned InBits = DL.getTypeAllocSizeInBits(Ty: Op->getType());
2518 const MCExpr *MaskExpr = MCConstantExpr::create(Value: ~0ULL >> (64-InBits), Ctx);
2519 return MCBinaryExpr::createAnd(LHS: OpExpr, RHS: MaskExpr, Ctx);
2520 }
2521
2522 // The MC library also has a right-shift operator, but it isn't consistently
2523 // signed or unsigned between different targets.
2524 case Instruction::Add: {
2525 const MCExpr *LHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2526 const MCExpr *RHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 1), ProcessingGeneric);
2527 switch (CE->getOpcode()) {
2528 default: llvm_unreachable("Unknown binary operator constant cast expr");
2529 case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx);
2530 }
2531 }
2532 }
2533
2534 // If the code isn't optimized, there may be outstanding folding
2535 // opportunities. Attempt to fold the expression using DataLayout as a
2536 // last resort before giving up.
2537 Constant *C = ConstantFoldConstant(C: CE, DL: getDataLayout());
2538 if (C != CE)
2539 return lowerConstantForGV(CV: C, ProcessingGeneric);
2540
2541 // Otherwise report the problem to the user.
2542 std::string S;
2543 raw_string_ostream OS(S);
2544 OS << "Unsupported expression in static initializer: ";
2545 CE->printAsOperand(O&: OS, /*PrintType=*/false,
2546 M: !MF ? nullptr : MF->getFunction().getParent());
2547 report_fatal_error(reason: Twine(OS.str()));
2548}
2549
2550void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) const {
2551 OutContext.getAsmInfo().printExpr(OS, Expr);
2552}
2553
2554/// PrintAsmOperand - Print out an operand for an inline asm expression.
2555///
2556bool NVPTXAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
2557 const char *ExtraCode, raw_ostream &O) {
2558 if (ExtraCode && ExtraCode[0]) {
2559 if (ExtraCode[1] != 0)
2560 return true; // Unknown modifier.
2561
2562 switch (ExtraCode[0]) {
2563 default:
2564 // See if this is a generic print operand
2565 return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O);
2566 case 'r':
2567 break;
2568 }
2569 }
2570
2571 printOperand(MI, OpNum: OpNo, O);
2572
2573 return false;
2574}
2575
2576bool NVPTXAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
2577 unsigned OpNo,
2578 const char *ExtraCode,
2579 raw_ostream &O) {
2580 if (ExtraCode && ExtraCode[0])
2581 return true; // Unknown modifier
2582
2583 O << '[';
2584 printMemOperand(MI, OpNum: OpNo, O);
2585 O << ']';
2586
2587 return false;
2588}
2589
2590void NVPTXAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
2591 raw_ostream &O) {
2592 const MachineOperand &MO = MI->getOperand(i: OpNum);
2593 switch (MO.getType()) {
2594 case MachineOperand::MO_Register:
2595 if (MO.getReg().isPhysical()) {
2596 if (MO.getReg() == NVPTX::VRDepot)
2597 getFunctionFrameSymbol()->print(OS&: O, MAI);
2598 else
2599 O << NVPTXInstPrinter::getRegisterName(Reg: MO.getReg());
2600 } else {
2601 O << getVirtualRegisterName(Reg: MO.getReg());
2602 }
2603 break;
2604
2605 case MachineOperand::MO_Immediate:
2606 O << MO.getImm();
2607 break;
2608
2609 case MachineOperand::MO_FPImmediate:
2610 printFPConstant(Fp: MO.getFPImm(), O);
2611 break;
2612
2613 case MachineOperand::MO_GlobalAddress:
2614 PrintSymbolOperand(MO, OS&: O);
2615 break;
2616
2617 case MachineOperand::MO_MachineBasicBlock:
2618 MO.getMBB()->getSymbol()->print(OS&: O, MAI);
2619 break;
2620
2621 default:
2622 llvm_unreachable("Operand type not supported.");
2623 }
2624}
2625
2626void NVPTXAsmPrinter::printMemOperand(const MachineInstr *MI, unsigned OpNum,
2627 raw_ostream &O, const char *Modifier) {
2628 printOperand(MI, OpNum, O);
2629
2630 if (Modifier && strcmp(s1: Modifier, s2: "add") == 0) {
2631 O << ", ";
2632 printOperand(MI, OpNum: OpNum + 1, O);
2633 } else {
2634 if (MI->getOperand(i: OpNum + 1).isImm() &&
2635 MI->getOperand(i: OpNum + 1).getImm() == 0)
2636 return; // don't print ',0' or '+0'
2637 O << "+";
2638 printOperand(MI, OpNum: OpNum + 1, O);
2639 }
2640}
2641
2642/// Returns true if \p Line begins with an alphabetic character or underscore,
2643/// indicating it is a PTX instruction that should receive a .loc directive.
2644static bool isPTXInstruction(StringRef Line) {
2645 StringRef Trimmed = Line.ltrim();
2646 return !Trimmed.empty() &&
2647 (std::isalpha(static_cast<unsigned char>(Trimmed[0])) ||
2648 Trimmed[0] == '_');
2649}
2650
2651/// Returns the DILocation for an inline asm MachineInstr if debug line info
2652/// should be emitted, or nullptr otherwise.
2653static const DILocation *getInlineAsmDebugLoc(const MachineInstr *MI) {
2654 if (!MI || !MI->getDebugLoc())
2655 return nullptr;
2656 const DISubprogram *SP = MI->getMF()->getFunction().getSubprogram();
2657 if (!SP || SP->getUnit()->getEmissionKind() == DICompileUnit::NoDebug)
2658 return nullptr;
2659 const DILocation *DL = MI->getDebugLoc();
2660 if (!DL->getFile() || !DL->getLine())
2661 return nullptr;
2662 return DL;
2663}
2664
2665namespace {
2666struct InlineAsmInliningContext {
2667 MCSymbol *FuncNameSym = nullptr;
2668 unsigned FileIA = 0;
2669 unsigned LineIA = 0;
2670 unsigned ColIA = 0;
2671
2672 bool hasInlinedAt() const { return FuncNameSym != nullptr; }
2673};
2674} // namespace
2675
2676/// Resolves the enhanced-lineinfo inlining context for an inline asm debug
2677/// location. Returns a default (empty) context if inlining info is unavailable.
2678static InlineAsmInliningContext
2679getInlineAsmInliningContext(const DILocation *DL, const MachineFunction &MF,
2680 NVPTXDwarfDebug *NVDD, MCStreamer &Streamer,
2681 unsigned CUID) {
2682 InlineAsmInliningContext Ctx;
2683 const DILocation *InlinedAt = DL->getInlinedAt();
2684 if (!InlinedAt || !InlinedAt->getFile() || !NVDD ||
2685 !NVDD->isEnhancedLineinfo(MF))
2686 return Ctx;
2687 const auto *SubProg = getDISubprogram(Scope: DL->getScope());
2688 if (!SubProg)
2689 return Ctx;
2690 Ctx.FuncNameSym = NVDD->getOrCreateFuncNameSymbol(LinkageName: SubProg->getLinkageName());
2691 Ctx.FileIA = Streamer.emitDwarfFileDirective(
2692 FileNo: 0, Directory: InlinedAt->getFile()->getDirectory(),
2693 Filename: InlinedAt->getFile()->getFilename(), Checksum: std::nullopt, Source: std::nullopt, CUID);
2694 Ctx.LineIA = InlinedAt->getLine();
2695 Ctx.ColIA = InlinedAt->getColumn();
2696 return Ctx;
2697}
2698
2699void NVPTXAsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
2700 const MCTargetOptions &MCOptions,
2701 const MDNode *LocMDNode,
2702 InlineAsm::AsmDialect Dialect,
2703 const MachineInstr *MI) {
2704 assert(!Str.empty() && "Can't emit empty inline asm block");
2705 if (Str.back() == 0)
2706 Str = Str.substr(Start: 0, N: Str.size() - 1);
2707
2708 auto emitAsmStr = [&](StringRef AsmStr) {
2709 emitInlineAsmStart();
2710 OutStreamer->emitRawText(String: AsmStr);
2711 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2712 };
2713
2714 const DILocation *DL = getInlineAsmDebugLoc(MI);
2715 if (!DL) {
2716 emitAsmStr(Str);
2717 return;
2718 }
2719
2720 const DIFile *File = DL->getFile();
2721 unsigned Line = DL->getLine();
2722 const unsigned Column = DL->getColumn();
2723 const unsigned CUID = OutStreamer->getContext().getDwarfCompileUnitID();
2724 const unsigned FileNumber = OutStreamer->emitDwarfFileDirective(
2725 FileNo: 0, Directory: File->getDirectory(), Filename: File->getFilename(), Checksum: std::nullopt, Source: std::nullopt,
2726 CUID);
2727
2728 auto *NVDD = static_cast<NVPTXDwarfDebug *>(getDwarfDebug());
2729 InlineAsmInliningContext InlineCtx =
2730 getInlineAsmInliningContext(DL, MF: *MI->getMF(), NVDD, Streamer&: *OutStreamer, CUID);
2731
2732 SmallVector<StringRef, 16> Lines;
2733 Str.split(A&: Lines, Separator: '\n');
2734 emitInlineAsmStart();
2735 for (const StringRef &L : Lines) {
2736 StringRef RTrimmed = L.rtrim(Char: '\r');
2737 if (isPTXInstruction(Line: L)) {
2738 if (InlineCtx.hasInlinedAt()) {
2739 OutStreamer->emitDwarfLocDirectiveWithInlinedAt(
2740 FileNo: FileNumber, Line, Column, FileIA: InlineCtx.FileIA, LineIA: InlineCtx.LineIA,
2741 ColumnIA: InlineCtx.ColIA, Sym: InlineCtx.FuncNameSym, DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2742 FileName: File->getFilename());
2743 } else {
2744 OutStreamer->emitDwarfLocDirective(FileNo: FileNumber, Line, Column,
2745 DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2746 FileName: File->getFilename());
2747 }
2748 }
2749 OutStreamer->emitRawText(String: RTrimmed);
2750 ++Line;
2751 }
2752 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2753}
2754
2755char NVPTXAsmPrinter::ID = 0;
2756
2757INITIALIZE_PASS(NVPTXAsmPrinter, "nvptx-asm-printer", "NVPTX Assembly Printer",
2758 false, false)
2759
2760// Force static initialization.
2761extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
2762LLVMInitializeNVPTXAsmPrinter() {
2763 RegisterAsmPrinter<NVPTXAsmPrinter> X(getTheNVPTXTarget32());
2764 RegisterAsmPrinter<NVPTXAsmPrinter> Y(getTheNVPTXTarget64());
2765}
2766
2767PreservedAnalyses NVPTXAsmPrinterBeginPass::run(Module &M,
2768 ModuleAnalysisManager &MAM) {
2769 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2770 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2771 Printer.doInitialization(M);
2772 return PreservedAnalyses::all();
2773}
2774
2775PreservedAnalyses
2776NVPTXAsmPrinterPass::run(MachineFunction &MF,
2777 MachineFunctionAnalysisManager &MFAM) {
2778 AsmPrinter &Printer =
2779 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2780 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2781 ->getPrinter();
2782 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter&: Printer);
2783 Printer.runOnMachineFunction(MF);
2784 return PreservedAnalyses::all();
2785}
2786
2787PreservedAnalyses NVPTXAsmPrinterEndPass::run(Module &M,
2788 ModuleAnalysisManager &MAM) {
2789 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2790 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2791 Printer.doFinalization(M);
2792 return PreservedAnalyses::all();
2793}
2794