1//===-- NVPTXAsmPrinter.cpp - NVPTX LLVM assembly writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to NVPTX assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXAsmPrinter.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "MCTargetDesc/NVPTXInstPrinter.h"
17#include "MCTargetDesc/NVPTXTargetStreamer.h"
18#include "NVPTX.h"
19#include "NVPTXDwarfDebug.h"
20#include "NVPTXMCExpr.h"
21#include "NVPTXMachineFunctionInfo.h"
22#include "NVPTXRegisterInfo.h"
23#include "NVPTXSubtarget.h"
24#include "NVPTXTargetMachine.h"
25#include "NVPTXUtilities.h"
26#include "NVVMProperties.h"
27#include "TargetInfo/NVPTXTargetInfo.h"
28#include "cl_common_defines.h"
29#include "llvm/ADT/APFloat.h"
30#include "llvm/ADT/APInt.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/DenseSet.h"
34#include "llvm/ADT/SCCIterator.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/Sequence.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/SmallString.h"
39#include "llvm/ADT/SmallVector.h"
40#include "llvm/ADT/StringExtras.h"
41#include "llvm/ADT/StringRef.h"
42#include "llvm/ADT/Twine.h"
43#include "llvm/ADT/iterator_range.h"
44#include "llvm/Analysis/ConstantFolding.h"
45#include "llvm/CodeGen/Analysis.h"
46#include "llvm/CodeGen/AsmPrinter.h"
47#include "llvm/CodeGen/AsmPrinterAnalysis.h"
48#include "llvm/CodeGen/MachineBasicBlock.h"
49#include "llvm/CodeGen/MachineFrameInfo.h"
50#include "llvm/CodeGen/MachineFunction.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineJumpTableInfo.h"
53#include "llvm/CodeGen/MachineLoopInfo.h"
54#include "llvm/CodeGen/MachineModuleInfo.h"
55#include "llvm/CodeGen/MachineOperand.h"
56#include "llvm/CodeGen/MachineRegisterInfo.h"
57#include "llvm/CodeGen/TargetRegisterInfo.h"
58#include "llvm/CodeGen/ValueTypes.h"
59#include "llvm/CodeGenTypes/MachineValueType.h"
60#include "llvm/IR/Argument.h"
61#include "llvm/IR/Attributes.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DebugLoc.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/Function.h"
71#include "llvm/IR/GlobalAlias.h"
72#include "llvm/IR/GlobalValue.h"
73#include "llvm/IR/GlobalVariable.h"
74#include "llvm/IR/InstrTypes.h"
75#include "llvm/IR/Instruction.h"
76#include "llvm/IR/LLVMContext.h"
77#include "llvm/IR/Module.h"
78#include "llvm/IR/Operator.h"
79#include "llvm/IR/Type.h"
80#include "llvm/IR/User.h"
81#include "llvm/IR/Value.h"
82#include "llvm/MC/MCExpr.h"
83#include "llvm/MC/MCInst.h"
84#include "llvm/MC/MCInstrDesc.h"
85#include "llvm/MC/MCStreamer.h"
86#include "llvm/MC/MCSymbol.h"
87#include "llvm/MC/TargetRegistry.h"
88#include "llvm/Pass.h"
89#include "llvm/Support/Alignment.h"
90#include "llvm/Support/Casting.h"
91#include "llvm/Support/Compiler.h"
92#include "llvm/Support/Endian.h"
93#include "llvm/Support/ErrorHandling.h"
94#include "llvm/Support/NativeFormatting.h"
95#include "llvm/Support/raw_ostream.h"
96#include "llvm/Target/TargetLoweringObjectFile.h"
97#include "llvm/Target/TargetMachine.h"
98#include "llvm/Transforms/Utils/UnrollLoop.h"
99#include <algorithm>
100#include <cassert>
101#include <cstdint>
102#include <cstring>
103#include <map>
104#include <memory>
105#include <set>
106#include <string>
107#include <vector>
108
109using namespace llvm;
110
111#define DEPOTNAME "__local_depot"
112
113// The ptx syntax and format is very different from that usually seem in a .s
114// file,
115// therefore we are not able to use the MCAsmStreamer interface here.
116//
117// We are handcrafting the output method here.
118//
119// A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer
120// (subclass of MCStreamer).
121
122namespace {
123
124class NVPTXAsmPrinter : public AsmPrinter {
125
126 class AggBuffer {
127 // Used to buffer the emitted string for initializing global aggregates.
128 //
129 // Normally an aggregate (array, vector, or structure) is emitted as a u8[].
130 // However, if either element/field of the aggregate is a non-NULL address,
131 // and all such addresses are properly aligned, then the aggregate is
132 // emitted as u32[] or u64[]. In the case of unaligned addresses, the
133 // aggregate is emitted as u8[], and the mask() operator is used for all
134 // pointers.
135 //
136 // We first layout the aggregate in 'buffer' in bytes, except for those
137 // symbol addresses. For the i-th symbol address in the aggregate, its
138 // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s.
139 // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1]
140 // records the Value*.
141 //
142 // Once we have this AggBuffer setup, we can choose how to print it out.
143 public:
144 // number of symbol addresses
145 unsigned numSymbols() const { return Symbols.size(); }
146
147 bool allSymbolsAligned(unsigned ptrSize) const {
148 return llvm::all_of(Range: symbolPosInBuffer,
149 P: [=](unsigned pos) { return pos % ptrSize == 0; });
150 }
151
152 private:
153 const unsigned Size; // size of the buffer in bytes
154 std::vector<unsigned char> buffer; // the buffer
155 SmallVector<unsigned, 4> symbolPosInBuffer;
156 SmallVector<const Value *, 4> Symbols;
157 // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
158 // stripping pointer casts, i.e.,
159 // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
160 //
161 // We need to keep these values because AggBuffer::print decides whether to
162 // emit a "generic()" cast for Symbols[i] depending on the address space of
163 // SymbolsBeforeStripping[i].
164 SmallVector<const Value *, 4> SymbolsBeforeStripping;
165 unsigned curpos;
166 const NVPTXAsmPrinter &AP;
167 const bool EmitGeneric;
168
169 public:
170 AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP)
171 : Size(Size), buffer(Size), curpos(0), AP(AP),
172 EmitGeneric(AP.EmitGeneric) {}
173
174 // Number of bytes written so far.
175 unsigned getCurpos() const { return curpos; }
176
177 // Copy Num bytes from Ptr.
178 // if Bytes > Num, zero fill up to Bytes.
179 void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
180 for (unsigned I : llvm::seq(Size: Num))
181 addByte(Byte: Ptr[I]);
182 if (Bytes > Num)
183 addZeros(Num: Bytes - Num);
184 }
185
186 void addByte(uint8_t Byte) {
187 assert(curpos < Size);
188 buffer[curpos] = Byte;
189 curpos++;
190 }
191
192 void addZeros(unsigned Num) {
193 for ([[maybe_unused]] unsigned _ : llvm::seq(Size: Num)) {
194 addByte(Byte: 0);
195 }
196 }
197
198 void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
199 symbolPosInBuffer.push_back(Elt: curpos);
200 Symbols.push_back(Elt: GVar);
201 SymbolsBeforeStripping.push_back(Elt: GVarBeforeStripping);
202 }
203
204 void printBytes(raw_ostream &os);
205 void printWords(raw_ostream &os);
206
207 private:
208 void printSymbol(unsigned nSym, raw_ostream &os);
209 };
210
211 friend class AggBuffer;
212
213public:
214 static char ID;
215
216 StringRef getPassName() const override { return "NVPTX Assembly Printer"; }
217
218private:
219 const Function *F;
220
221 NVPTXTargetStreamer *getTargetStreamer() const;
222
223 void emitStartOfAsmFile(Module &M) override;
224 void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
225 void emitFunctionEntryLabel() override;
226 void emitFunctionBodyStart() override;
227 void emitFunctionBodyEnd() override;
228 void emitImplicitDef(const MachineInstr *MI) const override;
229
230 void emitInstruction(const MachineInstr *) override;
231 void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
232 MCOperand lowerOperand(const MachineOperand &MO);
233 MCOperand GetSymbolRef(const MCSymbol *Symbol);
234 MCRegister encodeVirtualRegister(Register Reg);
235
236 /// The number \p Reg was assigned within its register class, as declared by
237 /// this function's .reg directives.
238 unsigned getVirtualRegisterNumber(Register Reg) const;
239
240 void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O,
241 const char *Modifier = nullptr);
242 void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O,
243 bool processDemoted, const NVPTXSubtarget &STI);
244 // PTX permits these directives only between a .func directive and its body,
245 // so the caller must not use this for a kernel.
246 template <typename T>
247 void printFunctionDirectives(const T &V, raw_ostream &O) const;
248
249 void emitGlobals(const Module &M);
250 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
251 void emitHeader(Module &M, const NVPTXSubtarget &STI);
252 void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const;
253 void emitFunctionParamList(const Function *, raw_ostream &O);
254 void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
255 void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
256 void emitCallPrototype(const CallBase &CB, MCSymbol *PrototypeSymbol) const;
257 void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const;
258
259 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
260 const char *ExtraCode, raw_ostream &) override;
261 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
262 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
263 const char *ExtraCode, raw_ostream &) override;
264
265 const MCExpr *lowerConstantForGV(const Constant *CV,
266 bool ProcessingGeneric) const;
267 void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const;
268 /// Emit a blob of inline asm to the output streamer.
269 void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
270 const MCTargetOptions &MCOptions, const MDNode *LocMDNode,
271 InlineAsm::AsmDialect Dialect,
272 const MachineInstr *MI) override;
273
274protected:
275 bool doInitialization(Module &M) override;
276 bool doFinalization(Module &M) override;
277
278 /// Create NVPTX-specific DwarfDebug handler.
279 DwarfDebug *createDwarfDebug() override;
280
281private:
282 bool GlobalsEmitted;
283
284 // This is specific per MachineFunction.
285 const MachineRegisterInfo *MRI;
286
287 // The number assigned to each virtual register within its class, populated
288 // by setAndEmitFunctionVirtualRegisters and cleared between functions.
289 using VRegMap = DenseMap<Register, unsigned>;
290 using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>;
291 VRegRCMap VRegMapping;
292
293 // List of variables demoted to a function scope.
294 std::map<const Function *, std::vector<const GlobalVariable *>> localDecls;
295
296 /// Print the state space, alignment, type, name, and — when
297 /// \p EmitInitializer is set — the initializer of \p GVar. Passing false
298 /// prints a declaration whose type still matches the definition, as an
299 /// `.extern` forward declaration requires.
300 void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar,
301 raw_ostream &O,
302 const NVPTXSubtarget &STI,
303 bool EmitInitializer);
304 void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const;
305 std::string getPTXFundamentalTypeStr(Type *Ty) const;
306 void printScalarConstant(const Constant *CPV, raw_ostream &O);
307 void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const;
308 void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer);
309 void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer);
310
311 void emitLinkageDirective(const GlobalValue *V, raw_ostream &O);
312 void emitDeclarations(const Module &, raw_ostream &O);
313 void emitDeclaration(const Function *, raw_ostream &O);
314 void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O);
315 void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O);
316 void emitDemotedVars(const Function *, raw_ostream &);
317
318 bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const;
319
320 // Used to control the need to emit .generic() in the initializer of
321 // module scope variables.
322 // Although ptx supports the hybrid mode like the following,
323 // .global .u32 a;
324 // .global .u32 b;
325 // .global .u32 addr[] = {a, generic(b)}
326 // we have difficulty representing the difference in the NVVM IR.
327 //
328 // Since the address value should always be generic in CUDA C and always
329 // be specific in OpenCL, we use this simple control here.
330 //
331 const bool EmitGeneric;
332
333public:
334 NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
335 : AsmPrinter(TM, std::move(Streamer), ID),
336 EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() ==
337 NVPTX::CUDA) {}
338
339 bool runOnMachineFunction(MachineFunction &F) override;
340
341 void getAnalysisUsage(AnalysisUsage &AU) const override {
342 AU.addRequired<MachineLoopInfoWrapperPass>();
343 AsmPrinter::getAnalysisUsage(AU);
344 }
345
346 std::string getVirtualRegisterName(Register Reg) const;
347
348 const MCSymbol *getFunctionFrameSymbol() const override;
349
350 // Make emitGlobalVariable() no-op for NVPTX.
351 // Global variables have been already emitted by the time the base AsmPrinter
352 // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()).
353 void emitGlobalVariable(const GlobalVariable *GV) override {}
354};
355
356} // end anonymous namespace
357
358/// Emits initial debug location directive.
359static void emitInitialRawDwarfLocDirective(const MachineFunction &MF,
360 DwarfDebug *DD,
361 MCStreamer &OutStreamer) {
362 if (!DD)
363 return;
364
365 assert(OutStreamer.hasRawTextSupport() && "Expected assembly output mode.");
366 // This is NVPTX specific and it's unclear why.
367 // PR51079: If we have code without debug information we need to give up.
368 const DISubprogram *SP = MF.getFunction().getSubprogram();
369 if (!SP)
370 return;
371 assert(SP->getUnit());
372 // NoDebug and DebugDirectivesOnly do not require emitting the initial loc
373 // directive. NoDebug does not require any debug directives and the initial
374 // loc directive is not needed for DebugDirectivesOnly as it is redundant
375 // assuming this is a non-empty function.
376 if (SP->getUnit()->isDebugDirectivesOnly() || SP->getUnit()->isNoDebug())
377 return;
378
379 (void)DD->emitInitialLocDirective(MF, /*CUID=*/0);
380}
381
382namespace {
383
384/// Return a list of GlobalVariables on which \p V depends.
385static void
386discoverDependentGlobals(const Value *V,
387 SmallVectorImpl<const GlobalVariable *> &Globals,
388 SmallPtrSetImpl<const GlobalVariable *> &Seen) {
389 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V)) {
390 if (Seen.insert(Ptr: GV).second)
391 Globals.push_back(Elt: GV);
392 return;
393 }
394
395 // Global values are emitted as symbols. Their operands do not contribute to
396 // the initializer expression that refers to that symbol.
397 if (isa<GlobalValue>(Val: V))
398 return;
399
400 // lowerConstantForGV emits a GEP as its base symbol plus a constant byte
401 // offset. Symbols used to compute an index are not part of that expression.
402 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(Val: V)) {
403 discoverDependentGlobals(V: GEP->getPointerOperand(), Globals, Seen);
404 return;
405 }
406
407 if (const User *U = dyn_cast<User>(Val: V))
408 for (const auto &O : U->operands())
409 discoverDependentGlobals(V: O, Globals, Seen);
410}
411
412struct GlobalVariableDependencyNode {
413 const GlobalVariable *GV = nullptr;
414 unsigned ModuleOrder = 0;
415 SmallVector<const GlobalVariableDependencyNode *, 4> Dependencies;
416};
417
418class GlobalVariableDependencyGraph {
419 // scc_iterator needs a single entry node. Global initializer dependencies
420 // may be disconnected, so use a synthetic root with an edge to every global.
421 GlobalVariableDependencyNode SyntheticRoot;
422 // Edges store pointers into Nodes, so node addresses must remain stable while
423 // the graph is constructed.
424 std::map<const GlobalVariable *, GlobalVariableDependencyNode> Nodes;
425
426public:
427 explicit GlobalVariableDependencyGraph(const Module &M) {
428 unsigned ModuleOrder = 0;
429 for (const GlobalVariable &GV : M.globals()) {
430 GlobalVariableDependencyNode &Node = Nodes.try_emplace(k: &GV).first->second;
431 Node.GV = &GV;
432 Node.ModuleOrder = ModuleOrder++;
433 SyntheticRoot.Dependencies.push_back(Elt: &Node);
434 }
435
436 for (auto &[GV, Node] : Nodes) {
437 SmallVector<const GlobalVariable *, 4> Dependencies;
438 SmallPtrSet<const GlobalVariable *, 4> Seen;
439 for (const Use &Operand : GV->operands())
440 discoverDependentGlobals(V: Operand, Globals&: Dependencies, Seen);
441
442 for (const GlobalVariable *Dependency : Dependencies) {
443 auto It = Nodes.find(x: Dependency);
444 if (It != Nodes.end())
445 Node.Dependencies.push_back(Elt: &It->second);
446 }
447 }
448 }
449
450 const GlobalVariableDependencyNode *getEntryNode() const {
451 return &SyntheticRoot;
452 }
453};
454
455struct GlobalVariableDependencyGraphTraits {
456 using NodeRef = const GlobalVariableDependencyNode *;
457 using ChildIteratorType =
458 SmallVectorImpl<const GlobalVariableDependencyNode *>::const_iterator;
459
460 static NodeRef getEntryNode(NodeRef Node) { return Node; }
461 static ChildIteratorType child_begin(NodeRef Node) {
462 return Node->Dependencies.begin();
463 }
464 static ChildIteratorType child_end(NodeRef Node) {
465 return Node->Dependencies.end();
466 }
467};
468
469using GlobalVariableSCCIterator =
470 scc_iterator<const GlobalVariableDependencyNode *,
471 GlobalVariableDependencyGraphTraits>;
472
473static bool shouldSkipModuleLevelGlobal(const GlobalVariable &GV) {
474 if (GV.hasSection() && GV.getSection() == "llvm.metadata")
475 return true;
476 return GV.getName().starts_with(Prefix: "llvm.") || GV.getName().starts_with(Prefix: "nvvm.");
477}
478
479static bool isForwardDeclarableGlobal(const GlobalVariable *GVar) {
480 if (shouldSkipModuleLevelGlobal(GV: *GVar) || GVar->isDeclaration() ||
481 getPTXOpaqueType(*GVar) != PTXOpaqueType::None)
482 return false;
483
484 // A PTX .extern declaration can be resolved by a later .visible, .weak, or
485 // .common definition, but not by a static definition.
486 if (GVar->hasExternalLinkage())
487 return GVar->hasInitializer();
488
489 if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
490 GVar->hasAvailableExternallyLinkage() || GVar->hasCommonLinkage())
491 return true;
492
493 return false;
494}
495
496/// Order definitions after treating references to forward-declared globals as
497/// already satisfied. A remaining cycle cannot be emitted portably because it
498/// requires an undeclared forward reference.
499static SmallVector<const GlobalVariable *, 4> orderDefinitionsInSCC(
500 ArrayRef<const GlobalVariableDependencyNode *> SCC,
501 const DenseSet<const GlobalVariableDependencyNode *> &ForwardDeclared) {
502 using Node = GlobalVariableDependencyNode;
503
504 DenseSet<const Node *> SCCSet;
505 SCCSet.insert_range(R&: SCC);
506
507 DenseMap<const Node *, unsigned> DependencyCount;
508 DenseMap<const Node *, SmallVector<const Node *, 4>> Dependents;
509 std::set<std::pair<unsigned, const Node *>> Ready;
510
511 // Dependencies outside this SCC have already been emitted. Forward-declared
512 // dependencies are also satisfied, so only count the remaining SCC edges.
513 for (const Node *N : SCC) {
514 unsigned &Count = DependencyCount[N];
515 for (const Node *Dependency : N->Dependencies) {
516 if (!SCCSet.count(V: Dependency) || ForwardDeclared.count(V: Dependency))
517 continue;
518 ++Count;
519 Dependents[Dependency].push_back(Elt: N);
520 }
521 if (Count == 0)
522 Ready.emplace(args: N->ModuleOrder, args&: N);
523 }
524
525 SmallVector<const GlobalVariable *, 4> Order;
526 while (!Ready.empty()) {
527 const Node *N = Ready.begin()->second;
528 Ready.erase(position: Ready.begin());
529 Order.push_back(Elt: N->GV);
530
531 auto It = Dependents.find(Val: N);
532 if (It == Dependents.end())
533 continue;
534 for (const Node *Dependent : It->second) {
535 assert(DependencyCount[Dependent] && "Dependency already satisfied");
536 if (--DependencyCount[Dependent] == 0)
537 Ready.emplace(args: Dependent->ModuleOrder, args&: Dependent);
538 }
539 }
540
541 if (Order.size() != SCC.size())
542 report_fatal_error(reason: "Circular dependency found in global variable set");
543 return Order;
544}
545
546} // namespace
547
548void NVPTXAsmPrinter::emitInstruction(const MachineInstr *MI) {
549 NVPTX_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(),
550 Features: getSubtargetInfo().getFeatureBits());
551
552 MCInst Inst;
553 lowerToMCInst(MI, OutMI&: Inst);
554 EmitToStreamer(S&: *OutStreamer, Inst);
555}
556
557void NVPTXAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
558 OutMI.setOpcode(MI->getOpcode());
559 for (const auto MO : MI->operands())
560 OutMI.addOperand(Op: lowerOperand(MO));
561}
562
563MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) {
564 switch (MO.getType()) {
565 default:
566 llvm_unreachable("unknown operand type");
567 case MachineOperand::MO_Register:
568 return MCOperand::createReg(Reg: encodeVirtualRegister(Reg: MO.getReg()));
569 case MachineOperand::MO_Immediate:
570 return MCOperand::createImm(Val: MO.getImm());
571 case MachineOperand::MO_MachineBasicBlock:
572 return MCOperand::createExpr(
573 Val: MCSymbolRefExpr::create(Symbol: MO.getMBB()->getSymbol(), Ctx&: OutContext));
574 case MachineOperand::MO_ExternalSymbol:
575 return GetSymbolRef(Symbol: GetExternalSymbolSymbol(Sym: MO.getSymbolName()));
576 case MachineOperand::MO_MCSymbol:
577 return GetSymbolRef(Symbol: MO.getMCSymbol());
578 case MachineOperand::MO_JumpTableIndex:
579 // The jump table index names the .branchtargets list emitted for a brx.idx
580 // (see emitJumpTable); reference it by that label.
581 return GetSymbolRef(Symbol: GetJTISymbol(JTID: MO.getIndex()));
582 case MachineOperand::MO_GlobalAddress:
583 return GetSymbolRef(Symbol: getSymbol(GV: MO.getGlobal()));
584 case MachineOperand::MO_FPImmediate: {
585 const ConstantFP *Cnt = MO.getFPImm();
586 const APFloat &Val = Cnt->getValueAPF();
587
588 switch (Cnt->getType()->getTypeID()) {
589 default:
590 report_fatal_error(reason: "Unsupported FP type");
591 break;
592 case Type::HalfTyID:
593 return MCOperand::createExpr(
594 Val: NVPTXFloatMCExpr::createConstantFPHalf(Flt: Val, Ctx&: OutContext));
595 case Type::BFloatTyID:
596 return MCOperand::createExpr(
597 Val: NVPTXFloatMCExpr::createConstantBFPHalf(Flt: Val, Ctx&: OutContext));
598 case Type::FloatTyID:
599 return MCOperand::createExpr(
600 Val: NVPTXFloatMCExpr::createConstantFPSingle(Flt: Val, Ctx&: OutContext));
601 case Type::DoubleTyID:
602 return MCOperand::createExpr(
603 Val: NVPTXFloatMCExpr::createConstantFPDouble(Flt: Val, Ctx&: OutContext));
604 }
605 break;
606 }
607 }
608}
609
610static NVPTX::VirtualRegisterKind
611getVirtualRegisterKind(const TargetRegisterClass *RC) {
612 if (RC == &NVPTX::B1RegClass)
613 return NVPTX::VirtualRegisterKind::B1;
614 if (RC == &NVPTX::B16RegClass)
615 return NVPTX::VirtualRegisterKind::B16;
616 if (RC == &NVPTX::B32RegClass)
617 return NVPTX::VirtualRegisterKind::B32;
618 if (RC == &NVPTX::B64RegClass)
619 return NVPTX::VirtualRegisterKind::B64;
620 if (RC == &NVPTX::B128RegClass)
621 return NVPTX::VirtualRegisterKind::B128;
622 llvm_unreachable("Bad register class");
623}
624
625unsigned NVPTXAsmPrinter::getVirtualRegisterNumber(Register Reg) const {
626 const auto It = VRegMapping.find(Val: MRI->getRegClass(Reg));
627 assert(It != VRegMapping.end() && "Bad register class");
628
629 const unsigned Num = It->second.lookup(Val: Reg);
630 assert(Num && "Bad virtual register");
631 return Num;
632}
633
634MCRegister NVPTXAsmPrinter::encodeVirtualRegister(Register Reg) {
635 if (Reg.isVirtual()) {
636 // Pack the register class into the upper bits so that
637 // NVPTXInstPrinter::printRegName can recover the declared name.
638 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
639 const unsigned Num = getVirtualRegisterNumber(Reg);
640 assert(Num <= NVPTX::VirtualRegisterNumMask &&
641 "Too many virtual registers");
642 return (static_cast<unsigned>(Kind) << NVPTX::VirtualRegisterKindShift) |
643 Num;
644 }
645
646 // Some special-use registers are actually physical registers.
647 // Encode this as the register class ID of 0 and the real register ID.
648 assert(Reg.id() <= NVPTX::VirtualRegisterNumMask &&
649 "Physical register would decode as a virtual register");
650 return Reg.asMCReg();
651}
652
653MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) {
654 const MCExpr *Expr;
655 Expr = MCSymbolRefExpr::create(Symbol, Ctx&: OutContext);
656 return MCOperand::createExpr(Val: Expr);
657}
658
659template <typename OwnerT>
660static void printParam(const OwnerT *Owner, Type *Ty, unsigned AttrIdx,
661 bool IsByVal, bool IsKernel, StringRef Name,
662 const DataLayout &DL, raw_ostream &O) {
663 O << ".param ";
664
665 if (IsByVal || shouldPassAsArray(Ty)) {
666 const Align ParamAlign =
667 IsByVal && !IsKernel ? getDeviceByValParamAlign(Owner, Ty, AttrIdx, DL)
668 : getPTXParamAlign(Owner, Ty, AttrIdx, DL);
669 O << ".align " << ParamAlign.value() << " .b8 " << Name << "["
670 << DL.getTypeAllocSize(Ty) << "]";
671 return;
672 }
673
674 assert((Ty->isFloatingPointTy() || Ty->isIntOrPtrTy()) &&
675 "Unknown parameter type");
676 const unsigned Size = DL.getTypeSizeInBits(Ty).getFixedValue();
677 O << ".b"
678 << (IsKernel ? promoteScalarKernelArgumentSize(Size)
679 : promoteScalarArgumentSize(Size))
680 << " " << Name;
681}
682
683template <typename OwnerT>
684static void printReturnValClause(const OwnerT *Owner, StringRef Name,
685 const DataLayout &DL, raw_ostream &O) {
686 Type *RetTy = Owner->getFunctionType()->getReturnType();
687
688 // A void or zero-sized return type (e.g. an empty struct) produces no return
689 // parameter.
690 if (RetTy->isVoidTy() || RetTy->isEmptyTy())
691 return;
692
693 // Only device functions return a value, so no kernel promotion applies.
694 O << "(";
695 printParam(Owner, RetTy, AttributeList::ReturnIndex, /*IsByVal=*/false,
696 /*IsKernel=*/false, Name, DL, O);
697 O << ") ";
698}
699
700void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB,
701 MCSymbol *PrototypeSymbol) const {
702 const DataLayout &DL = getDataLayout();
703 const NVPTXSubtarget &STI = MF->getSubtarget<NVPTXSubtarget>();
704
705 OutStreamer->emitLabel(Symbol: PrototypeSymbol);
706
707 SmallString<128> Str;
708 raw_svector_ostream O(Str);
709
710 O << ".callprototype ";
711 printReturnValClause(Owner: &CB, Name: "_", DL, O);
712 O << "_ (";
713
714 auto MakeArg = [&](const unsigned I) {
715 const bool IsByVal = CB.isByValArgument(ArgNo: I);
716 Type *Ty =
717 IsByVal ? CB.getParamByValType(ArgNo: I) : CB.getArgOperand(i: I)->getType();
718
719 printParam(Owner: &CB, Ty, AttrIdx: I + AttributeList::FirstArgIndex, IsByVal,
720 /*IsKernel=*/false, Name: "_", DL, O);
721 };
722
723 const FunctionType *FTy = CB.getFunctionType();
724 const unsigned NumArgs = FTy->getNumParams();
725
726 // Zero-sized arguments (e.g. empty structs) are not passed and so do not
727 // appear in the prototype.
728 const auto NonEmptyArgs = make_filter_range(Range: seq(Size: NumArgs), Pred: [&](unsigned I) {
729 return !CB.getArgOperand(i: I)->getType()->isEmptyTy();
730 });
731
732 interleave(c: NonEmptyArgs, os&: O, each_fn: MakeArg, separator: ", ");
733
734 if (FTy->isVarArg() && CB.arg_size() > NumArgs)
735 O << (NonEmptyArgs.empty() ? "" : ",") << " .param .align "
736 << STI.getMaxRequiredAlignment() << " .b8 _[]";
737
738 O << ")\n";
739 printFunctionDirectives(V: CB, O);
740 O << ";\n";
741
742 OutStreamer->emitRawText(String: O.str());
743}
744
745void NVPTXAsmPrinter::emitJumpTable(const MachineJumpTableEntry &MJT,
746 unsigned MJTI) const {
747 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: MJTI));
748
749 if (MJT.MBBs.empty())
750 return;
751
752 const auto Targets = to_vector(
753 Range: map_range(C: MJT.MBBs, F: [](const MachineBasicBlock *MBB) -> const MCSymbol * {
754 return MBB->getSymbol();
755 }));
756 getTargetStreamer()->emitBranchTargetsDirective(Targets);
757}
758
759// Return true if MBB is the header of a loop marked with
760// llvm.loop.unroll.disable or llvm.loop.unroll.count=1.
761bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
762 const MachineBasicBlock &MBB) const {
763 const MachineLoopInfo *LI = GetMLI(*MF);
764 assert(LI && "NVPTXAsmPrinter requires MachineLoopInfo");
765 // We insert .pragma "nounroll" only to the loop header.
766 if (!LI->isLoopHeader(BB: &MBB))
767 return false;
768
769 // llvm.loop.unroll.disable is marked on the back edges of a loop. Therefore,
770 // we iterate through each back edge of the loop with header MBB, and check
771 // whether its metadata contains llvm.loop.unroll.disable.
772 for (const MachineBasicBlock *PMBB : MBB.predecessors()) {
773 if (LI->getLoopFor(BB: PMBB) != LI->getLoopFor(BB: &MBB)) {
774 // Edges from other loops to MBB are not back edges.
775 continue;
776 }
777 if (const BasicBlock *PBB = PMBB->getBasicBlock()) {
778 if (MDNode *LoopID =
779 PBB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop)) {
780 if (GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.disable"))
781 return true;
782 if (MDNode *UnrollCountMD =
783 GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.count")) {
784 if (mdconst::extract<ConstantInt>(MD: UnrollCountMD->getOperand(I: 1))
785 ->isOne())
786 return true;
787 }
788 }
789 }
790 }
791 return false;
792}
793
794void NVPTXAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
795 AsmPrinter::emitBasicBlockStart(MBB);
796 if (isLoopHeaderOfNoUnroll(MBB))
797 getTargetStreamer()->emitPragmaDirective(Pragma: "nounroll");
798}
799
800void NVPTXAsmPrinter::emitFunctionEntryLabel() {
801 SmallString<128> Str;
802 raw_svector_ostream O(Str);
803
804 if (!GlobalsEmitted) {
805 emitGlobals(M: *MF->getFunction().getParent());
806 GlobalsEmitted = true;
807 }
808
809 // Set up
810 MRI = &MF->getRegInfo();
811 F = &MF->getFunction();
812 emitLinkageDirective(V: F, O);
813 if (isKernelFunction(F: *F))
814 O << ".entry ";
815 else {
816 O << ".func ";
817 printReturnValClause(Owner: F, Name: "func_retval0", DL: getDataLayout(), O);
818 }
819
820 CurrentFnSym->print(OS&: O, MAI);
821
822 emitFunctionParamList(F, O);
823 O << "\n";
824
825 if (isKernelFunction(F: *F))
826 emitKernelFunctionDirectives(F: *F, O);
827 else
828 printFunctionDirectives(V: *F, O);
829
830 OutStreamer->emitRawText(String: O.str());
831
832 VRegMapping.clear();
833 // Emit open brace for function body.
834 OutStreamer->emitRawText(String: StringRef("{\n"));
835 setAndEmitFunctionVirtualRegisters(*MF);
836 encodeDebugInfoRegisterNumbers(MF: *MF);
837 // Emit initial .loc debug directive for correct relocation symbol data.
838 emitInitialRawDwarfLocDirective(MF: *MF, DD: getDwarfDebug(), OutStreamer&: *OutStreamer);
839}
840
841bool NVPTXAsmPrinter::runOnMachineFunction(MachineFunction &F) {
842 bool Result = AsmPrinter::runOnMachineFunction(MF&: F);
843 // Emit closing brace for the body of function F.
844 // The closing brace must be emitted here because we need to emit additional
845 // debug labels/data after the last basic block.
846 // We need to emit the closing brace here because we don't have function that
847 // finished emission of the function body.
848 OutStreamer->emitRawText(String: StringRef("}\n"));
849 return Result;
850}
851
852void NVPTXAsmPrinter::emitFunctionBodyStart() {
853 SmallString<128> Str;
854 raw_svector_ostream O(Str);
855 emitDemotedVars(&MF->getFunction(), O);
856 OutStreamer->emitRawText(String: O.str());
857
858 const auto *MFI = MF->getInfo<NVPTXMachineFunctionInfo>();
859 for (const auto &[CB, Symbol] : MFI->getCallPrototypes())
860 emitCallPrototype(CB: *CB, PrototypeSymbol: Symbol);
861
862 if (const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo())
863 for (const auto &[Idx, JT] : enumerate(First: MJTI->getJumpTables()))
864 emitJumpTable(MJT: JT, MJTI: Idx);
865}
866
867void NVPTXAsmPrinter::emitFunctionBodyEnd() {
868 VRegMapping.clear();
869}
870
871const MCSymbol *NVPTXAsmPrinter::getFunctionFrameSymbol() const {
872 return OutContext.getOrCreateSymbol(DEPOTNAME + Twine(getFunctionNumber()));
873}
874
875void NVPTXAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
876 Register RegNo = MI->getOperand(i: 0).getReg();
877 if (RegNo.isVirtual())
878 OutStreamer->AddComment(T: Twine("implicit-def: ") +
879 getVirtualRegisterName(Reg: RegNo));
880 else
881 OutStreamer->AddComment(T: Twine("implicit-def: ") +
882 NVPTXInstPrinter::getRegisterName(Reg: RegNo));
883 OutStreamer->addBlankLine();
884}
885
886void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
887 raw_ostream &O) const {
888 // If the NVVM IR has some of reqntid* specified, then output
889 // the reqntid directive, and set the unspecified ones to 1.
890 // If none of Reqntid* is specified, don't output reqntid directive.
891 const auto ReqNTID = getReqNTID(F);
892 if (!ReqNTID.empty())
893 O << formatv(Fmt: ".reqntid {0:$[, ]}\n",
894 Vals: make_range(x: ReqNTID.begin(), y: ReqNTID.end()));
895
896 const auto MaxNTID = getMaxNTID(F);
897 if (!MaxNTID.empty())
898 O << formatv(Fmt: ".maxntid {0:$[, ]}\n",
899 Vals: make_range(x: MaxNTID.begin(), y: MaxNTID.end()));
900
901 if (const auto Mincta = getMinCTASm(F))
902 O << ".minnctapersm " << *Mincta << "\n";
903
904 if (const auto Maxnreg = getMaxNReg(F))
905 O << ".maxnreg " << *Maxnreg << "\n";
906
907 // .maxclusterrank directive requires SM_90 or higher, make sure that we
908 // filter it out for lower SM versions, as it causes a hard ptxas crash.
909 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
910 const NVPTXSubtarget *STI = &NTM.getSubtarget<NVPTXSubtarget>(F);
911
912 if (STI->hasFeature(Feature: NVPTX::SM90)) {
913 const auto ClusterDim = getClusterDim(F);
914 const bool BlocksAreClusters = hasBlocksAreClusters(F);
915
916 if (!ClusterDim.empty()) {
917
918 if (!BlocksAreClusters)
919 O << ".explicitcluster\n";
920
921 if (ClusterDim[0] != 0) {
922 assert(llvm::all_of(ClusterDim, not_equal_to(0)) &&
923 "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z "
924 "should be non-zero as well");
925
926 O << formatv(Fmt: ".reqnctapercluster {0:$[, ]}\n",
927 Vals: make_range(x: ClusterDim.begin(), y: ClusterDim.end()));
928 } else {
929 assert(llvm::all_of(ClusterDim, equal_to(0)) &&
930 "cluster_dim_x == 0 implies cluster_dim_y and cluster_dim_z "
931 "should be 0 as well");
932 }
933 }
934
935 if (BlocksAreClusters) {
936 LLVMContext &Ctx = F.getContext();
937 if (ReqNTID.empty() || ClusterDim.empty())
938 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
939 F, "blocksareclusters requires reqntid and cluster_dim attributes",
940 F.getSubprogram()));
941 else if (!STI->hasFeature(Feature: NVPTX::PTX90))
942 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
943 F, "blocksareclusters requires PTX version >= 9.0",
944 F.getSubprogram()));
945 else
946 O << ".blocksareclusters\n";
947 }
948
949 if (const auto Maxclusterrank = getMaxClusterRank(F))
950 O << ".maxclusterrank " << *Maxclusterrank << "\n";
951 }
952}
953
954std::string NVPTXAsmPrinter::getVirtualRegisterName(Register Reg) const {
955 const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg));
956
957 std::string Name;
958 raw_string_ostream(Name) << NVPTX::getVirtualRegisterPrefix(Kind)
959 << getVirtualRegisterNumber(Reg);
960 return Name;
961}
962
963void NVPTXAsmPrinter::emitAliasDeclaration(const GlobalAlias *GA,
964 raw_ostream &O) {
965 const Function *F = dyn_cast_or_null<Function>(Val: GA->getAliaseeObject());
966 if (!F || isKernelFunction(F: *F) || F->isDeclaration())
967 report_fatal_error(
968 reason: "NVPTX aliasee must be a non-kernel function definition");
969
970 if (GA->hasLinkOnceLinkage() || GA->hasWeakLinkage() ||
971 GA->hasAvailableExternallyLinkage() || GA->hasCommonLinkage())
972 report_fatal_error(reason: "NVPTX aliasee must not be '.weak'");
973
974 emitDeclarationWithName(F, getSymbol(GV: GA), O);
975}
976
977void NVPTXAsmPrinter::emitDeclaration(const Function *F, raw_ostream &O) {
978 emitDeclarationWithName(F, getSymbol(GV: F), O);
979}
980
981void NVPTXAsmPrinter::emitDeclarationWithName(const Function *F, MCSymbol *S,
982 raw_ostream &O) {
983 emitLinkageDirective(V: F, O);
984 if (isKernelFunction(F: *F)) {
985 O << ".entry ";
986 } else {
987 O << ".func ";
988 printReturnValClause(Owner: F, Name: "func_retval0", DL: getDataLayout(), O);
989 }
990 S->print(OS&: O, MAI);
991 O << "\n";
992 emitFunctionParamList(F, O);
993 O << "\n";
994 if (!isKernelFunction(F: *F))
995 printFunctionDirectives(V: *F, O);
996 O << ";\n";
997}
998
999static bool usedInGlobalVarDef(const Constant *C) {
1000 if (!C)
1001 return false;
1002
1003 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C))
1004 return GV->getName() != "llvm.used";
1005
1006 for (const User *U : C->users())
1007 if (const Constant *C = dyn_cast<Constant>(Val: U))
1008 if (usedInGlobalVarDef(C))
1009 return true;
1010
1011 return false;
1012}
1013
1014static bool usedInOneFunc(const User *U, Function const *&OneFunc) {
1015 if (const GlobalVariable *OtherGV = dyn_cast<GlobalVariable>(Val: U))
1016 if (OtherGV->getName() == "llvm.used")
1017 return true;
1018
1019 if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1020 if (const Function *CurFunc = I->getFunction()) {
1021 if (OneFunc && (CurFunc != OneFunc))
1022 return false;
1023 OneFunc = CurFunc;
1024 return true;
1025 }
1026 return false;
1027 }
1028
1029 for (const User *UU : U->users())
1030 if (!usedInOneFunc(U: UU, OneFunc))
1031 return false;
1032
1033 return true;
1034}
1035
1036/* Find out if a global variable can be demoted to local scope.
1037 * Currently, this is valid for CUDA shared variables, which have local
1038 * scope and global lifetime. So the conditions to check are :
1039 * 1. Is the global variable in shared address space?
1040 * 2. Does it have local linkage?
1041 * 3. Is the global variable referenced only in one function?
1042 */
1043static bool canDemoteGlobalVar(const GlobalVariable *GV, Function const *&f) {
1044 if (!GV->hasLocalLinkage())
1045 return false;
1046 if (GV->getAddressSpace() != ADDRESS_SPACE_SHARED)
1047 return false;
1048
1049 const Function *oneFunc = nullptr;
1050
1051 bool flag = usedInOneFunc(U: GV, OneFunc&: oneFunc);
1052 if (!flag)
1053 return false;
1054 if (!oneFunc)
1055 return false;
1056 f = oneFunc;
1057 return true;
1058}
1059
1060static bool useFuncSeen(const Constant *C,
1061 const SmallPtrSetImpl<const Function *> &SeenSet) {
1062 for (const User *U : C->users()) {
1063 if (const Constant *cu = dyn_cast<Constant>(Val: U)) {
1064 if (useFuncSeen(C: cu, SeenSet))
1065 return true;
1066 } else if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
1067 if (const Function *Caller = I->getFunction())
1068 if (SeenSet.contains(Ptr: Caller))
1069 return true;
1070 }
1071 }
1072 return false;
1073}
1074
1075void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) {
1076 SmallPtrSet<const Function *, 32> SeenSet;
1077 for (const Function &F : M) {
1078 if (F.getAttributes().hasFnAttr(Kind: "nvptx-libcall-callee")) {
1079 emitDeclaration(F: &F, O);
1080 continue;
1081 }
1082
1083 if (F.isDeclaration()) {
1084 if (F.use_empty())
1085 continue;
1086 if (F.getIntrinsicID())
1087 continue;
1088 // An unrecognized intrinsic would produce an invalid PTX declaration. Let
1089 // the user know that, and skip it.
1090 if (F.isIntrinsic()) {
1091 LLVMContext &Ctx = F.getContext();
1092 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
1093 F, "unknown intrinsic '" + F.getName() +
1094 "' cannot be lowered by the NVPTX backend"));
1095 continue;
1096 }
1097 emitDeclaration(F: &F, O);
1098 continue;
1099 }
1100 for (const User *U : F.users()) {
1101 if (const Constant *C = dyn_cast<Constant>(Val: U)) {
1102 if (usedInGlobalVarDef(C)) {
1103 // The use is in the initialization of a global variable
1104 // that is a function pointer, so print a declaration
1105 // for the original function
1106 emitDeclaration(F: &F, O);
1107 break;
1108 }
1109 // Emit a declaration of this function if the function that
1110 // uses this constant expr has already been seen.
1111 if (useFuncSeen(C, SeenSet)) {
1112 emitDeclaration(F: &F, O);
1113 break;
1114 }
1115 }
1116
1117 if (!isa<Instruction>(Val: U))
1118 continue;
1119 const Function *Caller = cast<Instruction>(Val: U)->getFunction();
1120 if (!Caller)
1121 continue;
1122
1123 // If a caller has already been seen, then the caller is
1124 // appearing in the module before the callee. so print out
1125 // a declaration for the callee.
1126 if (SeenSet.contains(Ptr: Caller)) {
1127 emitDeclaration(F: &F, O);
1128 break;
1129 }
1130 }
1131 SeenSet.insert(Ptr: &F);
1132 }
1133 for (const GlobalAlias &GA : M.aliases())
1134 emitAliasDeclaration(GA: &GA, O);
1135}
1136
1137void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) {
1138 // Construct a default subtarget off of the TargetMachine defaults. The
1139 // rest of NVPTX isn't friendly to change subtargets per function and
1140 // so the default TargetMachine will have all of the options.
1141 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1142 const NVPTXSubtarget *STI = NTM.getSubtargetImpl();
1143
1144 // Emit header before any dwarf directives are emitted below.
1145 emitHeader(M, STI: *STI);
1146}
1147
1148/// Create NVPTX-specific DwarfDebug handler.
1149DwarfDebug *NVPTXAsmPrinter::createDwarfDebug() {
1150 return new NVPTXDwarfDebug(this);
1151}
1152
1153bool NVPTXAsmPrinter::doInitialization(Module &M) {
1154 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1155 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1156 if (M.alias_size() &&
1157 (!STI.hasFeature(Feature: NVPTX::PTX63) || !STI.hasFeature(Feature: NVPTX::SM30)))
1158 report_fatal_error(reason: ".alias requires PTX version >= 6.3 and sm_30");
1159
1160 // We need to call the parent's one explicitly.
1161 bool Result = AsmPrinter::doInitialization(M);
1162
1163 GlobalsEmitted = false;
1164
1165 // Ensure globals are in the symbol table before ISel so any temp symbols are
1166 // guaranteed not to collide with user symbols
1167 for (const GlobalValue &GV : M.global_values())
1168 getSymbol(GV: &GV);
1169
1170 return Result;
1171}
1172
1173void NVPTXAsmPrinter::emitGlobals(const Module &M) {
1174 SmallString<128> Str2;
1175 raw_svector_ostream OS2(Str2);
1176
1177 emitDeclarations(M, O&: OS2);
1178
1179 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1180 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1181
1182 // ptxas requires global symbols referenced by initializers to be known
1183 // before use. Acyclic dependencies can be handled by dependency-first
1184 // emission. Cyclic SCCs need compatible .extern declarations first.
1185 // Edges point from each global to the globals used by its initializer.
1186 // Reverse-topological SCC iteration therefore emits dependencies first.
1187 GlobalVariableDependencyGraph DependencyGraph(M);
1188 for (GlobalVariableSCCIterator I =
1189 GlobalVariableSCCIterator::begin(G: DependencyGraph.getEntryNode());
1190 !I.isAtEnd(); ++I) {
1191 SmallVector<const GlobalVariableDependencyNode *, 4> SCC(I->begin(),
1192 I->end());
1193
1194 // Nothing points to the synthetic root, so it is always in its own SCC.
1195 if (!SCC.front()->GV) {
1196 assert(SCC.size() == 1 && "Synthetic root must be in its own SCC");
1197 continue;
1198 }
1199
1200 llvm::sort(C&: SCC, Comp: [](const auto *LHS, const auto *RHS) {
1201 return LHS->ModuleOrder < RHS->ModuleOrder;
1202 });
1203
1204 const bool IsCyclic = I.hasCycle();
1205 DenseSet<const GlobalVariableDependencyNode *> ForwardDeclared;
1206 if (IsCyclic)
1207 for (const auto *Node : SCC)
1208 if (isForwardDeclarableGlobal(GVar: Node->GV))
1209 ForwardDeclared.insert(V: Node);
1210
1211 // Check that declarations break every cycle before writing any output.
1212 SmallVector<const GlobalVariable *, 4> OrderedGlobals =
1213 IsCyclic ? orderDefinitionsInSCC(SCC, ForwardDeclared)
1214 : SmallVector<const GlobalVariable *, 4>{SCC.front()->GV};
1215
1216 for (const auto *Node : SCC) {
1217 if (!ForwardDeclared.count(V: Node))
1218 continue;
1219 OS2 << ".extern ";
1220 emitPTXGlobalVariableDefinition(GVar: Node->GV, O&: OS2, STI,
1221 /*EmitInitializer=*/false);
1222 OS2 << ";\n";
1223 }
1224
1225 for (const GlobalVariable *GV : OrderedGlobals)
1226 printModuleLevelGV(GVar: GV, O&: OS2, /*ProcessDemoted=*/processDemoted: false, STI);
1227 }
1228
1229 OS2 << '\n';
1230
1231 OutStreamer->emitRawText(String: OS2.str());
1232}
1233
1234void NVPTXAsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) {
1235 getTargetStreamer()->emitAliasDirective(Name: getSymbol(GV: &GA),
1236 Aliasee: getSymbol(GV: GA.getAliaseeObject()));
1237}
1238
1239NVPTXTargetStreamer *NVPTXAsmPrinter::getTargetStreamer() const {
1240 return static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1241}
1242
1243static bool hasFullDebugInfo(Module &M) {
1244 for (DICompileUnit *CU : M.debug_compile_units()) {
1245 switch(CU->getEmissionKind()) {
1246 case DICompileUnit::NoDebug:
1247 case DICompileUnit::DebugDirectivesOnly:
1248 break;
1249 case DICompileUnit::LineTablesOnly:
1250 case DICompileUnit::FullDebug:
1251 return true;
1252 }
1253 }
1254
1255 return false;
1256}
1257
1258void NVPTXAsmPrinter::emitHeader(Module &M, const NVPTXSubtarget &STI) {
1259 auto *TS = getTargetStreamer();
1260
1261 TS->emitBanner();
1262
1263 const unsigned PTXVersion = STI.getPTXVersion();
1264 TS->emitVersionDirective(PTXVersion);
1265
1266 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1267 bool TexModeIndependent = NTM.getDrvInterface() == NVPTX::NVCL;
1268
1269 TS->emitTargetDirective(Target: STI.getTargetName(), TexModeIndependent,
1270 HasDebug: hasFullDebugInfo(M));
1271 TS->emitAddressSizeDirective(AddrSize: M.getDataLayout().getPointerSizeInBits());
1272}
1273
1274bool NVPTXAsmPrinter::doFinalization(Module &M) {
1275 // If we did not emit any functions, then the global declarations have not
1276 // yet been emitted.
1277 if (!GlobalsEmitted) {
1278 emitGlobals(M);
1279 GlobalsEmitted = true;
1280 }
1281
1282 // call doFinalization
1283 bool ret = AsmPrinter::doFinalization(M);
1284
1285 clearAnnotationCache(&M);
1286
1287 auto *TS =
1288 static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer());
1289 // Close the last emitted section
1290 if (hasDebugInfo()) {
1291 TS->closeLastSection();
1292 // Emit empty .debug_macinfo section for better support of the empty files.
1293 TS->emitEmptySectionDirective(Name: ".debug_macinfo");
1294 }
1295
1296 // Output last DWARF .file directives, if any.
1297 TS->outputDwarfFileDirectives();
1298
1299 return ret;
1300}
1301
1302// This function emits appropriate linkage directives for
1303// functions and global variables.
1304//
1305// extern function declaration -> .extern
1306// extern function definition -> .visible
1307// external global variable with init -> .visible
1308// external without init -> .extern
1309// appending -> not allowed, assert.
1310// for any linkage other than
1311// internal, private, linker_private,
1312// linker_private_weak, linker_private_weak_def_auto,
1313// we emit -> .weak.
1314
1315void NVPTXAsmPrinter::emitLinkageDirective(const GlobalValue *V,
1316 raw_ostream &O) {
1317 if (static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() == NVPTX::CUDA) {
1318 if (V->hasExternalLinkage()) {
1319 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: V))
1320 O << (GVar->hasInitializer() ? ".visible " : ".extern ");
1321 else if (V->isDeclaration())
1322 O << ".extern ";
1323 else
1324 O << ".visible ";
1325 } else if (V->hasAppendingLinkage()) {
1326 report_fatal_error(reason: "Symbol '" + (V->hasName() ? V->getName() : "") +
1327 "' has unsupported appending linkage type");
1328 } else if (!V->hasInternalLinkage() && !V->hasPrivateLinkage()) {
1329 O << ".weak ";
1330 }
1331 }
1332}
1333
1334void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
1335 raw_ostream &O, bool ProcessDemoted,
1336 const NVPTXSubtarget &STI) {
1337 // Skip metadata and LLVM intrinsic global variables.
1338 if (shouldSkipModuleLevelGlobal(GV: *GVar))
1339 return;
1340
1341 if (GVar->hasExternalLinkage()) {
1342 if (GVar->hasInitializer())
1343 O << ".visible ";
1344 else
1345 O << ".extern ";
1346 } else if (STI.hasFeature(Feature: NVPTX::PTX50) && GVar->hasCommonLinkage() &&
1347 GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) {
1348 O << ".common ";
1349 } else if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() ||
1350 GVar->hasAvailableExternallyLinkage() ||
1351 GVar->hasCommonLinkage()) {
1352 O << ".weak ";
1353 }
1354
1355 const PTXOpaqueType OpaqueType = getPTXOpaqueType(*GVar);
1356
1357 if (OpaqueType == PTXOpaqueType::Texture) {
1358 O << ".global .texref ";
1359 getSymbol(GV: GVar)->print(OS&: O, MAI);
1360 O << ";\n";
1361 return;
1362 }
1363
1364 if (OpaqueType == PTXOpaqueType::Surface) {
1365 O << ".global .surfref ";
1366 getSymbol(GV: GVar)->print(OS&: O, MAI);
1367 O << ";\n";
1368 return;
1369 }
1370
1371 if (GVar->isDeclaration()) {
1372 // (extern) declarations, no definition or initializer
1373 // Currently the only known declaration is for an automatic __local
1374 // (.shared) promoted to global.
1375 emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/false);
1376 O << ";\n";
1377 return;
1378 }
1379
1380 if (OpaqueType == PTXOpaqueType::Sampler) {
1381 O << ".global .samplerref ";
1382 getSymbol(GV: GVar)->print(OS&: O, MAI);
1383
1384 const Constant *Initializer = nullptr;
1385 if (GVar->hasInitializer())
1386 Initializer = GVar->getInitializer();
1387 const ConstantInt *CI = nullptr;
1388 if (Initializer)
1389 CI = dyn_cast<ConstantInt>(Val: Initializer);
1390 if (CI) {
1391 unsigned sample = CI->getZExtValue();
1392
1393 O << " = { ";
1394
1395 for (int i = 0,
1396 addr = ((sample & __CLK_ADDRESS_MASK) >> __CLK_ADDRESS_BASE);
1397 i < 3; i++) {
1398 O << "addr_mode_" << i << " = ";
1399 switch (addr) {
1400 case 0:
1401 O << "wrap";
1402 break;
1403 case 1:
1404 O << "clamp_to_border";
1405 break;
1406 case 2:
1407 O << "clamp_to_edge";
1408 break;
1409 case 3:
1410 O << "wrap";
1411 break;
1412 case 4:
1413 O << "mirror";
1414 break;
1415 }
1416 O << ", ";
1417 }
1418 O << "filter_mode = ";
1419 switch ((sample & __CLK_FILTER_MASK) >> __CLK_FILTER_BASE) {
1420 case 0:
1421 O << "nearest";
1422 break;
1423 case 1:
1424 O << "linear";
1425 break;
1426 case 2:
1427 llvm_unreachable("Anisotropic filtering is not supported");
1428 default:
1429 O << "nearest";
1430 break;
1431 }
1432 if (!((sample & __CLK_NORMALIZED_MASK) >> __CLK_NORMALIZED_BASE)) {
1433 O << ", force_unnormalized_coords = 1";
1434 }
1435 O << " }";
1436 }
1437
1438 O << ";\n";
1439 return;
1440 }
1441
1442 if (GVar->hasPrivateLinkage()) {
1443 if (GVar->getName().starts_with(Prefix: "unrollpragma"))
1444 return;
1445
1446 // FIXME - need better way (e.g. Metadata) to avoid generating this global
1447 if (GVar->getName().starts_with(Prefix: "filename"))
1448 return;
1449 if (GVar->use_empty())
1450 return;
1451 }
1452
1453 const Function *DemotedFunc = nullptr;
1454 if (!ProcessDemoted && canDemoteGlobalVar(GV: GVar, f&: DemotedFunc)) {
1455 O << "// " << GVar->getName() << " has been demoted\n";
1456 localDecls[DemotedFunc].push_back(x: GVar);
1457 return;
1458 }
1459
1460 emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/true);
1461 O << ";\n";
1462}
1463
1464template <typename T>
1465void NVPTXAsmPrinter::printFunctionDirectives(const T &V,
1466 raw_ostream &O) const {
1467 const NVPTXSubtarget &STI =
1468 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
1469
1470 if (STI.hasNoReturn() && V.doesNotReturn() &&
1471 V.getFunctionType()->getReturnType()->isVoidTy())
1472 O << ".noreturn\n";
1473
1474 // Don't emit abi_preserve directives on targets where they are
1475 // unsupported.
1476 if (!STI.hasABIPreserve())
1477 return;
1478
1479 const ABIPreserveInfo ABI = getABIPreserve(V.getAttributes());
1480 if (ABI.Preserve)
1481 O << ".abi_preserve " << *ABI.Preserve << "\n";
1482 if (ABI.PreserveControl)
1483 O << ".abi_preserve_control " << *ABI.PreserveControl << "\n";
1484}
1485
1486void NVPTXAsmPrinter::emitPTXGlobalVariableDefinition(
1487 const GlobalVariable *GVar, raw_ostream &O, const NVPTXSubtarget &STI,
1488 bool EmitInitializer) {
1489 const DataLayout &DL = getDataLayout();
1490
1491 Type *ETy = GVar->getValueType();
1492
1493 emitPTXAddressSpace(AddressSpace: GVar->getAddressSpace(), O);
1494
1495 if (isManaged(*GVar)) {
1496 if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30))
1497 report_fatal_error(
1498 reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1499 O << " .attribute(.managed)";
1500 }
1501
1502 O << " .align "
1503 << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value();
1504
1505 const Constant *Initializer = nullptr;
1506 if (GVar->hasInitializer()) {
1507 const Constant *Init = GVar->getInitializer();
1508 if (!Init->isNullValue() && !isa<UndefValue>(Val: Init)) {
1509 if (GVar->getAddressSpace() != ADDRESS_SPACE_GLOBAL &&
1510 GVar->getAddressSpace() != ADDRESS_SPACE_CONST)
1511 report_fatal_error(reason: "initial value of '" + GVar->getName() +
1512 "' is not allowed in addrspace(" +
1513 Twine(GVar->getAddressSpace()) + ")");
1514 Initializer = Init;
1515 }
1516 }
1517
1518 if (ETy->isPointerTy() || ((ETy->isIntegerTy() || ETy->isFloatingPointTy()) &&
1519 ETy->getScalarSizeInBits() <= 64)) {
1520 O << " ." << getPTXFundamentalTypeStr(Ty: ETy) << " ";
1521 getSymbol(GV: GVar)->print(OS&: O, MAI);
1522
1523 if (EmitInitializer && Initializer) {
1524 O << " = ";
1525 printScalarConstant(CPV: Initializer, O);
1526 }
1527 return;
1528 }
1529
1530 // Although PTX has direct support for struct type and array type and LLVM IR
1531 // is very similar to PTX, the LLVM CodeGen does not support for targets that
1532 // support these high level field accesses. Structs, arrays and vectors are
1533 // lowered into arrays of bytes.
1534 assert((ETy->isIntegerTy() || ETy->isFP128Ty() || ETy->isAggregateType() ||
1535 isa<FixedVectorType>(ETy)) &&
1536 "type not supported yet");
1537
1538 const uint64_t ElementSize = DL.getTypeStoreSize(Ty: ETy);
1539
1540 if (!Initializer) {
1541 O << " .b8 ";
1542 getSymbol(GV: GVar)->print(OS&: O, MAI);
1543 if (ElementSize)
1544 O << "[" << ElementSize << "]";
1545 else if (!EmitInitializer)
1546 O << "[]";
1547 return;
1548 }
1549
1550 AggBuffer aggBuffer(ElementSize, *this);
1551 bufferAggregateConstant(CV: Initializer, aggBuffer: &aggBuffer);
1552 if (aggBuffer.numSymbols()) {
1553 const unsigned int ptrSize = MAI.getCodePointerSize();
1554 if (ElementSize % ptrSize || !aggBuffer.allSymbolsAligned(ptrSize)) {
1555 // Print in bytes and use the mask() operator for pointers.
1556 if (!STI.hasMaskOperator())
1557 report_fatal_error(reason: "initialized packed aggregate with pointers '" +
1558 GVar->getName() +
1559 "' requires at least PTX ISA version 7.1");
1560 O << " .u8 ";
1561 getSymbol(GV: GVar)->print(OS&: O, MAI);
1562 O << "[" << ElementSize << "]";
1563 if (EmitInitializer) {
1564 O << " = {";
1565 aggBuffer.printBytes(os&: O);
1566 O << "}";
1567 }
1568 } else {
1569 O << " .u" << ptrSize * 8 << " ";
1570 getSymbol(GV: GVar)->print(OS&: O, MAI);
1571 O << "[" << ElementSize / ptrSize << "]";
1572 if (EmitInitializer) {
1573 O << " = {";
1574 aggBuffer.printWords(os&: O);
1575 O << "}";
1576 }
1577 }
1578 } else {
1579 O << " .b8 ";
1580 getSymbol(GV: GVar)->print(OS&: O, MAI);
1581 O << "[" << ElementSize << "]";
1582 if (EmitInitializer) {
1583 O << " = {";
1584 aggBuffer.printBytes(os&: O);
1585 O << "}";
1586 }
1587 }
1588}
1589
1590void NVPTXAsmPrinter::AggBuffer::printSymbol(unsigned nSym, raw_ostream &os) {
1591 const Value *v = Symbols[nSym];
1592 const Value *v0 = SymbolsBeforeStripping[nSym];
1593 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: v)) {
1594 MCSymbol *Name = AP.getSymbol(GV: GVar);
1595 PointerType *PTy = dyn_cast<PointerType>(Val: v0->getType());
1596 // Is v0 a generic pointer?
1597 bool isGenericPointer = PTy && PTy->getAddressSpace() == 0;
1598 if (EmitGeneric && isGenericPointer && !isa<Function>(Val: v)) {
1599 os << "generic(";
1600 Name->print(OS&: os, MAI: AP.MAI);
1601 os << ")";
1602 } else {
1603 Name->print(OS&: os, MAI: AP.MAI);
1604 }
1605 } else if (const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(Val: v0)) {
1606 const MCExpr *Expr = AP.lowerConstantForGV(CV: CExpr, ProcessingGeneric: false);
1607 AP.printMCExpr(Expr: *Expr, OS&: os);
1608 } else
1609 llvm_unreachable("symbol type unknown");
1610}
1611
1612void NVPTXAsmPrinter::AggBuffer::printBytes(raw_ostream &os) {
1613 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1614 // Do not emit trailing zero initializers. They will be zero-initialized by
1615 // ptxas. This saves on both space requirements for the generated PTX and on
1616 // memory use by ptxas. (See:
1617 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#global-state-space)
1618 unsigned int InitializerCount = Size;
1619 // TODO: symbols make this harder, but it would still be good to trim trailing
1620 // 0s for aggs with symbols as well.
1621 if (numSymbols() == 0)
1622 while (InitializerCount >= 1 && !buffer[InitializerCount - 1])
1623 InitializerCount--;
1624
1625 symbolPosInBuffer.push_back(Elt: InitializerCount);
1626 unsigned int nSym = 0;
1627 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1628 for (unsigned int pos = 0; pos < InitializerCount;) {
1629 if (pos)
1630 os << ", ";
1631 if (pos != nextSymbolPos) {
1632 os << (unsigned int)buffer[pos];
1633 ++pos;
1634 continue;
1635 }
1636 // Generate a per-byte mask() operator for the symbol, which looks like:
1637 // .global .u8 addr[] = {0xFF(foo), 0xFF00(foo), 0xFF0000(foo), ...};
1638 // See https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#initializers
1639 std::string symText;
1640 llvm::raw_string_ostream oss(symText);
1641 printSymbol(nSym, os&: oss);
1642 for (unsigned i = 0; i < ptrSize; ++i) {
1643 if (i)
1644 os << ", ";
1645 llvm::write_hex(S&: os, N: 0xFFULL << i * 8, Style: HexPrintStyle::PrefixUpper);
1646 os << "(" << symText << ")";
1647 }
1648 pos += ptrSize;
1649 nextSymbolPos = symbolPosInBuffer[++nSym];
1650 assert(nextSymbolPos >= pos);
1651 }
1652}
1653
1654void NVPTXAsmPrinter::AggBuffer::printWords(raw_ostream &os) {
1655 unsigned int ptrSize = AP.MAI.getCodePointerSize();
1656 symbolPosInBuffer.push_back(Elt: Size);
1657 unsigned int nSym = 0;
1658 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1659 assert(nextSymbolPos % ptrSize == 0);
1660 for (unsigned int pos = 0; pos < Size; pos += ptrSize) {
1661 if (pos)
1662 os << ", ";
1663 if (pos == nextSymbolPos) {
1664 printSymbol(nSym, os);
1665 nextSymbolPos = symbolPosInBuffer[++nSym];
1666 assert(nextSymbolPos % ptrSize == 0);
1667 assert(nextSymbolPos >= pos + ptrSize);
1668 } else if (ptrSize == 4)
1669 os << support::endian::read32le(P: &buffer[pos]);
1670 else
1671 os << support::endian::read64le(P: &buffer[pos]);
1672 }
1673}
1674
1675void NVPTXAsmPrinter::emitDemotedVars(const Function *F, raw_ostream &O) {
1676 auto It = localDecls.find(x: F);
1677 if (It == localDecls.end())
1678 return;
1679
1680 ArrayRef<const GlobalVariable *> GVars = It->second;
1681
1682 const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
1683 const NVPTXSubtarget &STI = *NTM.getSubtargetImpl();
1684
1685 for (const GlobalVariable *GV : GVars) {
1686 O << "\t// demoted variable\n\t";
1687 printModuleLevelGV(GVar: GV, O, /*processDemoted=*/ProcessDemoted: true, STI);
1688 }
1689}
1690
1691/// The PTX state space directive for \p AddressSpace, or an empty string if it
1692/// does not name one, as is the case for the generic address space.
1693static StringRef getPTXAddressSpaceName(unsigned AddressSpace) {
1694 switch (AddressSpace) {
1695 case ADDRESS_SPACE_LOCAL:
1696 return ".local";
1697 case ADDRESS_SPACE_GLOBAL:
1698 return ".global";
1699 case ADDRESS_SPACE_CONST:
1700 return ".const";
1701 case ADDRESS_SPACE_SHARED:
1702 return ".shared";
1703 default:
1704 return {};
1705 }
1706}
1707
1708/// The PTX opaque type directive for an image or sampler handle, or an empty
1709/// string for PTXOpaqueType::None.
1710static StringRef getPTXOpaqueTypeName(PTXOpaqueType OpaqueType) {
1711 switch (OpaqueType) {
1712 case PTXOpaqueType::Sampler:
1713 return ".samplerref";
1714 case PTXOpaqueType::Texture:
1715 return ".texref";
1716 case PTXOpaqueType::Surface:
1717 return ".surfref";
1718 case PTXOpaqueType::None:
1719 return {};
1720 }
1721 llvm_unreachable("unexpected PTXOpaqueType");
1722}
1723
1724void NVPTXAsmPrinter::emitPTXAddressSpace(unsigned int AddressSpace,
1725 raw_ostream &O) const {
1726 const StringRef Name = getPTXAddressSpaceName(AddressSpace);
1727 if (Name.empty())
1728 report_fatal_error(reason: "Bad address space found while emitting PTX: " +
1729 llvm::Twine(AddressSpace));
1730 O << Name;
1731}
1732
1733std::string NVPTXAsmPrinter::getPTXFundamentalTypeStr(Type *Ty) const {
1734 switch (Ty->getTypeID()) {
1735 case Type::IntegerTyID:
1736 case Type::PointerTyID: {
1737 const uint64_t NumBits = getDataLayout().getTypeStoreSizeInBits(Ty);
1738 assert(NumBits <= 64 && "type too large");
1739 return "u" + utostr(X: promoteScalarKernelArgumentSize(Size: NumBits));
1740 }
1741 case Type::BFloatTyID:
1742 case Type::HalfTyID:
1743 case Type::FloatTyID:
1744 case Type::DoubleTyID:
1745 return "b" + utostr(X: Ty->getScalarSizeInBits());
1746 default:
1747 break;
1748 }
1749 llvm_unreachable("unexpected type");
1750}
1751
1752void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
1753 const DataLayout &DL = getDataLayout();
1754 const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F);
1755 const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering());
1756 const NVPTXMachineFunctionInfo *MFI =
1757 MF ? MF->getInfo<NVPTXMachineFunctionInfo>() : nullptr;
1758
1759 const bool IsKernelFunc = isKernelFunction(F: *F);
1760
1761 // Zero-sized arguments (e.g. empty structs) do not produce a parameter.
1762 // Number the emitted parameters contiguously, skipping the zero-sized ones,
1763 // so that the names match those used in LowerFormalArguments and the
1764 // contiguous numbering used by callers (see LowerCall).
1765 const auto NonEmptyArgs =
1766 make_filter_range(Range: F->args(), Pred: [](const Argument &Arg) {
1767 return !Arg.getType()->isEmptyTy();
1768 });
1769
1770 if (NonEmptyArgs.empty() && !F->isVarArg()) {
1771 O << "()";
1772 return;
1773 }
1774
1775 O << "(\n";
1776
1777 auto MakeParam = [&](const auto &IndexedArg) {
1778 const auto &[ParamIndex, Arg] = IndexedArg;
1779 Type *Ty = Arg.getType();
1780 MCSymbol *const ParamSym = TLI->getParamSymbol(Ctx&: OutContext, F, Idx: ParamIndex);
1781
1782 O << "\t";
1783
1784 // A byval param is passed as a copy of the pointee and an aggregate is
1785 // passed as a blob of bytes; both are declared as a byte array.
1786 const bool IsByVal = Arg.hasByValAttr();
1787 const bool AsArray = IsByVal || shouldPassAsArray(Ty);
1788
1789 // Kernels declare image/sampler handles and the address space of a
1790 // pointee. Both of those are scalar handles, so a byte-array param is
1791 // neither.
1792 if (IsKernelFunc && !AsArray) {
1793 const StringRef OpaqueType = getPTXOpaqueTypeName(getPTXOpaqueType(Arg));
1794 if (!OpaqueType.empty()) {
1795 O << ".param ";
1796 if (!MFI || !MFI->checkImageHandleSymbol(Symbol: ParamSym))
1797 O << ".u64 .ptr ";
1798
1799 O << OpaqueType << " " << *ParamSym;
1800 return;
1801 }
1802
1803 if (auto *PTy = dyn_cast<PointerType>(Val: Ty)) {
1804 const unsigned AS = PTy->getAddressSpace();
1805 O << ".param .u" << DL.getPointerSizeInBits(AS) << " .ptr";
1806
1807 const StringRef Space = getPTXAddressSpaceName(AddressSpace: AS);
1808 if (!Space.empty())
1809 O << " " << Space;
1810
1811 O << " .align " << Arg.getParamAlign().valueOrOne().value() << " "
1812 << *ParamSym;
1813 return;
1814 }
1815 }
1816
1817 printParam(F, IsByVal ? Arg.getParamByValType() : Ty,
1818 Arg.getArgNo() + AttributeList::FirstArgIndex, IsByVal,
1819 IsKernelFunc, ParamSym->getName(), DL, O);
1820 };
1821
1822 interleave(c: enumerate(First: NonEmptyArgs), os&: O, each_fn: MakeParam, separator: ",\n");
1823
1824 if (F->isVarArg())
1825 O << (NonEmptyArgs.empty() ? "" : ",\n") << "\t.param .align "
1826 << STI.getMaxRequiredAlignment() << " .b8 "
1827 << *TLI->getParamSymbol(Ctx&: OutContext, F, /* vararg */ Idx: -1) << "[]";
1828
1829 O << "\n)";
1830}
1831
1832void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
1833 const MachineFunction &MF) {
1834 auto *TS = getTargetStreamer();
1835
1836 // Emit the Fake Stack Object
1837 const MachineFrameInfo &MFI = MF.getFrameInfo();
1838 if (const int64_t NumBytes = MFI.getStackSize()) {
1839 TS->emitLocalDirective(Alignment: MFI.getMaxAlign(), Name: getFunctionFrameSymbol(),
1840 Size: NumBytes);
1841
1842 // Declare the frame pointers that NVPTXFrameLowering's prologue defines.
1843 const NVPTXRegisterInfo *NRI =
1844 MF.getSubtarget<NVPTXSubtarget>().getRegisterInfo();
1845 for (const Register FrameReg :
1846 {NRI->getFrameRegister(MF), NRI->getFrameLocalRegister(MF)})
1847 TS->emitRegDirective(
1848 SizeInBits: NRI->getRegSizeInBits(Reg: FrameReg, MRI: *MRI).getFixedValue(),
1849 Name: NVPTXInstPrinter::getRegisterName(Reg: FrameReg));
1850 }
1851
1852 // Go through all virtual registers to establish the mapping between the
1853 // global virtual
1854 // register number and the per class virtual register number.
1855 // We use the per class virtual register number in the ptx output.
1856 for (unsigned I : llvm::seq(Size: MRI->getNumVirtRegs())) {
1857 Register VR = Register::index2VirtReg(Index: I);
1858 if (MRI->use_empty(RegNo: VR) && MRI->def_empty(RegNo: VR))
1859 continue;
1860 auto &RCRegMap = VRegMapping[MRI->getRegClass(Reg: VR)];
1861 RCRegMap[VR] = RCRegMap.size() + 1;
1862 }
1863
1864 // Emit declaration of the virtual registers or 'physical' registers for
1865 // each register class
1866 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1867 for (const TargetRegisterClass &RC : TRI->regclasses()) {
1868 // Only declare those registers that may be used.
1869 const auto It = VRegMapping.find(Val: &RC);
1870 if (It == VRegMapping.end() || It->second.empty())
1871 continue;
1872
1873 TS->emitRegDirective(
1874 SizeInBits: TRI->getRegSizeInBits(RC).getFixedValue(),
1875 Name: NVPTX::getVirtualRegisterPrefix(Kind: getVirtualRegisterKind(RC: &RC)),
1876 Count: It->second.size() + 1);
1877 }
1878}
1879
1880/// Translate virtual register numbers in DebugInfo locations to their printed
1881/// encodings, as used by CUDA-GDB.
1882void NVPTXAsmPrinter::encodeDebugInfoRegisterNumbers(
1883 const MachineFunction &MF) {
1884 const NVPTXSubtarget &STI = MF.getSubtarget<NVPTXSubtarget>();
1885 const NVPTXRegisterInfo *NRI = STI.getRegisterInfo();
1886
1887 // Clear the old mapping, and add the new one. This mapping is used after the
1888 // printing of the current function is complete, but before the next function
1889 // is printed.
1890 NRI->clearDebugRegisterMap();
1891
1892 for (const VRegMap &RegMap : make_second_range(c&: VRegMapping))
1893 for (const Register Reg : make_first_range(c: RegMap))
1894 NRI->addToDebugRegisterMap(VirtReg: Reg, RegisterName: getVirtualRegisterName(Reg));
1895}
1896
1897void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp,
1898 raw_ostream &O) const {
1899 if (Fp->getType()->isFloatTy())
1900 O << "0f";
1901 else if (Fp->getType()->isDoubleTy())
1902 O << "0d";
1903 else
1904 llvm_unreachable("unsupported fp type");
1905
1906 const APInt API = Fp->getValueAPF().bitcastToAPInt();
1907 O << format_hex_no_prefix(N: API.getZExtValue(), Width: API.getBitWidth() / 4,
1908 /*Upper=*/true);
1909}
1910
1911void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) {
1912 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
1913 O << CI->getValue();
1914 return;
1915 }
1916 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
1917 const APInt API = CFP->getValueAPF().bitcastToAPInt();
1918 O << "0x"
1919 << format_hex_no_prefix(N: API.getZExtValue(), Width: API.getBitWidth() / 4,
1920 /*Upper=*/true);
1921 return;
1922 }
1923 if (isa<ConstantPointerNull>(Val: CPV)) {
1924 O << "0";
1925 return;
1926 }
1927 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
1928 const bool IsNonGenericPointer = GVar->getAddressSpace() != 0;
1929 if (EmitGeneric && !isa<Function>(Val: CPV) && !IsNonGenericPointer) {
1930 O << "generic(";
1931 getSymbol(GV: GVar)->print(OS&: O, MAI);
1932 O << ")";
1933 } else {
1934 getSymbol(GV: GVar)->print(OS&: O, MAI);
1935 }
1936 return;
1937 }
1938 if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
1939 const MCExpr *E = lowerConstantForGV(CV: cast<Constant>(Val: Cexpr), ProcessingGeneric: false);
1940 printMCExpr(Expr: *E, OS&: O);
1941 return;
1942 }
1943 llvm_unreachable("Not scalar type found in printScalarConstant()");
1944}
1945
1946void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
1947 AggBuffer *AggBuffer) {
1948 const DataLayout &DL = getDataLayout();
1949 int AllocSize = DL.getTypeAllocSize(Ty: CPV->getType());
1950 if (isa<UndefValue>(Val: CPV) || CPV->isNullValue()) {
1951 // Non-zero Bytes indicates that we need to zero-fill everything. Otherwise,
1952 // only the space allocated by CPV.
1953 AggBuffer->addZeros(Num: Bytes ? Bytes : AllocSize);
1954 return;
1955 }
1956
1957 // Helper for filling AggBuffer with APInts.
1958 auto AddIntToBuffer = [AggBuffer, Bytes](const APInt &Val) {
1959 size_t NumBytes = (Val.getBitWidth() + 7) / 8;
1960 SmallVector<unsigned char, 16> Buf(NumBytes);
1961 // `extractBitsAsZExtValue` does not allow the extraction of bits beyond the
1962 // input's bit width, and i1 arrays may not have a length that is a multuple
1963 // of 8. We handle the last byte separately, so we never request out of
1964 // bounds bits.
1965 for (unsigned I = 0; I < NumBytes - 1; ++I) {
1966 Buf[I] = Val.extractBitsAsZExtValue(numBits: 8, bitPosition: I * 8);
1967 }
1968 size_t LastBytePosition = (NumBytes - 1) * 8;
1969 size_t LastByteBits = Val.getBitWidth() - LastBytePosition;
1970 Buf[NumBytes - 1] =
1971 Val.extractBitsAsZExtValue(numBits: LastByteBits, bitPosition: LastBytePosition);
1972 AggBuffer->addBytes(Ptr: Buf.data(), Num: NumBytes, Bytes);
1973 };
1974
1975 switch (CPV->getType()->getTypeID()) {
1976 case Type::IntegerTyID:
1977 if (const auto *CI = dyn_cast<ConstantInt>(Val: CPV)) {
1978 AddIntToBuffer(CI->getValue());
1979 break;
1980 }
1981 if (const auto *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
1982 if (const auto *CI =
1983 dyn_cast<ConstantInt>(Val: ConstantFoldConstant(C: Cexpr, DL))) {
1984 AddIntToBuffer(CI->getValue());
1985 break;
1986 }
1987 if (Cexpr->getOpcode() == Instruction::PtrToInt) {
1988 Value *V = Cexpr->getOperand(i_nocapture: 0)->stripPointerCasts();
1989 AggBuffer->addSymbol(GVar: V, GVarBeforeStripping: Cexpr->getOperand(i_nocapture: 0));
1990 AggBuffer->addZeros(Num: AllocSize);
1991 break;
1992 }
1993 // A symbol-relative integer whose offset is applied outside the
1994 // ptrtoint, e.g. add(ptrtoint(@g), C). It can't fold to a ConstantInt
1995 // because it references a symbol; emit it through lowerConstantForGV, the
1996 // same path scalar symbol-relative integer globals use.
1997 AggBuffer->addSymbol(GVar: Cexpr, GVarBeforeStripping: Cexpr);
1998 AggBuffer->addZeros(Num: AllocSize);
1999 break;
2000 }
2001 llvm_unreachable("unsupported integer const type");
2002 break;
2003
2004 case Type::HalfTyID:
2005 case Type::BFloatTyID:
2006 case Type::FloatTyID:
2007 case Type::DoubleTyID:
2008 case Type::FP128TyID:
2009 AddIntToBuffer(cast<ConstantFP>(Val: CPV)->getValueAPF().bitcastToAPInt());
2010 break;
2011
2012 case Type::PointerTyID: {
2013 if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) {
2014 AggBuffer->addSymbol(GVar, GVarBeforeStripping: GVar);
2015 } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) {
2016 const Value *v = Cexpr->stripPointerCasts();
2017 AggBuffer->addSymbol(GVar: v, GVarBeforeStripping: Cexpr);
2018 }
2019 AggBuffer->addZeros(Num: AllocSize);
2020 break;
2021 }
2022
2023 case Type::ArrayTyID:
2024 case Type::FixedVectorTyID:
2025 case Type::StructTyID: {
2026 if (isa<ConstantAggregate, ConstantDataSequential, ConstantInt, ConstantFP>(
2027 Val: CPV)) {
2028 // bufferAggregateConstant doesn't emit tail-padding, i.e. it writes
2029 // `store_size` bytes, not `alloc_size` bytes. Do it ourselves here.
2030 unsigned StartPos = AggBuffer->getCurpos();
2031 bufferAggregateConstant(CV: CPV, aggBuffer: AggBuffer);
2032 unsigned Written = AggBuffer->getCurpos() - StartPos;
2033 unsigned SlotSize = std::max<int>(a: Bytes, b: AllocSize);
2034 assert(Written <= SlotSize && "Constant overflows its slot");
2035 if (SlotSize > Written)
2036 AggBuffer->addZeros(Num: SlotSize - Written);
2037 } else if (isa<ConstantAggregateZero>(Val: CPV))
2038 AggBuffer->addZeros(Num: Bytes);
2039 else
2040 llvm_unreachable("Unexpected Constant type");
2041 break;
2042 }
2043
2044 default:
2045 llvm_unreachable("unsupported type");
2046 }
2047}
2048
2049void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
2050 AggBuffer *aggBuffer) {
2051 const DataLayout &DL = getDataLayout();
2052
2053 auto ExtendBuffer = [](APInt Val, AggBuffer *Buffer) {
2054 unsigned NumBytes = divideCeil(Numerator: Val.getBitWidth(), Denominator: 8);
2055 for (unsigned I : llvm::seq(Size: NumBytes)) {
2056 unsigned NumBits = std::min(a: 8u, b: Val.getBitWidth() - I * 8);
2057 Buffer->addByte(Byte: Val.extractBitsAsZExtValue(numBits: NumBits, bitPosition: I * 8));
2058 }
2059 };
2060
2061 if (auto *VTy = dyn_cast<FixedVectorType>(Val: CPV->getType())) {
2062 // Elements that aren't a whole number of bytes are bit-packed, so buffer
2063 // the vector as one integer.
2064 if (!DL.typeSizeEqualsStoreSize(Ty: VTy->getElementType())) {
2065 Type *IntTy =
2066 IntegerType::get(C&: CPV->getContext(), NumBits: VTy->getPrimitiveSizeInBits());
2067 auto *Packed = dyn_cast<ConstantInt>(Val: ConstantFoldConstant(
2068 C: ConstantExpr::getBitCast(C: const_cast<Constant *>(CPV), Ty: IntTy), DL));
2069 if (!Packed)
2070 report_fatal_error(
2071 reason: "Cannot lower vector global with unusual element type");
2072 ExtendBuffer(Packed->getValue(), aggBuffer);
2073 return;
2074 }
2075
2076 // Integer or floating point vector splats.
2077 if (isa<ConstantInt, ConstantFP>(Val: CPV)) {
2078 Constant *Splat = CPV->getSplatValue();
2079 for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I)
2080 bufferLEByte(CPV: Splat, Bytes: 0, AggBuffer: aggBuffer);
2081 return;
2082 }
2083 }
2084
2085 // Integers of arbitrary width
2086 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) {
2087 assert(CI->getType()->isIntegerTy() && "Expected integer constant!");
2088 ExtendBuffer(CI->getValue(), aggBuffer);
2089 return;
2090 }
2091
2092 // f128
2093 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) {
2094 assert(CFP->getType()->isFloatingPointTy() && "Expected fp constant!");
2095 if (CFP->getType()->isFP128Ty()) {
2096 ExtendBuffer(CFP->getValueAPF().bitcastToAPInt(), aggBuffer);
2097 return;
2098 }
2099 }
2100
2101 // Buffer arrays and vectors one element at a time.
2102 if (isa<ConstantArray, ConstantVector>(Val: CPV)) {
2103 for (const auto &Op : CPV->operands())
2104 bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer);
2105 return;
2106 }
2107
2108 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: CPV)) {
2109 for (unsigned I : llvm::seq(Size: CDS->getNumElements()))
2110 bufferLEByte(CPV: cast<Constant>(Val: CDS->getElementAsConstant(i: I)), Bytes: 0, AggBuffer: aggBuffer);
2111 return;
2112 }
2113
2114 if (isa<ConstantStruct>(Val: CPV)) {
2115 if (CPV->getNumOperands()) {
2116 StructType *ST = cast<StructType>(Val: CPV->getType());
2117 for (unsigned I : llvm::seq(Size: CPV->getNumOperands())) {
2118 int EndOffset = (I + 1 == CPV->getNumOperands())
2119 ? DL.getStructLayout(Ty: ST)->getElementOffset(Idx: 0) +
2120 DL.getTypeAllocSize(Ty: ST)
2121 : DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I + 1);
2122 int Bytes = EndOffset - DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I);
2123 bufferLEByte(CPV: cast<Constant>(Val: CPV->getOperand(i: I)), Bytes, AggBuffer: aggBuffer);
2124 }
2125 }
2126 return;
2127 }
2128 llvm_unreachable("unsupported constant type in printAggregateConstant()");
2129}
2130
2131/// lowerConstantForGV - Return an MCExpr for the given Constant. This is mostly
2132/// a copy from AsmPrinter::lowerConstant, except customized to only handle
2133/// expressions that are representable in PTX and create
2134/// NVPTXGenericMCSymbolRefExpr nodes for addrspacecast instructions.
2135const MCExpr *
2136NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV,
2137 bool ProcessingGeneric) const {
2138 MCContext &Ctx = OutContext;
2139
2140 if (CV->isNullValue() || isa<UndefValue>(Val: CV))
2141 return MCConstantExpr::create(Value: 0, Ctx);
2142
2143 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CV))
2144 return MCConstantExpr::create(Value: CI->getZExtValue(), Ctx);
2145
2146 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
2147 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Symbol: getSymbol(GV), Ctx);
2148 if (ProcessingGeneric)
2149 return NVPTXGenericMCSymbolRefExpr::create(SymExpr: Expr, Ctx);
2150 return Expr;
2151 }
2152
2153 const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: CV);
2154 if (!CE) {
2155 llvm_unreachable("Unknown constant value to lower!");
2156 }
2157
2158 switch (CE->getOpcode()) {
2159 default:
2160 break; // Error
2161
2162 case Instruction::AddrSpaceCast: {
2163 // Strip the addrspacecast and pass along the operand
2164 PointerType *DstTy = cast<PointerType>(Val: CE->getType());
2165 if (DstTy->getAddressSpace() == 0)
2166 return lowerConstantForGV(CV: cast<const Constant>(Val: CE->getOperand(i_nocapture: 0)), ProcessingGeneric: true);
2167
2168 break; // Error
2169 }
2170
2171 case Instruction::GetElementPtr: {
2172 const DataLayout &DL = getDataLayout();
2173
2174 // Generate a symbolic expression for the byte address
2175 APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0);
2176 cast<GEPOperator>(Val: CE)->accumulateConstantOffset(DL, Offset&: OffsetAI);
2177
2178 const MCExpr *Base = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0),
2179 ProcessingGeneric);
2180 if (!OffsetAI)
2181 return Base;
2182
2183 int64_t Offset = OffsetAI.getSExtValue();
2184 return MCBinaryExpr::createAdd(LHS: Base, RHS: MCConstantExpr::create(Value: Offset, Ctx),
2185 Ctx);
2186 }
2187
2188 case Instruction::Trunc:
2189 // We emit the value and depend on the assembler to truncate the generated
2190 // expression properly. This is important for differences between
2191 // blockaddress labels. Since the two labels are in the same function, it
2192 // is reasonable to treat their delta as a 32-bit value.
2193 [[fallthrough]];
2194 case Instruction::BitCast:
2195 return lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2196
2197 case Instruction::IntToPtr: {
2198 const DataLayout &DL = getDataLayout();
2199
2200 // Handle casts to pointers by changing them into casts to the appropriate
2201 // integer type. This promotes constant folding and simplifies this code.
2202 Constant *Op = CE->getOperand(i_nocapture: 0);
2203 Op = ConstantFoldIntegerCast(C: Op, DestTy: DL.getIntPtrType(CV->getType()),
2204 /*IsSigned*/ false, DL);
2205 if (Op)
2206 return lowerConstantForGV(CV: Op, ProcessingGeneric);
2207
2208 break; // Error
2209 }
2210
2211 case Instruction::PtrToInt: {
2212 const DataLayout &DL = getDataLayout();
2213
2214 // Support only foldable casts to/from pointers that can be eliminated by
2215 // changing the pointer to the appropriately sized integer type.
2216 Constant *Op = CE->getOperand(i_nocapture: 0);
2217 Type *Ty = CE->getType();
2218
2219 const MCExpr *OpExpr = lowerConstantForGV(CV: Op, ProcessingGeneric);
2220
2221 // We can emit the pointer value into this slot if the slot is an
2222 // integer slot equal to the size of the pointer.
2223 if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Ty: Op->getType()))
2224 return OpExpr;
2225
2226 // Otherwise the pointer is smaller than the resultant integer, mask off
2227 // the high bits so we are sure to get a proper truncation if the input is
2228 // a constant expr.
2229 unsigned InBits = DL.getTypeAllocSizeInBits(Ty: Op->getType());
2230 const MCExpr *MaskExpr = MCConstantExpr::create(Value: ~0ULL >> (64-InBits), Ctx);
2231 return MCBinaryExpr::createAnd(LHS: OpExpr, RHS: MaskExpr, Ctx);
2232 }
2233
2234 // The MC library also has a right-shift operator, but it isn't consistently
2235 // signed or unsigned between different targets.
2236 case Instruction::Add: {
2237 const MCExpr *LHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric);
2238 const MCExpr *RHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 1), ProcessingGeneric);
2239 switch (CE->getOpcode()) {
2240 default: llvm_unreachable("Unknown binary operator constant cast expr");
2241 case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx);
2242 }
2243 }
2244 }
2245
2246 // If the code isn't optimized, there may be outstanding folding
2247 // opportunities. Attempt to fold the expression using DataLayout as a
2248 // last resort before giving up.
2249 Constant *C = ConstantFoldConstant(C: CE, DL: getDataLayout());
2250 if (C != CE)
2251 return lowerConstantForGV(CV: C, ProcessingGeneric);
2252
2253 // Otherwise report the problem to the user.
2254 std::string S;
2255 raw_string_ostream OS(S);
2256 OS << "Unsupported expression in static initializer: ";
2257 CE->printAsOperand(O&: OS, /*PrintType=*/false,
2258 M: !MF ? nullptr : MF->getFunction().getParent());
2259 report_fatal_error(reason: Twine(OS.str()));
2260}
2261
2262void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) const {
2263 OutContext.getAsmInfo().printExpr(OS, Expr);
2264}
2265
2266/// PrintAsmOperand - Print out an operand for an inline asm expression.
2267///
2268bool NVPTXAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
2269 const char *ExtraCode, raw_ostream &O) {
2270 if (ExtraCode && ExtraCode[0]) {
2271 if (ExtraCode[1] != 0)
2272 return true; // Unknown modifier.
2273
2274 switch (ExtraCode[0]) {
2275 default:
2276 // See if this is a generic print operand
2277 return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O);
2278 case 'r':
2279 break;
2280 }
2281 }
2282
2283 printOperand(MI, OpNum: OpNo, O);
2284
2285 return false;
2286}
2287
2288bool NVPTXAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
2289 unsigned OpNo,
2290 const char *ExtraCode,
2291 raw_ostream &O) {
2292 if (ExtraCode && ExtraCode[0])
2293 return true; // Unknown modifier
2294
2295 O << '[';
2296 printMemOperand(MI, OpNum: OpNo, O);
2297 O << ']';
2298
2299 return false;
2300}
2301
2302void NVPTXAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
2303 raw_ostream &O) {
2304 const MachineOperand &MO = MI->getOperand(i: OpNum);
2305 switch (MO.getType()) {
2306 case MachineOperand::MO_Register:
2307 if (MO.getReg().isPhysical()) {
2308 if (MO.getReg() == NVPTX::VRDepot)
2309 getFunctionFrameSymbol()->print(OS&: O, MAI);
2310 else
2311 O << NVPTXInstPrinter::getRegisterName(Reg: MO.getReg());
2312 } else {
2313 O << getVirtualRegisterName(Reg: MO.getReg());
2314 }
2315 break;
2316
2317 case MachineOperand::MO_Immediate:
2318 O << MO.getImm();
2319 break;
2320
2321 case MachineOperand::MO_FPImmediate:
2322 printFPConstant(Fp: MO.getFPImm(), O);
2323 break;
2324
2325 case MachineOperand::MO_GlobalAddress:
2326 PrintSymbolOperand(MO, OS&: O);
2327 break;
2328
2329 case MachineOperand::MO_MCSymbol:
2330 MO.getMCSymbol()->print(OS&: O, MAI);
2331 break;
2332
2333 case MachineOperand::MO_MachineBasicBlock:
2334 MO.getMBB()->getSymbol()->print(OS&: O, MAI);
2335 break;
2336
2337 default:
2338 llvm_unreachable("Operand type not supported.");
2339 }
2340}
2341
2342void NVPTXAsmPrinter::printMemOperand(const MachineInstr *MI, unsigned OpNum,
2343 raw_ostream &O, const char *Modifier) {
2344 printOperand(MI, OpNum, O);
2345
2346 if (Modifier && strcmp(s1: Modifier, s2: "add") == 0) {
2347 O << ", ";
2348 printOperand(MI, OpNum: OpNum + 1, O);
2349 } else {
2350 if (MI->getOperand(i: OpNum + 1).isImm() &&
2351 MI->getOperand(i: OpNum + 1).getImm() == 0)
2352 return; // don't print ',0' or '+0'
2353 O << "+";
2354 printOperand(MI, OpNum: OpNum + 1, O);
2355 }
2356}
2357
2358/// Returns true if \p Line begins with an alphabetic character or underscore,
2359/// indicating it is a PTX instruction that should receive a .loc directive.
2360static bool isPTXInstruction(StringRef Line) {
2361 StringRef Trimmed = Line.ltrim();
2362 return !Trimmed.empty() &&
2363 (std::isalpha(static_cast<unsigned char>(Trimmed[0])) ||
2364 Trimmed[0] == '_');
2365}
2366
2367/// Returns the DILocation for an inline asm MachineInstr if debug line info
2368/// should be emitted, or nullptr otherwise.
2369static const DILocation *getInlineAsmDebugLoc(const MachineInstr *MI) {
2370 if (!MI || !MI->getDebugLoc())
2371 return nullptr;
2372 const DISubprogram *SP = MI->getMF()->getFunction().getSubprogram();
2373 if (!SP || SP->getUnit()->getEmissionKind() == DICompileUnit::NoDebug)
2374 return nullptr;
2375 const DILocation *DL = MI->getDebugLoc();
2376 if (!DL->getFile() || !DL->getLine() || DL->isImplicitCode())
2377 return nullptr;
2378 return DL;
2379}
2380
2381namespace {
2382struct InlineAsmInliningContext {
2383 MCSymbol *FuncNameSym = nullptr;
2384 unsigned FileIA = 0;
2385 unsigned LineIA = 0;
2386 unsigned ColIA = 0;
2387
2388 bool hasInlinedAt() const { return FuncNameSym != nullptr; }
2389};
2390} // namespace
2391
2392/// Resolves the enhanced-lineinfo inlining context for an inline asm debug
2393/// location. Returns a default (empty) context if inlining info is unavailable.
2394static InlineAsmInliningContext
2395getInlineAsmInliningContext(const DILocation *DL, const MachineFunction &MF,
2396 NVPTXDwarfDebug *NVDD, MCStreamer &Streamer,
2397 unsigned CUID) {
2398 InlineAsmInliningContext Ctx;
2399 const DILocation *InlinedAt = DL->getInlinedAt();
2400 if (!InlinedAt || !InlinedAt->getFile() || !NVDD ||
2401 !NVDD->isEnhancedLineinfo(MF))
2402 return Ctx;
2403 const auto *SubProg = getDISubprogram(Scope: DL->getScope());
2404 if (!SubProg)
2405 return Ctx;
2406 Ctx.FuncNameSym = NVDD->getOrCreateFuncNameSymbol(LinkageName: SubProg->getLinkageName());
2407 Ctx.FileIA = Streamer.emitDwarfFileDirective(
2408 FileNo: 0, Directory: InlinedAt->getFile()->getDirectory(),
2409 Filename: InlinedAt->getFile()->getFilename(), Checksum: std::nullopt, Source: std::nullopt, CUID);
2410 Ctx.LineIA = InlinedAt->getLine();
2411 Ctx.ColIA = InlinedAt->getColumn();
2412 return Ctx;
2413}
2414
2415void NVPTXAsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
2416 const MCTargetOptions &MCOptions,
2417 const MDNode *LocMDNode,
2418 InlineAsm::AsmDialect Dialect,
2419 const MachineInstr *MI) {
2420 assert(!Str.empty() && "Can't emit empty inline asm block");
2421 if (Str.back() == 0)
2422 Str = Str.substr(Start: 0, N: Str.size() - 1);
2423
2424 auto emitAsmStr = [&](StringRef AsmStr) {
2425 emitInlineAsmStart();
2426 OutStreamer->emitRawText(String: AsmStr);
2427 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2428 };
2429
2430 const DILocation *DL = getInlineAsmDebugLoc(MI);
2431 if (!DL) {
2432 emitAsmStr(Str);
2433 return;
2434 }
2435
2436 const DIFile *File = DL->getFile();
2437 unsigned Line = DL->getLine();
2438 const unsigned Column = DL->getColumn();
2439 const unsigned CUID = OutStreamer->getContext().getDwarfCompileUnitID();
2440 const unsigned FileNumber = OutStreamer->emitDwarfFileDirective(
2441 FileNo: 0, Directory: File->getDirectory(), Filename: File->getFilename(), Checksum: std::nullopt, Source: std::nullopt,
2442 CUID);
2443
2444 auto *NVDD = static_cast<NVPTXDwarfDebug *>(getDwarfDebug());
2445 InlineAsmInliningContext InlineCtx =
2446 getInlineAsmInliningContext(DL, MF: *MI->getMF(), NVDD, Streamer&: *OutStreamer, CUID);
2447
2448 SmallVector<StringRef, 16> Lines;
2449 Str.split(A&: Lines, Separator: '\n');
2450 emitInlineAsmStart();
2451 for (const StringRef &L : Lines) {
2452 StringRef RTrimmed = L.rtrim(Char: '\r');
2453 if (isPTXInstruction(Line: L)) {
2454 if (InlineCtx.hasInlinedAt()) {
2455 OutStreamer->emitDwarfLocDirectiveWithInlinedAt(
2456 FileNo: FileNumber, Line, Column, FileIA: InlineCtx.FileIA, LineIA: InlineCtx.LineIA,
2457 ColumnIA: InlineCtx.ColIA, Sym: InlineCtx.FuncNameSym, DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2458 FileName: File->getFilename());
2459 } else {
2460 OutStreamer->emitDwarfLocDirective(FileNo: FileNumber, Line, Column,
2461 DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0,
2462 FileName: File->getFilename());
2463 }
2464 }
2465 OutStreamer->emitRawText(String: RTrimmed);
2466 ++Line;
2467 }
2468 emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI);
2469}
2470
2471char NVPTXAsmPrinter::ID = 0;
2472
2473INITIALIZE_PASS(NVPTXAsmPrinter, "nvptx-asm-printer", "NVPTX Assembly Printer",
2474 false, false)
2475
2476// Force static initialization.
2477extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
2478LLVMInitializeNVPTXAsmPrinter() {
2479 RegisterAsmPrinter<NVPTXAsmPrinter> X(getTheNVPTXTarget32());
2480 RegisterAsmPrinter<NVPTXAsmPrinter> Y(getTheNVPTXTarget64());
2481}
2482
2483PreservedAnalyses NVPTXAsmPrinterBeginPass::run(Module &M,
2484 ModuleAnalysisManager &MAM) {
2485 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2486 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2487 Printer.doInitialization(M);
2488 return PreservedAnalyses::all();
2489}
2490
2491PreservedAnalyses
2492NVPTXAsmPrinterPass::run(MachineFunction &MF,
2493 MachineFunctionAnalysisManager &MFAM) {
2494 AsmPrinter &Printer =
2495 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2496 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2497 ->getPrinter();
2498 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter&: Printer);
2499 Printer.runOnMachineFunction(MF);
2500 return PreservedAnalyses::all();
2501}
2502
2503PreservedAnalyses NVPTXAsmPrinterEndPass::run(Module &M,
2504 ModuleAnalysisManager &MAM) {
2505 AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter();
2506 setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer);
2507 Printer.doFinalization(M);
2508 return PreservedAnalyses::all();
2509}
2510