| 1 | //===-- NVPTXAsmPrinter.cpp - NVPTX LLVM assembly writer ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains a printer that converts from our internal representation |
| 10 | // of machine-dependent LLVM code to NVPTX assembly language. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "NVPTXAsmPrinter.h" |
| 15 | #include "MCTargetDesc/NVPTXBaseInfo.h" |
| 16 | #include "MCTargetDesc/NVPTXInstPrinter.h" |
| 17 | #include "MCTargetDesc/NVPTXTargetStreamer.h" |
| 18 | #include "NVPTX.h" |
| 19 | #include "NVPTXDwarfDebug.h" |
| 20 | #include "NVPTXMCExpr.h" |
| 21 | #include "NVPTXMachineFunctionInfo.h" |
| 22 | #include "NVPTXRegisterInfo.h" |
| 23 | #include "NVPTXSubtarget.h" |
| 24 | #include "NVPTXTargetMachine.h" |
| 25 | #include "NVPTXUtilities.h" |
| 26 | #include "NVVMProperties.h" |
| 27 | #include "TargetInfo/NVPTXTargetInfo.h" |
| 28 | #include "cl_common_defines.h" |
| 29 | #include "llvm/ADT/APFloat.h" |
| 30 | #include "llvm/ADT/APInt.h" |
| 31 | #include "llvm/ADT/ArrayRef.h" |
| 32 | #include "llvm/ADT/DenseMap.h" |
| 33 | #include "llvm/ADT/DenseSet.h" |
| 34 | #include "llvm/ADT/SCCIterator.h" |
| 35 | #include "llvm/ADT/STLExtras.h" |
| 36 | #include "llvm/ADT/Sequence.h" |
| 37 | #include "llvm/ADT/SmallPtrSet.h" |
| 38 | #include "llvm/ADT/SmallString.h" |
| 39 | #include "llvm/ADT/SmallVector.h" |
| 40 | #include "llvm/ADT/StringExtras.h" |
| 41 | #include "llvm/ADT/StringRef.h" |
| 42 | #include "llvm/ADT/Twine.h" |
| 43 | #include "llvm/ADT/iterator_range.h" |
| 44 | #include "llvm/Analysis/ConstantFolding.h" |
| 45 | #include "llvm/CodeGen/Analysis.h" |
| 46 | #include "llvm/CodeGen/AsmPrinter.h" |
| 47 | #include "llvm/CodeGen/AsmPrinterAnalysis.h" |
| 48 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 49 | #include "llvm/CodeGen/MachineFrameInfo.h" |
| 50 | #include "llvm/CodeGen/MachineFunction.h" |
| 51 | #include "llvm/CodeGen/MachineInstr.h" |
| 52 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
| 53 | #include "llvm/CodeGen/MachineLoopInfo.h" |
| 54 | #include "llvm/CodeGen/MachineModuleInfo.h" |
| 55 | #include "llvm/CodeGen/MachineOperand.h" |
| 56 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 57 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
| 58 | #include "llvm/CodeGen/ValueTypes.h" |
| 59 | #include "llvm/CodeGenTypes/MachineValueType.h" |
| 60 | #include "llvm/IR/Argument.h" |
| 61 | #include "llvm/IR/Attributes.h" |
| 62 | #include "llvm/IR/BasicBlock.h" |
| 63 | #include "llvm/IR/Constant.h" |
| 64 | #include "llvm/IR/Constants.h" |
| 65 | #include "llvm/IR/DataLayout.h" |
| 66 | #include "llvm/IR/DebugInfo.h" |
| 67 | #include "llvm/IR/DebugInfoMetadata.h" |
| 68 | #include "llvm/IR/DebugLoc.h" |
| 69 | #include "llvm/IR/DerivedTypes.h" |
| 70 | #include "llvm/IR/Function.h" |
| 71 | #include "llvm/IR/GlobalAlias.h" |
| 72 | #include "llvm/IR/GlobalValue.h" |
| 73 | #include "llvm/IR/GlobalVariable.h" |
| 74 | #include "llvm/IR/InstrTypes.h" |
| 75 | #include "llvm/IR/Instruction.h" |
| 76 | #include "llvm/IR/LLVMContext.h" |
| 77 | #include "llvm/IR/Module.h" |
| 78 | #include "llvm/IR/Operator.h" |
| 79 | #include "llvm/IR/Type.h" |
| 80 | #include "llvm/IR/User.h" |
| 81 | #include "llvm/IR/Value.h" |
| 82 | #include "llvm/MC/MCExpr.h" |
| 83 | #include "llvm/MC/MCInst.h" |
| 84 | #include "llvm/MC/MCInstrDesc.h" |
| 85 | #include "llvm/MC/MCStreamer.h" |
| 86 | #include "llvm/MC/MCSymbol.h" |
| 87 | #include "llvm/MC/TargetRegistry.h" |
| 88 | #include "llvm/Pass.h" |
| 89 | #include "llvm/Support/Alignment.h" |
| 90 | #include "llvm/Support/Casting.h" |
| 91 | #include "llvm/Support/Compiler.h" |
| 92 | #include "llvm/Support/Endian.h" |
| 93 | #include "llvm/Support/ErrorHandling.h" |
| 94 | #include "llvm/Support/NativeFormatting.h" |
| 95 | #include "llvm/Support/raw_ostream.h" |
| 96 | #include "llvm/Target/TargetLoweringObjectFile.h" |
| 97 | #include "llvm/Target/TargetMachine.h" |
| 98 | #include "llvm/Transforms/Utils/UnrollLoop.h" |
| 99 | #include <algorithm> |
| 100 | #include <cassert> |
| 101 | #include <cstdint> |
| 102 | #include <cstring> |
| 103 | #include <map> |
| 104 | #include <memory> |
| 105 | #include <set> |
| 106 | #include <string> |
| 107 | #include <vector> |
| 108 | |
| 109 | using namespace llvm; |
| 110 | |
| 111 | #define DEPOTNAME "__local_depot" |
| 112 | |
| 113 | // The ptx syntax and format is very different from that usually seem in a .s |
| 114 | // file, |
| 115 | // therefore we are not able to use the MCAsmStreamer interface here. |
| 116 | // |
| 117 | // We are handcrafting the output method here. |
| 118 | // |
| 119 | // A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer |
| 120 | // (subclass of MCStreamer). |
| 121 | |
| 122 | namespace { |
| 123 | |
| 124 | class NVPTXAsmPrinter : public AsmPrinter { |
| 125 | |
| 126 | class AggBuffer { |
| 127 | // Used to buffer the emitted string for initializing global aggregates. |
| 128 | // |
| 129 | // Normally an aggregate (array, vector, or structure) is emitted as a u8[]. |
| 130 | // However, if either element/field of the aggregate is a non-NULL address, |
| 131 | // and all such addresses are properly aligned, then the aggregate is |
| 132 | // emitted as u32[] or u64[]. In the case of unaligned addresses, the |
| 133 | // aggregate is emitted as u8[], and the mask() operator is used for all |
| 134 | // pointers. |
| 135 | // |
| 136 | // We first layout the aggregate in 'buffer' in bytes, except for those |
| 137 | // symbol addresses. For the i-th symbol address in the aggregate, its |
| 138 | // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s. |
| 139 | // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1] |
| 140 | // records the Value*. |
| 141 | // |
| 142 | // Once we have this AggBuffer setup, we can choose how to print it out. |
| 143 | public: |
| 144 | // number of symbol addresses |
| 145 | unsigned numSymbols() const { return Symbols.size(); } |
| 146 | |
| 147 | bool allSymbolsAligned(unsigned ptrSize) const { |
| 148 | return llvm::all_of(Range: symbolPosInBuffer, |
| 149 | P: [=](unsigned pos) { return pos % ptrSize == 0; }); |
| 150 | } |
| 151 | |
| 152 | private: |
| 153 | const unsigned Size; // size of the buffer in bytes |
| 154 | std::vector<unsigned char> buffer; // the buffer |
| 155 | SmallVector<unsigned, 4> symbolPosInBuffer; |
| 156 | SmallVector<const Value *, 4> Symbols; |
| 157 | // SymbolsBeforeStripping[i] is the original form of Symbols[i] before |
| 158 | // stripping pointer casts, i.e., |
| 159 | // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts(). |
| 160 | // |
| 161 | // We need to keep these values because AggBuffer::print decides whether to |
| 162 | // emit a "generic()" cast for Symbols[i] depending on the address space of |
| 163 | // SymbolsBeforeStripping[i]. |
| 164 | SmallVector<const Value *, 4> SymbolsBeforeStripping; |
| 165 | unsigned curpos; |
| 166 | const NVPTXAsmPrinter &AP; |
| 167 | const bool EmitGeneric; |
| 168 | |
| 169 | public: |
| 170 | AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP) |
| 171 | : Size(Size), buffer(Size), curpos(0), AP(AP), |
| 172 | EmitGeneric(AP.EmitGeneric) {} |
| 173 | |
| 174 | // Number of bytes written so far. |
| 175 | unsigned getCurpos() const { return curpos; } |
| 176 | |
| 177 | // Copy Num bytes from Ptr. |
| 178 | // if Bytes > Num, zero fill up to Bytes. |
| 179 | void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) { |
| 180 | for (unsigned I : llvm::seq(Size: Num)) |
| 181 | addByte(Byte: Ptr[I]); |
| 182 | if (Bytes > Num) |
| 183 | addZeros(Num: Bytes - Num); |
| 184 | } |
| 185 | |
| 186 | void addByte(uint8_t Byte) { |
| 187 | assert(curpos < Size); |
| 188 | buffer[curpos] = Byte; |
| 189 | curpos++; |
| 190 | } |
| 191 | |
| 192 | void addZeros(unsigned Num) { |
| 193 | for ([[maybe_unused]] unsigned _ : llvm::seq(Size: Num)) { |
| 194 | addByte(Byte: 0); |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) { |
| 199 | symbolPosInBuffer.push_back(Elt: curpos); |
| 200 | Symbols.push_back(Elt: GVar); |
| 201 | SymbolsBeforeStripping.push_back(Elt: GVarBeforeStripping); |
| 202 | } |
| 203 | |
| 204 | void printBytes(raw_ostream &os); |
| 205 | void printWords(raw_ostream &os); |
| 206 | |
| 207 | private: |
| 208 | void printSymbol(unsigned nSym, raw_ostream &os); |
| 209 | }; |
| 210 | |
| 211 | friend class AggBuffer; |
| 212 | |
| 213 | public: |
| 214 | static char ID; |
| 215 | |
| 216 | StringRef getPassName() const override { return "NVPTX Assembly Printer" ; } |
| 217 | |
| 218 | private: |
| 219 | const Function *F; |
| 220 | |
| 221 | NVPTXTargetStreamer *getTargetStreamer() const; |
| 222 | |
| 223 | void emitStartOfAsmFile(Module &M) override; |
| 224 | void emitBasicBlockStart(const MachineBasicBlock &MBB) override; |
| 225 | void emitFunctionEntryLabel() override; |
| 226 | void emitFunctionBodyStart() override; |
| 227 | void emitFunctionBodyEnd() override; |
| 228 | void emitImplicitDef(const MachineInstr *MI) const override; |
| 229 | |
| 230 | void emitInstruction(const MachineInstr *) override; |
| 231 | void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI); |
| 232 | MCOperand lowerOperand(const MachineOperand &MO); |
| 233 | MCOperand GetSymbolRef(const MCSymbol *Symbol); |
| 234 | MCRegister encodeVirtualRegister(Register Reg); |
| 235 | |
| 236 | /// The number \p Reg was assigned within its register class, as declared by |
| 237 | /// this function's .reg directives. |
| 238 | unsigned getVirtualRegisterNumber(Register Reg) const; |
| 239 | |
| 240 | void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O, |
| 241 | const char *Modifier = nullptr); |
| 242 | void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O, |
| 243 | bool processDemoted, const NVPTXSubtarget &STI); |
| 244 | // PTX permits these directives only between a .func directive and its body, |
| 245 | // so the caller must not use this for a kernel. |
| 246 | template <typename T> |
| 247 | void printFunctionDirectives(const T &V, raw_ostream &O) const; |
| 248 | |
| 249 | void emitGlobals(const Module &M); |
| 250 | void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override; |
| 251 | void emitHeader(Module &M, const NVPTXSubtarget &STI); |
| 252 | void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const; |
| 253 | void emitFunctionParamList(const Function *, raw_ostream &O); |
| 254 | void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF); |
| 255 | void encodeDebugInfoRegisterNumbers(const MachineFunction &MF); |
| 256 | void emitCallPrototype(const CallBase &CB, MCSymbol *PrototypeSymbol) const; |
| 257 | void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const; |
| 258 | |
| 259 | bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, |
| 260 | const char *, raw_ostream &) override; |
| 261 | void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); |
| 262 | bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, |
| 263 | const char *, raw_ostream &) override; |
| 264 | |
| 265 | const MCExpr *lowerConstantForGV(const Constant *CV, |
| 266 | bool ProcessingGeneric) const; |
| 267 | void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const; |
| 268 | /// Emit a blob of inline asm to the output streamer. |
| 269 | void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, |
| 270 | const MCTargetOptions &MCOptions, const MDNode *LocMDNode, |
| 271 | InlineAsm::AsmDialect Dialect, |
| 272 | const MachineInstr *MI) override; |
| 273 | |
| 274 | protected: |
| 275 | bool doInitialization(Module &M) override; |
| 276 | bool doFinalization(Module &M) override; |
| 277 | |
| 278 | /// Create NVPTX-specific DwarfDebug handler. |
| 279 | DwarfDebug *createDwarfDebug() override; |
| 280 | |
| 281 | private: |
| 282 | bool GlobalsEmitted; |
| 283 | |
| 284 | // This is specific per MachineFunction. |
| 285 | const MachineRegisterInfo *MRI; |
| 286 | |
| 287 | // The number assigned to each virtual register within its class, populated |
| 288 | // by setAndEmitFunctionVirtualRegisters and cleared between functions. |
| 289 | using VRegMap = DenseMap<Register, unsigned>; |
| 290 | using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>; |
| 291 | VRegRCMap VRegMapping; |
| 292 | |
| 293 | // List of variables demoted to a function scope. |
| 294 | std::map<const Function *, std::vector<const GlobalVariable *>> localDecls; |
| 295 | |
| 296 | /// Print the state space, alignment, type, name, and — when |
| 297 | /// \p EmitInitializer is set — the initializer of \p GVar. Passing false |
| 298 | /// prints a declaration whose type still matches the definition, as an |
| 299 | /// `.extern` forward declaration requires. |
| 300 | void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar, |
| 301 | raw_ostream &O, |
| 302 | const NVPTXSubtarget &STI, |
| 303 | bool EmitInitializer); |
| 304 | void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const; |
| 305 | std::string getPTXFundamentalTypeStr(Type *Ty) const; |
| 306 | void printScalarConstant(const Constant *CPV, raw_ostream &O); |
| 307 | void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const; |
| 308 | void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer); |
| 309 | void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer); |
| 310 | |
| 311 | void emitLinkageDirective(const GlobalValue *V, raw_ostream &O); |
| 312 | void emitDeclarations(const Module &, raw_ostream &O); |
| 313 | void emitDeclaration(const Function *, raw_ostream &O); |
| 314 | void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O); |
| 315 | void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O); |
| 316 | void emitDemotedVars(const Function *, raw_ostream &); |
| 317 | |
| 318 | bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const; |
| 319 | |
| 320 | // Used to control the need to emit .generic() in the initializer of |
| 321 | // module scope variables. |
| 322 | // Although ptx supports the hybrid mode like the following, |
| 323 | // .global .u32 a; |
| 324 | // .global .u32 b; |
| 325 | // .global .u32 addr[] = {a, generic(b)} |
| 326 | // we have difficulty representing the difference in the NVVM IR. |
| 327 | // |
| 328 | // Since the address value should always be generic in CUDA C and always |
| 329 | // be specific in OpenCL, we use this simple control here. |
| 330 | // |
| 331 | const bool EmitGeneric; |
| 332 | |
| 333 | public: |
| 334 | NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) |
| 335 | : AsmPrinter(TM, std::move(Streamer), ID), |
| 336 | EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() == |
| 337 | NVPTX::CUDA) {} |
| 338 | |
| 339 | bool runOnMachineFunction(MachineFunction &F) override; |
| 340 | |
| 341 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 342 | AU.addRequired<MachineLoopInfoWrapperPass>(); |
| 343 | AsmPrinter::getAnalysisUsage(AU); |
| 344 | } |
| 345 | |
| 346 | std::string getVirtualRegisterName(Register Reg) const; |
| 347 | |
| 348 | const MCSymbol *getFunctionFrameSymbol() const override; |
| 349 | |
| 350 | // Make emitGlobalVariable() no-op for NVPTX. |
| 351 | // Global variables have been already emitted by the time the base AsmPrinter |
| 352 | // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()). |
| 353 | void emitGlobalVariable(const GlobalVariable *GV) override {} |
| 354 | }; |
| 355 | |
| 356 | } // end anonymous namespace |
| 357 | |
| 358 | /// Emits initial debug location directive. |
| 359 | static void emitInitialRawDwarfLocDirective(const MachineFunction &MF, |
| 360 | DwarfDebug *DD, |
| 361 | MCStreamer &OutStreamer) { |
| 362 | if (!DD) |
| 363 | return; |
| 364 | |
| 365 | assert(OutStreamer.hasRawTextSupport() && "Expected assembly output mode." ); |
| 366 | // This is NVPTX specific and it's unclear why. |
| 367 | // PR51079: If we have code without debug information we need to give up. |
| 368 | const DISubprogram *SP = MF.getFunction().getSubprogram(); |
| 369 | if (!SP) |
| 370 | return; |
| 371 | assert(SP->getUnit()); |
| 372 | // NoDebug and DebugDirectivesOnly do not require emitting the initial loc |
| 373 | // directive. NoDebug does not require any debug directives and the initial |
| 374 | // loc directive is not needed for DebugDirectivesOnly as it is redundant |
| 375 | // assuming this is a non-empty function. |
| 376 | if (SP->getUnit()->isDebugDirectivesOnly() || SP->getUnit()->isNoDebug()) |
| 377 | return; |
| 378 | |
| 379 | (void)DD->emitInitialLocDirective(MF, /*CUID=*/0); |
| 380 | } |
| 381 | |
| 382 | namespace { |
| 383 | |
| 384 | /// Return a list of GlobalVariables on which \p V depends. |
| 385 | static void |
| 386 | discoverDependentGlobals(const Value *V, |
| 387 | SmallVectorImpl<const GlobalVariable *> &Globals, |
| 388 | SmallPtrSetImpl<const GlobalVariable *> &Seen) { |
| 389 | if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V)) { |
| 390 | if (Seen.insert(Ptr: GV).second) |
| 391 | Globals.push_back(Elt: GV); |
| 392 | return; |
| 393 | } |
| 394 | |
| 395 | // Global values are emitted as symbols. Their operands do not contribute to |
| 396 | // the initializer expression that refers to that symbol. |
| 397 | if (isa<GlobalValue>(Val: V)) |
| 398 | return; |
| 399 | |
| 400 | // lowerConstantForGV emits a GEP as its base symbol plus a constant byte |
| 401 | // offset. Symbols used to compute an index are not part of that expression. |
| 402 | if (const GEPOperator *GEP = dyn_cast<GEPOperator>(Val: V)) { |
| 403 | discoverDependentGlobals(V: GEP->getPointerOperand(), Globals, Seen); |
| 404 | return; |
| 405 | } |
| 406 | |
| 407 | if (const User *U = dyn_cast<User>(Val: V)) |
| 408 | for (const auto &O : U->operands()) |
| 409 | discoverDependentGlobals(V: O, Globals, Seen); |
| 410 | } |
| 411 | |
| 412 | struct GlobalVariableDependencyNode { |
| 413 | const GlobalVariable *GV = nullptr; |
| 414 | unsigned ModuleOrder = 0; |
| 415 | SmallVector<const GlobalVariableDependencyNode *, 4> Dependencies; |
| 416 | }; |
| 417 | |
| 418 | class GlobalVariableDependencyGraph { |
| 419 | // scc_iterator needs a single entry node. Global initializer dependencies |
| 420 | // may be disconnected, so use a synthetic root with an edge to every global. |
| 421 | GlobalVariableDependencyNode SyntheticRoot; |
| 422 | // Edges store pointers into Nodes, so node addresses must remain stable while |
| 423 | // the graph is constructed. |
| 424 | std::map<const GlobalVariable *, GlobalVariableDependencyNode> Nodes; |
| 425 | |
| 426 | public: |
| 427 | explicit GlobalVariableDependencyGraph(const Module &M) { |
| 428 | unsigned ModuleOrder = 0; |
| 429 | for (const GlobalVariable &GV : M.globals()) { |
| 430 | GlobalVariableDependencyNode &Node = Nodes.try_emplace(k: &GV).first->second; |
| 431 | Node.GV = &GV; |
| 432 | Node.ModuleOrder = ModuleOrder++; |
| 433 | SyntheticRoot.Dependencies.push_back(Elt: &Node); |
| 434 | } |
| 435 | |
| 436 | for (auto &[GV, Node] : Nodes) { |
| 437 | SmallVector<const GlobalVariable *, 4> Dependencies; |
| 438 | SmallPtrSet<const GlobalVariable *, 4> Seen; |
| 439 | for (const Use &Operand : GV->operands()) |
| 440 | discoverDependentGlobals(V: Operand, Globals&: Dependencies, Seen); |
| 441 | |
| 442 | for (const GlobalVariable *Dependency : Dependencies) { |
| 443 | auto It = Nodes.find(x: Dependency); |
| 444 | if (It != Nodes.end()) |
| 445 | Node.Dependencies.push_back(Elt: &It->second); |
| 446 | } |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | const GlobalVariableDependencyNode *getEntryNode() const { |
| 451 | return &SyntheticRoot; |
| 452 | } |
| 453 | }; |
| 454 | |
| 455 | struct GlobalVariableDependencyGraphTraits { |
| 456 | using NodeRef = const GlobalVariableDependencyNode *; |
| 457 | using ChildIteratorType = |
| 458 | SmallVectorImpl<const GlobalVariableDependencyNode *>::const_iterator; |
| 459 | |
| 460 | static NodeRef getEntryNode(NodeRef Node) { return Node; } |
| 461 | static ChildIteratorType child_begin(NodeRef Node) { |
| 462 | return Node->Dependencies.begin(); |
| 463 | } |
| 464 | static ChildIteratorType child_end(NodeRef Node) { |
| 465 | return Node->Dependencies.end(); |
| 466 | } |
| 467 | }; |
| 468 | |
| 469 | using GlobalVariableSCCIterator = |
| 470 | scc_iterator<const GlobalVariableDependencyNode *, |
| 471 | GlobalVariableDependencyGraphTraits>; |
| 472 | |
| 473 | static bool shouldSkipModuleLevelGlobal(const GlobalVariable &GV) { |
| 474 | if (GV.hasSection() && GV.getSection() == "llvm.metadata" ) |
| 475 | return true; |
| 476 | return GV.getName().starts_with(Prefix: "llvm." ) || GV.getName().starts_with(Prefix: "nvvm." ); |
| 477 | } |
| 478 | |
| 479 | static bool isForwardDeclarableGlobal(const GlobalVariable *GVar) { |
| 480 | if (shouldSkipModuleLevelGlobal(GV: *GVar) || GVar->isDeclaration() || |
| 481 | getPTXOpaqueType(*GVar) != PTXOpaqueType::None) |
| 482 | return false; |
| 483 | |
| 484 | // A PTX .extern declaration can be resolved by a later .visible, .weak, or |
| 485 | // .common definition, but not by a static definition. |
| 486 | if (GVar->hasExternalLinkage()) |
| 487 | return GVar->hasInitializer(); |
| 488 | |
| 489 | if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() || |
| 490 | GVar->hasAvailableExternallyLinkage() || GVar->hasCommonLinkage()) |
| 491 | return true; |
| 492 | |
| 493 | return false; |
| 494 | } |
| 495 | |
| 496 | /// Order definitions after treating references to forward-declared globals as |
| 497 | /// already satisfied. A remaining cycle cannot be emitted portably because it |
| 498 | /// requires an undeclared forward reference. |
| 499 | static SmallVector<const GlobalVariable *, 4> orderDefinitionsInSCC( |
| 500 | ArrayRef<const GlobalVariableDependencyNode *> SCC, |
| 501 | const DenseSet<const GlobalVariableDependencyNode *> &ForwardDeclared) { |
| 502 | using Node = GlobalVariableDependencyNode; |
| 503 | |
| 504 | DenseSet<const Node *> SCCSet; |
| 505 | SCCSet.insert_range(R&: SCC); |
| 506 | |
| 507 | DenseMap<const Node *, unsigned> DependencyCount; |
| 508 | DenseMap<const Node *, SmallVector<const Node *, 4>> Dependents; |
| 509 | std::set<std::pair<unsigned, const Node *>> Ready; |
| 510 | |
| 511 | // Dependencies outside this SCC have already been emitted. Forward-declared |
| 512 | // dependencies are also satisfied, so only count the remaining SCC edges. |
| 513 | for (const Node *N : SCC) { |
| 514 | unsigned &Count = DependencyCount[N]; |
| 515 | for (const Node *Dependency : N->Dependencies) { |
| 516 | if (!SCCSet.count(V: Dependency) || ForwardDeclared.count(V: Dependency)) |
| 517 | continue; |
| 518 | ++Count; |
| 519 | Dependents[Dependency].push_back(Elt: N); |
| 520 | } |
| 521 | if (Count == 0) |
| 522 | Ready.emplace(args: N->ModuleOrder, args&: N); |
| 523 | } |
| 524 | |
| 525 | SmallVector<const GlobalVariable *, 4> Order; |
| 526 | while (!Ready.empty()) { |
| 527 | const Node *N = Ready.begin()->second; |
| 528 | Ready.erase(position: Ready.begin()); |
| 529 | Order.push_back(Elt: N->GV); |
| 530 | |
| 531 | auto It = Dependents.find(Val: N); |
| 532 | if (It == Dependents.end()) |
| 533 | continue; |
| 534 | for (const Node *Dependent : It->second) { |
| 535 | assert(DependencyCount[Dependent] && "Dependency already satisfied" ); |
| 536 | if (--DependencyCount[Dependent] == 0) |
| 537 | Ready.emplace(args: Dependent->ModuleOrder, args&: Dependent); |
| 538 | } |
| 539 | } |
| 540 | |
| 541 | if (Order.size() != SCC.size()) |
| 542 | report_fatal_error(reason: "Circular dependency found in global variable set" ); |
| 543 | return Order; |
| 544 | } |
| 545 | |
| 546 | } // namespace |
| 547 | |
| 548 | void NVPTXAsmPrinter::emitInstruction(const MachineInstr *MI) { |
| 549 | NVPTX_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(), |
| 550 | Features: getSubtargetInfo().getFeatureBits()); |
| 551 | |
| 552 | MCInst Inst; |
| 553 | lowerToMCInst(MI, OutMI&: Inst); |
| 554 | EmitToStreamer(S&: *OutStreamer, Inst); |
| 555 | } |
| 556 | |
| 557 | void NVPTXAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) { |
| 558 | OutMI.setOpcode(MI->getOpcode()); |
| 559 | for (const auto MO : MI->operands()) |
| 560 | OutMI.addOperand(Op: lowerOperand(MO)); |
| 561 | } |
| 562 | |
| 563 | MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) { |
| 564 | switch (MO.getType()) { |
| 565 | default: |
| 566 | llvm_unreachable("unknown operand type" ); |
| 567 | case MachineOperand::MO_Register: |
| 568 | return MCOperand::createReg(Reg: encodeVirtualRegister(Reg: MO.getReg())); |
| 569 | case MachineOperand::MO_Immediate: |
| 570 | return MCOperand::createImm(Val: MO.getImm()); |
| 571 | case MachineOperand::MO_MachineBasicBlock: |
| 572 | return MCOperand::createExpr( |
| 573 | Val: MCSymbolRefExpr::create(Symbol: MO.getMBB()->getSymbol(), Ctx&: OutContext)); |
| 574 | case MachineOperand::MO_ExternalSymbol: |
| 575 | return GetSymbolRef(Symbol: GetExternalSymbolSymbol(Sym: MO.getSymbolName())); |
| 576 | case MachineOperand::MO_MCSymbol: |
| 577 | return GetSymbolRef(Symbol: MO.getMCSymbol()); |
| 578 | case MachineOperand::MO_JumpTableIndex: |
| 579 | // The jump table index names the .branchtargets list emitted for a brx.idx |
| 580 | // (see emitJumpTable); reference it by that label. |
| 581 | return GetSymbolRef(Symbol: GetJTISymbol(JTID: MO.getIndex())); |
| 582 | case MachineOperand::MO_GlobalAddress: |
| 583 | return GetSymbolRef(Symbol: getSymbol(GV: MO.getGlobal())); |
| 584 | case MachineOperand::MO_FPImmediate: { |
| 585 | const ConstantFP *Cnt = MO.getFPImm(); |
| 586 | const APFloat &Val = Cnt->getValueAPF(); |
| 587 | |
| 588 | switch (Cnt->getType()->getTypeID()) { |
| 589 | default: |
| 590 | report_fatal_error(reason: "Unsupported FP type" ); |
| 591 | break; |
| 592 | case Type::HalfTyID: |
| 593 | return MCOperand::createExpr( |
| 594 | Val: NVPTXFloatMCExpr::createConstantFPHalf(Flt: Val, Ctx&: OutContext)); |
| 595 | case Type::BFloatTyID: |
| 596 | return MCOperand::createExpr( |
| 597 | Val: NVPTXFloatMCExpr::createConstantBFPHalf(Flt: Val, Ctx&: OutContext)); |
| 598 | case Type::FloatTyID: |
| 599 | return MCOperand::createExpr( |
| 600 | Val: NVPTXFloatMCExpr::createConstantFPSingle(Flt: Val, Ctx&: OutContext)); |
| 601 | case Type::DoubleTyID: |
| 602 | return MCOperand::createExpr( |
| 603 | Val: NVPTXFloatMCExpr::createConstantFPDouble(Flt: Val, Ctx&: OutContext)); |
| 604 | } |
| 605 | break; |
| 606 | } |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | static NVPTX::VirtualRegisterKind |
| 611 | getVirtualRegisterKind(const TargetRegisterClass *RC) { |
| 612 | if (RC == &NVPTX::B1RegClass) |
| 613 | return NVPTX::VirtualRegisterKind::B1; |
| 614 | if (RC == &NVPTX::B16RegClass) |
| 615 | return NVPTX::VirtualRegisterKind::B16; |
| 616 | if (RC == &NVPTX::B32RegClass) |
| 617 | return NVPTX::VirtualRegisterKind::B32; |
| 618 | if (RC == &NVPTX::B64RegClass) |
| 619 | return NVPTX::VirtualRegisterKind::B64; |
| 620 | if (RC == &NVPTX::B128RegClass) |
| 621 | return NVPTX::VirtualRegisterKind::B128; |
| 622 | llvm_unreachable("Bad register class" ); |
| 623 | } |
| 624 | |
| 625 | unsigned NVPTXAsmPrinter::getVirtualRegisterNumber(Register Reg) const { |
| 626 | const auto It = VRegMapping.find(Val: MRI->getRegClass(Reg)); |
| 627 | assert(It != VRegMapping.end() && "Bad register class" ); |
| 628 | |
| 629 | const unsigned Num = It->second.lookup(Val: Reg); |
| 630 | assert(Num && "Bad virtual register" ); |
| 631 | return Num; |
| 632 | } |
| 633 | |
| 634 | MCRegister NVPTXAsmPrinter::encodeVirtualRegister(Register Reg) { |
| 635 | if (Reg.isVirtual()) { |
| 636 | // Pack the register class into the upper bits so that |
| 637 | // NVPTXInstPrinter::printRegName can recover the declared name. |
| 638 | const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg)); |
| 639 | const unsigned Num = getVirtualRegisterNumber(Reg); |
| 640 | assert(Num <= NVPTX::VirtualRegisterNumMask && |
| 641 | "Too many virtual registers" ); |
| 642 | return (static_cast<unsigned>(Kind) << NVPTX::VirtualRegisterKindShift) | |
| 643 | Num; |
| 644 | } |
| 645 | |
| 646 | // Some special-use registers are actually physical registers. |
| 647 | // Encode this as the register class ID of 0 and the real register ID. |
| 648 | assert(Reg.id() <= NVPTX::VirtualRegisterNumMask && |
| 649 | "Physical register would decode as a virtual register" ); |
| 650 | return Reg.asMCReg(); |
| 651 | } |
| 652 | |
| 653 | MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) { |
| 654 | const MCExpr *Expr; |
| 655 | Expr = MCSymbolRefExpr::create(Symbol, Ctx&: OutContext); |
| 656 | return MCOperand::createExpr(Val: Expr); |
| 657 | } |
| 658 | |
| 659 | template <typename OwnerT> |
| 660 | static void printParam(const OwnerT *Owner, Type *Ty, unsigned AttrIdx, |
| 661 | bool IsByVal, bool IsKernel, StringRef Name, |
| 662 | const DataLayout &DL, raw_ostream &O) { |
| 663 | O << ".param " ; |
| 664 | |
| 665 | if (IsByVal || shouldPassAsArray(Ty)) { |
| 666 | const Align ParamAlign = |
| 667 | IsByVal && !IsKernel ? getDeviceByValParamAlign(Owner, Ty, AttrIdx, DL) |
| 668 | : getPTXParamAlign(Owner, Ty, AttrIdx, DL); |
| 669 | O << ".align " << ParamAlign.value() << " .b8 " << Name << "[" |
| 670 | << DL.getTypeAllocSize(Ty) << "]" ; |
| 671 | return; |
| 672 | } |
| 673 | |
| 674 | assert((Ty->isFloatingPointTy() || Ty->isIntOrPtrTy()) && |
| 675 | "Unknown parameter type" ); |
| 676 | const unsigned Size = DL.getTypeSizeInBits(Ty).getFixedValue(); |
| 677 | O << ".b" |
| 678 | << (IsKernel ? promoteScalarKernelArgumentSize(Size) |
| 679 | : promoteScalarArgumentSize(Size)) |
| 680 | << " " << Name; |
| 681 | } |
| 682 | |
| 683 | template <typename OwnerT> |
| 684 | static void printReturnValClause(const OwnerT *Owner, StringRef Name, |
| 685 | const DataLayout &DL, raw_ostream &O) { |
| 686 | Type *RetTy = Owner->getFunctionType()->getReturnType(); |
| 687 | |
| 688 | // A void or zero-sized return type (e.g. an empty struct) produces no return |
| 689 | // parameter. |
| 690 | if (RetTy->isVoidTy() || RetTy->isEmptyTy()) |
| 691 | return; |
| 692 | |
| 693 | // Only device functions return a value, so no kernel promotion applies. |
| 694 | O << "(" ; |
| 695 | printParam(Owner, RetTy, AttributeList::ReturnIndex, /*IsByVal=*/false, |
| 696 | /*IsKernel=*/false, Name, DL, O); |
| 697 | O << ") " ; |
| 698 | } |
| 699 | |
| 700 | void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB, |
| 701 | MCSymbol *PrototypeSymbol) const { |
| 702 | const DataLayout &DL = getDataLayout(); |
| 703 | const NVPTXSubtarget &STI = MF->getSubtarget<NVPTXSubtarget>(); |
| 704 | |
| 705 | OutStreamer->emitLabel(Symbol: PrototypeSymbol); |
| 706 | |
| 707 | SmallString<128> Str; |
| 708 | raw_svector_ostream O(Str); |
| 709 | |
| 710 | O << ".callprototype " ; |
| 711 | printReturnValClause(Owner: &CB, Name: "_" , DL, O); |
| 712 | O << "_ (" ; |
| 713 | |
| 714 | auto MakeArg = [&](const unsigned I) { |
| 715 | const bool IsByVal = CB.isByValArgument(ArgNo: I); |
| 716 | Type *Ty = |
| 717 | IsByVal ? CB.getParamByValType(ArgNo: I) : CB.getArgOperand(i: I)->getType(); |
| 718 | |
| 719 | printParam(Owner: &CB, Ty, AttrIdx: I + AttributeList::FirstArgIndex, IsByVal, |
| 720 | /*IsKernel=*/false, Name: "_" , DL, O); |
| 721 | }; |
| 722 | |
| 723 | const FunctionType *FTy = CB.getFunctionType(); |
| 724 | const unsigned NumArgs = FTy->getNumParams(); |
| 725 | |
| 726 | // Zero-sized arguments (e.g. empty structs) are not passed and so do not |
| 727 | // appear in the prototype. |
| 728 | const auto NonEmptyArgs = make_filter_range(Range: seq(Size: NumArgs), Pred: [&](unsigned I) { |
| 729 | return !CB.getArgOperand(i: I)->getType()->isEmptyTy(); |
| 730 | }); |
| 731 | |
| 732 | interleave(c: NonEmptyArgs, os&: O, each_fn: MakeArg, separator: ", " ); |
| 733 | |
| 734 | if (FTy->isVarArg() && CB.arg_size() > NumArgs) |
| 735 | O << (NonEmptyArgs.empty() ? "" : "," ) << " .param .align " |
| 736 | << STI.getMaxRequiredAlignment() << " .b8 _[]" ; |
| 737 | |
| 738 | O << ")\n" ; |
| 739 | printFunctionDirectives(V: CB, O); |
| 740 | O << ";\n" ; |
| 741 | |
| 742 | OutStreamer->emitRawText(String: O.str()); |
| 743 | } |
| 744 | |
| 745 | void NVPTXAsmPrinter::emitJumpTable(const MachineJumpTableEntry &MJT, |
| 746 | unsigned MJTI) const { |
| 747 | OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: MJTI)); |
| 748 | |
| 749 | if (MJT.MBBs.empty()) |
| 750 | return; |
| 751 | |
| 752 | const auto Targets = to_vector( |
| 753 | Range: map_range(C: MJT.MBBs, F: [](const MachineBasicBlock *MBB) -> const MCSymbol * { |
| 754 | return MBB->getSymbol(); |
| 755 | })); |
| 756 | getTargetStreamer()->emitBranchTargetsDirective(Targets); |
| 757 | } |
| 758 | |
| 759 | // Return true if MBB is the header of a loop marked with |
| 760 | // llvm.loop.unroll.disable or llvm.loop.unroll.count=1. |
| 761 | bool NVPTXAsmPrinter::( |
| 762 | const MachineBasicBlock &MBB) const { |
| 763 | const MachineLoopInfo *LI = GetMLI(*MF); |
| 764 | assert(LI && "NVPTXAsmPrinter requires MachineLoopInfo" ); |
| 765 | // We insert .pragma "nounroll" only to the loop header. |
| 766 | if (!LI->isLoopHeader(BB: &MBB)) |
| 767 | return false; |
| 768 | |
| 769 | // llvm.loop.unroll.disable is marked on the back edges of a loop. Therefore, |
| 770 | // we iterate through each back edge of the loop with header MBB, and check |
| 771 | // whether its metadata contains llvm.loop.unroll.disable. |
| 772 | for (const MachineBasicBlock *PMBB : MBB.predecessors()) { |
| 773 | if (LI->getLoopFor(BB: PMBB) != LI->getLoopFor(BB: &MBB)) { |
| 774 | // Edges from other loops to MBB are not back edges. |
| 775 | continue; |
| 776 | } |
| 777 | if (const BasicBlock *PBB = PMBB->getBasicBlock()) { |
| 778 | if (MDNode *LoopID = |
| 779 | PBB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop)) { |
| 780 | if (GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.disable" )) |
| 781 | return true; |
| 782 | if (MDNode *UnrollCountMD = |
| 783 | GetUnrollMetadata(LoopID, Name: "llvm.loop.unroll.count" )) { |
| 784 | if (mdconst::extract<ConstantInt>(MD: UnrollCountMD->getOperand(I: 1)) |
| 785 | ->isOne()) |
| 786 | return true; |
| 787 | } |
| 788 | } |
| 789 | } |
| 790 | } |
| 791 | return false; |
| 792 | } |
| 793 | |
| 794 | void NVPTXAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { |
| 795 | AsmPrinter::emitBasicBlockStart(MBB); |
| 796 | if (isLoopHeaderOfNoUnroll(MBB)) |
| 797 | getTargetStreamer()->emitPragmaDirective(Pragma: "nounroll" ); |
| 798 | } |
| 799 | |
| 800 | void NVPTXAsmPrinter::emitFunctionEntryLabel() { |
| 801 | SmallString<128> Str; |
| 802 | raw_svector_ostream O(Str); |
| 803 | |
| 804 | if (!GlobalsEmitted) { |
| 805 | emitGlobals(M: *MF->getFunction().getParent()); |
| 806 | GlobalsEmitted = true; |
| 807 | } |
| 808 | |
| 809 | // Set up |
| 810 | MRI = &MF->getRegInfo(); |
| 811 | F = &MF->getFunction(); |
| 812 | emitLinkageDirective(V: F, O); |
| 813 | if (isKernelFunction(F: *F)) |
| 814 | O << ".entry " ; |
| 815 | else { |
| 816 | O << ".func " ; |
| 817 | printReturnValClause(Owner: F, Name: "func_retval0" , DL: getDataLayout(), O); |
| 818 | } |
| 819 | |
| 820 | CurrentFnSym->print(OS&: O, MAI); |
| 821 | |
| 822 | emitFunctionParamList(F, O); |
| 823 | O << "\n" ; |
| 824 | |
| 825 | if (isKernelFunction(F: *F)) |
| 826 | emitKernelFunctionDirectives(F: *F, O); |
| 827 | else |
| 828 | printFunctionDirectives(V: *F, O); |
| 829 | |
| 830 | OutStreamer->emitRawText(String: O.str()); |
| 831 | |
| 832 | VRegMapping.clear(); |
| 833 | // Emit open brace for function body. |
| 834 | OutStreamer->emitRawText(String: StringRef("{\n" )); |
| 835 | setAndEmitFunctionVirtualRegisters(*MF); |
| 836 | encodeDebugInfoRegisterNumbers(MF: *MF); |
| 837 | // Emit initial .loc debug directive for correct relocation symbol data. |
| 838 | emitInitialRawDwarfLocDirective(MF: *MF, DD: getDwarfDebug(), OutStreamer&: *OutStreamer); |
| 839 | } |
| 840 | |
| 841 | bool NVPTXAsmPrinter::runOnMachineFunction(MachineFunction &F) { |
| 842 | bool Result = AsmPrinter::runOnMachineFunction(MF&: F); |
| 843 | // Emit closing brace for the body of function F. |
| 844 | // The closing brace must be emitted here because we need to emit additional |
| 845 | // debug labels/data after the last basic block. |
| 846 | // We need to emit the closing brace here because we don't have function that |
| 847 | // finished emission of the function body. |
| 848 | OutStreamer->emitRawText(String: StringRef("}\n" )); |
| 849 | return Result; |
| 850 | } |
| 851 | |
| 852 | void NVPTXAsmPrinter::emitFunctionBodyStart() { |
| 853 | SmallString<128> Str; |
| 854 | raw_svector_ostream O(Str); |
| 855 | emitDemotedVars(&MF->getFunction(), O); |
| 856 | OutStreamer->emitRawText(String: O.str()); |
| 857 | |
| 858 | const auto *MFI = MF->getInfo<NVPTXMachineFunctionInfo>(); |
| 859 | for (const auto &[CB, Symbol] : MFI->getCallPrototypes()) |
| 860 | emitCallPrototype(CB: *CB, PrototypeSymbol: Symbol); |
| 861 | |
| 862 | if (const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo()) |
| 863 | for (const auto &[Idx, JT] : enumerate(First: MJTI->getJumpTables())) |
| 864 | emitJumpTable(MJT: JT, MJTI: Idx); |
| 865 | } |
| 866 | |
| 867 | void NVPTXAsmPrinter::emitFunctionBodyEnd() { |
| 868 | VRegMapping.clear(); |
| 869 | } |
| 870 | |
| 871 | const MCSymbol *NVPTXAsmPrinter::getFunctionFrameSymbol() const { |
| 872 | return OutContext.getOrCreateSymbol(DEPOTNAME + Twine(getFunctionNumber())); |
| 873 | } |
| 874 | |
| 875 | void NVPTXAsmPrinter::emitImplicitDef(const MachineInstr *MI) const { |
| 876 | Register RegNo = MI->getOperand(i: 0).getReg(); |
| 877 | if (RegNo.isVirtual()) |
| 878 | OutStreamer->AddComment(T: Twine("implicit-def: " ) + |
| 879 | getVirtualRegisterName(Reg: RegNo)); |
| 880 | else |
| 881 | OutStreamer->AddComment(T: Twine("implicit-def: " ) + |
| 882 | NVPTXInstPrinter::getRegisterName(Reg: RegNo)); |
| 883 | OutStreamer->addBlankLine(); |
| 884 | } |
| 885 | |
| 886 | void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F, |
| 887 | raw_ostream &O) const { |
| 888 | // If the NVVM IR has some of reqntid* specified, then output |
| 889 | // the reqntid directive, and set the unspecified ones to 1. |
| 890 | // If none of Reqntid* is specified, don't output reqntid directive. |
| 891 | const auto ReqNTID = getReqNTID(F); |
| 892 | if (!ReqNTID.empty()) |
| 893 | O << formatv(Fmt: ".reqntid {0:$[, ]}\n" , |
| 894 | Vals: make_range(x: ReqNTID.begin(), y: ReqNTID.end())); |
| 895 | |
| 896 | const auto MaxNTID = getMaxNTID(F); |
| 897 | if (!MaxNTID.empty()) |
| 898 | O << formatv(Fmt: ".maxntid {0:$[, ]}\n" , |
| 899 | Vals: make_range(x: MaxNTID.begin(), y: MaxNTID.end())); |
| 900 | |
| 901 | if (const auto Mincta = getMinCTASm(F)) |
| 902 | O << ".minnctapersm " << *Mincta << "\n" ; |
| 903 | |
| 904 | if (const auto Maxnreg = getMaxNReg(F)) |
| 905 | O << ".maxnreg " << *Maxnreg << "\n" ; |
| 906 | |
| 907 | // .maxclusterrank directive requires SM_90 or higher, make sure that we |
| 908 | // filter it out for lower SM versions, as it causes a hard ptxas crash. |
| 909 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 910 | const NVPTXSubtarget *STI = &NTM.getSubtarget<NVPTXSubtarget>(F); |
| 911 | |
| 912 | if (STI->hasFeature(Feature: NVPTX::SM90)) { |
| 913 | const auto ClusterDim = getClusterDim(F); |
| 914 | const bool BlocksAreClusters = hasBlocksAreClusters(F); |
| 915 | |
| 916 | if (!ClusterDim.empty()) { |
| 917 | |
| 918 | if (!BlocksAreClusters) |
| 919 | O << ".explicitcluster\n" ; |
| 920 | |
| 921 | if (ClusterDim[0] != 0) { |
| 922 | assert(llvm::all_of(ClusterDim, not_equal_to(0)) && |
| 923 | "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z " |
| 924 | "should be non-zero as well" ); |
| 925 | |
| 926 | O << formatv(Fmt: ".reqnctapercluster {0:$[, ]}\n" , |
| 927 | Vals: make_range(x: ClusterDim.begin(), y: ClusterDim.end())); |
| 928 | } else { |
| 929 | assert(llvm::all_of(ClusterDim, equal_to(0)) && |
| 930 | "cluster_dim_x == 0 implies cluster_dim_y and cluster_dim_z " |
| 931 | "should be 0 as well" ); |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | if (BlocksAreClusters) { |
| 936 | LLVMContext &Ctx = F.getContext(); |
| 937 | if (ReqNTID.empty() || ClusterDim.empty()) |
| 938 | Ctx.diagnose(DI: DiagnosticInfoUnsupported( |
| 939 | F, "blocksareclusters requires reqntid and cluster_dim attributes" , |
| 940 | F.getSubprogram())); |
| 941 | else if (!STI->hasFeature(Feature: NVPTX::PTX90)) |
| 942 | Ctx.diagnose(DI: DiagnosticInfoUnsupported( |
| 943 | F, "blocksareclusters requires PTX version >= 9.0" , |
| 944 | F.getSubprogram())); |
| 945 | else |
| 946 | O << ".blocksareclusters\n" ; |
| 947 | } |
| 948 | |
| 949 | if (const auto Maxclusterrank = getMaxClusterRank(F)) |
| 950 | O << ".maxclusterrank " << *Maxclusterrank << "\n" ; |
| 951 | } |
| 952 | } |
| 953 | |
| 954 | std::string NVPTXAsmPrinter::getVirtualRegisterName(Register Reg) const { |
| 955 | const auto Kind = getVirtualRegisterKind(RC: MRI->getRegClass(Reg)); |
| 956 | |
| 957 | std::string Name; |
| 958 | raw_string_ostream(Name) << NVPTX::getVirtualRegisterPrefix(Kind) |
| 959 | << getVirtualRegisterNumber(Reg); |
| 960 | return Name; |
| 961 | } |
| 962 | |
| 963 | void NVPTXAsmPrinter::emitAliasDeclaration(const GlobalAlias *GA, |
| 964 | raw_ostream &O) { |
| 965 | const Function *F = dyn_cast_or_null<Function>(Val: GA->getAliaseeObject()); |
| 966 | if (!F || isKernelFunction(F: *F) || F->isDeclaration()) |
| 967 | report_fatal_error( |
| 968 | reason: "NVPTX aliasee must be a non-kernel function definition" ); |
| 969 | |
| 970 | if (GA->hasLinkOnceLinkage() || GA->hasWeakLinkage() || |
| 971 | GA->hasAvailableExternallyLinkage() || GA->hasCommonLinkage()) |
| 972 | report_fatal_error(reason: "NVPTX aliasee must not be '.weak'" ); |
| 973 | |
| 974 | emitDeclarationWithName(F, getSymbol(GV: GA), O); |
| 975 | } |
| 976 | |
| 977 | void NVPTXAsmPrinter::emitDeclaration(const Function *F, raw_ostream &O) { |
| 978 | emitDeclarationWithName(F, getSymbol(GV: F), O); |
| 979 | } |
| 980 | |
| 981 | void NVPTXAsmPrinter::emitDeclarationWithName(const Function *F, MCSymbol *S, |
| 982 | raw_ostream &O) { |
| 983 | emitLinkageDirective(V: F, O); |
| 984 | if (isKernelFunction(F: *F)) { |
| 985 | O << ".entry " ; |
| 986 | } else { |
| 987 | O << ".func " ; |
| 988 | printReturnValClause(Owner: F, Name: "func_retval0" , DL: getDataLayout(), O); |
| 989 | } |
| 990 | S->print(OS&: O, MAI); |
| 991 | O << "\n" ; |
| 992 | emitFunctionParamList(F, O); |
| 993 | O << "\n" ; |
| 994 | if (!isKernelFunction(F: *F)) |
| 995 | printFunctionDirectives(V: *F, O); |
| 996 | O << ";\n" ; |
| 997 | } |
| 998 | |
| 999 | static bool usedInGlobalVarDef(const Constant *C) { |
| 1000 | if (!C) |
| 1001 | return false; |
| 1002 | |
| 1003 | if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: C)) |
| 1004 | return GV->getName() != "llvm.used" ; |
| 1005 | |
| 1006 | for (const User *U : C->users()) |
| 1007 | if (const Constant *C = dyn_cast<Constant>(Val: U)) |
| 1008 | if (usedInGlobalVarDef(C)) |
| 1009 | return true; |
| 1010 | |
| 1011 | return false; |
| 1012 | } |
| 1013 | |
| 1014 | static bool usedInOneFunc(const User *U, Function const *&OneFunc) { |
| 1015 | if (const GlobalVariable *OtherGV = dyn_cast<GlobalVariable>(Val: U)) |
| 1016 | if (OtherGV->getName() == "llvm.used" ) |
| 1017 | return true; |
| 1018 | |
| 1019 | if (const Instruction *I = dyn_cast<Instruction>(Val: U)) { |
| 1020 | if (const Function *CurFunc = I->getFunction()) { |
| 1021 | if (OneFunc && (CurFunc != OneFunc)) |
| 1022 | return false; |
| 1023 | OneFunc = CurFunc; |
| 1024 | return true; |
| 1025 | } |
| 1026 | return false; |
| 1027 | } |
| 1028 | |
| 1029 | for (const User *UU : U->users()) |
| 1030 | if (!usedInOneFunc(U: UU, OneFunc)) |
| 1031 | return false; |
| 1032 | |
| 1033 | return true; |
| 1034 | } |
| 1035 | |
| 1036 | /* Find out if a global variable can be demoted to local scope. |
| 1037 | * Currently, this is valid for CUDA shared variables, which have local |
| 1038 | * scope and global lifetime. So the conditions to check are : |
| 1039 | * 1. Is the global variable in shared address space? |
| 1040 | * 2. Does it have local linkage? |
| 1041 | * 3. Is the global variable referenced only in one function? |
| 1042 | */ |
| 1043 | static bool canDemoteGlobalVar(const GlobalVariable *GV, Function const *&f) { |
| 1044 | if (!GV->hasLocalLinkage()) |
| 1045 | return false; |
| 1046 | if (GV->getAddressSpace() != ADDRESS_SPACE_SHARED) |
| 1047 | return false; |
| 1048 | |
| 1049 | const Function *oneFunc = nullptr; |
| 1050 | |
| 1051 | bool flag = usedInOneFunc(U: GV, OneFunc&: oneFunc); |
| 1052 | if (!flag) |
| 1053 | return false; |
| 1054 | if (!oneFunc) |
| 1055 | return false; |
| 1056 | f = oneFunc; |
| 1057 | return true; |
| 1058 | } |
| 1059 | |
| 1060 | static bool useFuncSeen(const Constant *C, |
| 1061 | const SmallPtrSetImpl<const Function *> &SeenSet) { |
| 1062 | for (const User *U : C->users()) { |
| 1063 | if (const Constant *cu = dyn_cast<Constant>(Val: U)) { |
| 1064 | if (useFuncSeen(C: cu, SeenSet)) |
| 1065 | return true; |
| 1066 | } else if (const Instruction *I = dyn_cast<Instruction>(Val: U)) { |
| 1067 | if (const Function *Caller = I->getFunction()) |
| 1068 | if (SeenSet.contains(Ptr: Caller)) |
| 1069 | return true; |
| 1070 | } |
| 1071 | } |
| 1072 | return false; |
| 1073 | } |
| 1074 | |
| 1075 | void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) { |
| 1076 | SmallPtrSet<const Function *, 32> SeenSet; |
| 1077 | for (const Function &F : M) { |
| 1078 | if (F.getAttributes().hasFnAttr(Kind: "nvptx-libcall-callee" )) { |
| 1079 | emitDeclaration(F: &F, O); |
| 1080 | continue; |
| 1081 | } |
| 1082 | |
| 1083 | if (F.isDeclaration()) { |
| 1084 | if (F.use_empty()) |
| 1085 | continue; |
| 1086 | if (F.getIntrinsicID()) |
| 1087 | continue; |
| 1088 | // An unrecognized intrinsic would produce an invalid PTX declaration. Let |
| 1089 | // the user know that, and skip it. |
| 1090 | if (F.isIntrinsic()) { |
| 1091 | LLVMContext &Ctx = F.getContext(); |
| 1092 | Ctx.diagnose(DI: DiagnosticInfoUnsupported( |
| 1093 | F, "unknown intrinsic '" + F.getName() + |
| 1094 | "' cannot be lowered by the NVPTX backend" )); |
| 1095 | continue; |
| 1096 | } |
| 1097 | emitDeclaration(F: &F, O); |
| 1098 | continue; |
| 1099 | } |
| 1100 | for (const User *U : F.users()) { |
| 1101 | if (const Constant *C = dyn_cast<Constant>(Val: U)) { |
| 1102 | if (usedInGlobalVarDef(C)) { |
| 1103 | // The use is in the initialization of a global variable |
| 1104 | // that is a function pointer, so print a declaration |
| 1105 | // for the original function |
| 1106 | emitDeclaration(F: &F, O); |
| 1107 | break; |
| 1108 | } |
| 1109 | // Emit a declaration of this function if the function that |
| 1110 | // uses this constant expr has already been seen. |
| 1111 | if (useFuncSeen(C, SeenSet)) { |
| 1112 | emitDeclaration(F: &F, O); |
| 1113 | break; |
| 1114 | } |
| 1115 | } |
| 1116 | |
| 1117 | if (!isa<Instruction>(Val: U)) |
| 1118 | continue; |
| 1119 | const Function *Caller = cast<Instruction>(Val: U)->getFunction(); |
| 1120 | if (!Caller) |
| 1121 | continue; |
| 1122 | |
| 1123 | // If a caller has already been seen, then the caller is |
| 1124 | // appearing in the module before the callee. so print out |
| 1125 | // a declaration for the callee. |
| 1126 | if (SeenSet.contains(Ptr: Caller)) { |
| 1127 | emitDeclaration(F: &F, O); |
| 1128 | break; |
| 1129 | } |
| 1130 | } |
| 1131 | SeenSet.insert(Ptr: &F); |
| 1132 | } |
| 1133 | for (const GlobalAlias &GA : M.aliases()) |
| 1134 | emitAliasDeclaration(GA: &GA, O); |
| 1135 | } |
| 1136 | |
| 1137 | void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) { |
| 1138 | // Construct a default subtarget off of the TargetMachine defaults. The |
| 1139 | // rest of NVPTX isn't friendly to change subtargets per function and |
| 1140 | // so the default TargetMachine will have all of the options. |
| 1141 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 1142 | const NVPTXSubtarget *STI = NTM.getSubtargetImpl(); |
| 1143 | |
| 1144 | // Emit header before any dwarf directives are emitted below. |
| 1145 | emitHeader(M, STI: *STI); |
| 1146 | } |
| 1147 | |
| 1148 | /// Create NVPTX-specific DwarfDebug handler. |
| 1149 | DwarfDebug *NVPTXAsmPrinter::createDwarfDebug() { |
| 1150 | return new NVPTXDwarfDebug(this); |
| 1151 | } |
| 1152 | |
| 1153 | bool NVPTXAsmPrinter::doInitialization(Module &M) { |
| 1154 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 1155 | const NVPTXSubtarget &STI = *NTM.getSubtargetImpl(); |
| 1156 | if (M.alias_size() && |
| 1157 | (!STI.hasFeature(Feature: NVPTX::PTX63) || !STI.hasFeature(Feature: NVPTX::SM30))) |
| 1158 | report_fatal_error(reason: ".alias requires PTX version >= 6.3 and sm_30" ); |
| 1159 | |
| 1160 | // We need to call the parent's one explicitly. |
| 1161 | bool Result = AsmPrinter::doInitialization(M); |
| 1162 | |
| 1163 | GlobalsEmitted = false; |
| 1164 | |
| 1165 | // Ensure globals are in the symbol table before ISel so any temp symbols are |
| 1166 | // guaranteed not to collide with user symbols |
| 1167 | for (const GlobalValue &GV : M.global_values()) |
| 1168 | getSymbol(GV: &GV); |
| 1169 | |
| 1170 | return Result; |
| 1171 | } |
| 1172 | |
| 1173 | void NVPTXAsmPrinter::emitGlobals(const Module &M) { |
| 1174 | SmallString<128> Str2; |
| 1175 | raw_svector_ostream OS2(Str2); |
| 1176 | |
| 1177 | emitDeclarations(M, O&: OS2); |
| 1178 | |
| 1179 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 1180 | const NVPTXSubtarget &STI = *NTM.getSubtargetImpl(); |
| 1181 | |
| 1182 | // ptxas requires global symbols referenced by initializers to be known |
| 1183 | // before use. Acyclic dependencies can be handled by dependency-first |
| 1184 | // emission. Cyclic SCCs need compatible .extern declarations first. |
| 1185 | // Edges point from each global to the globals used by its initializer. |
| 1186 | // Reverse-topological SCC iteration therefore emits dependencies first. |
| 1187 | GlobalVariableDependencyGraph DependencyGraph(M); |
| 1188 | for (GlobalVariableSCCIterator I = |
| 1189 | GlobalVariableSCCIterator::begin(G: DependencyGraph.getEntryNode()); |
| 1190 | !I.isAtEnd(); ++I) { |
| 1191 | SmallVector<const GlobalVariableDependencyNode *, 4> SCC(I->begin(), |
| 1192 | I->end()); |
| 1193 | |
| 1194 | // Nothing points to the synthetic root, so it is always in its own SCC. |
| 1195 | if (!SCC.front()->GV) { |
| 1196 | assert(SCC.size() == 1 && "Synthetic root must be in its own SCC" ); |
| 1197 | continue; |
| 1198 | } |
| 1199 | |
| 1200 | llvm::sort(C&: SCC, Comp: [](const auto *LHS, const auto *RHS) { |
| 1201 | return LHS->ModuleOrder < RHS->ModuleOrder; |
| 1202 | }); |
| 1203 | |
| 1204 | const bool IsCyclic = I.hasCycle(); |
| 1205 | DenseSet<const GlobalVariableDependencyNode *> ForwardDeclared; |
| 1206 | if (IsCyclic) |
| 1207 | for (const auto *Node : SCC) |
| 1208 | if (isForwardDeclarableGlobal(GVar: Node->GV)) |
| 1209 | ForwardDeclared.insert(V: Node); |
| 1210 | |
| 1211 | // Check that declarations break every cycle before writing any output. |
| 1212 | SmallVector<const GlobalVariable *, 4> OrderedGlobals = |
| 1213 | IsCyclic ? orderDefinitionsInSCC(SCC, ForwardDeclared) |
| 1214 | : SmallVector<const GlobalVariable *, 4>{SCC.front()->GV}; |
| 1215 | |
| 1216 | for (const auto *Node : SCC) { |
| 1217 | if (!ForwardDeclared.count(V: Node)) |
| 1218 | continue; |
| 1219 | OS2 << ".extern " ; |
| 1220 | emitPTXGlobalVariableDefinition(GVar: Node->GV, O&: OS2, STI, |
| 1221 | /*EmitInitializer=*/false); |
| 1222 | OS2 << ";\n" ; |
| 1223 | } |
| 1224 | |
| 1225 | for (const GlobalVariable *GV : OrderedGlobals) |
| 1226 | printModuleLevelGV(GVar: GV, O&: OS2, /*ProcessDemoted=*/processDemoted: false, STI); |
| 1227 | } |
| 1228 | |
| 1229 | OS2 << '\n'; |
| 1230 | |
| 1231 | OutStreamer->emitRawText(String: OS2.str()); |
| 1232 | } |
| 1233 | |
| 1234 | void NVPTXAsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) { |
| 1235 | getTargetStreamer()->emitAliasDirective(Name: getSymbol(GV: &GA), |
| 1236 | Aliasee: getSymbol(GV: GA.getAliaseeObject())); |
| 1237 | } |
| 1238 | |
| 1239 | NVPTXTargetStreamer *NVPTXAsmPrinter::getTargetStreamer() const { |
| 1240 | return static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer()); |
| 1241 | } |
| 1242 | |
| 1243 | static bool hasFullDebugInfo(Module &M) { |
| 1244 | for (DICompileUnit *CU : M.debug_compile_units()) { |
| 1245 | switch(CU->getEmissionKind()) { |
| 1246 | case DICompileUnit::NoDebug: |
| 1247 | case DICompileUnit::DebugDirectivesOnly: |
| 1248 | break; |
| 1249 | case DICompileUnit::LineTablesOnly: |
| 1250 | case DICompileUnit::FullDebug: |
| 1251 | return true; |
| 1252 | } |
| 1253 | } |
| 1254 | |
| 1255 | return false; |
| 1256 | } |
| 1257 | |
| 1258 | void NVPTXAsmPrinter::(Module &M, const NVPTXSubtarget &STI) { |
| 1259 | auto *TS = getTargetStreamer(); |
| 1260 | |
| 1261 | TS->emitBanner(); |
| 1262 | |
| 1263 | const unsigned PTXVersion = STI.getPTXVersion(); |
| 1264 | TS->emitVersionDirective(PTXVersion); |
| 1265 | |
| 1266 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 1267 | bool TexModeIndependent = NTM.getDrvInterface() == NVPTX::NVCL; |
| 1268 | |
| 1269 | TS->emitTargetDirective(Target: STI.getTargetName(), TexModeIndependent, |
| 1270 | HasDebug: hasFullDebugInfo(M)); |
| 1271 | TS->emitAddressSizeDirective(AddrSize: M.getDataLayout().getPointerSizeInBits()); |
| 1272 | } |
| 1273 | |
| 1274 | bool NVPTXAsmPrinter::doFinalization(Module &M) { |
| 1275 | // If we did not emit any functions, then the global declarations have not |
| 1276 | // yet been emitted. |
| 1277 | if (!GlobalsEmitted) { |
| 1278 | emitGlobals(M); |
| 1279 | GlobalsEmitted = true; |
| 1280 | } |
| 1281 | |
| 1282 | // call doFinalization |
| 1283 | bool ret = AsmPrinter::doFinalization(M); |
| 1284 | |
| 1285 | clearAnnotationCache(&M); |
| 1286 | |
| 1287 | auto *TS = |
| 1288 | static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer()); |
| 1289 | // Close the last emitted section |
| 1290 | if (hasDebugInfo()) { |
| 1291 | TS->closeLastSection(); |
| 1292 | // Emit empty .debug_macinfo section for better support of the empty files. |
| 1293 | TS->emitEmptySectionDirective(Name: ".debug_macinfo" ); |
| 1294 | } |
| 1295 | |
| 1296 | // Output last DWARF .file directives, if any. |
| 1297 | TS->outputDwarfFileDirectives(); |
| 1298 | |
| 1299 | return ret; |
| 1300 | } |
| 1301 | |
| 1302 | // This function emits appropriate linkage directives for |
| 1303 | // functions and global variables. |
| 1304 | // |
| 1305 | // extern function declaration -> .extern |
| 1306 | // extern function definition -> .visible |
| 1307 | // external global variable with init -> .visible |
| 1308 | // external without init -> .extern |
| 1309 | // appending -> not allowed, assert. |
| 1310 | // for any linkage other than |
| 1311 | // internal, private, linker_private, |
| 1312 | // linker_private_weak, linker_private_weak_def_auto, |
| 1313 | // we emit -> .weak. |
| 1314 | |
| 1315 | void NVPTXAsmPrinter::emitLinkageDirective(const GlobalValue *V, |
| 1316 | raw_ostream &O) { |
| 1317 | if (static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() == NVPTX::CUDA) { |
| 1318 | if (V->hasExternalLinkage()) { |
| 1319 | if (const auto *GVar = dyn_cast<GlobalVariable>(Val: V)) |
| 1320 | O << (GVar->hasInitializer() ? ".visible " : ".extern " ); |
| 1321 | else if (V->isDeclaration()) |
| 1322 | O << ".extern " ; |
| 1323 | else |
| 1324 | O << ".visible " ; |
| 1325 | } else if (V->hasAppendingLinkage()) { |
| 1326 | report_fatal_error(reason: "Symbol '" + (V->hasName() ? V->getName() : "" ) + |
| 1327 | "' has unsupported appending linkage type" ); |
| 1328 | } else if (!V->hasInternalLinkage() && !V->hasPrivateLinkage()) { |
| 1329 | O << ".weak " ; |
| 1330 | } |
| 1331 | } |
| 1332 | } |
| 1333 | |
| 1334 | void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, |
| 1335 | raw_ostream &O, bool ProcessDemoted, |
| 1336 | const NVPTXSubtarget &STI) { |
| 1337 | // Skip metadata and LLVM intrinsic global variables. |
| 1338 | if (shouldSkipModuleLevelGlobal(GV: *GVar)) |
| 1339 | return; |
| 1340 | |
| 1341 | if (GVar->hasExternalLinkage()) { |
| 1342 | if (GVar->hasInitializer()) |
| 1343 | O << ".visible " ; |
| 1344 | else |
| 1345 | O << ".extern " ; |
| 1346 | } else if (STI.hasFeature(Feature: NVPTX::PTX50) && GVar->hasCommonLinkage() && |
| 1347 | GVar->getAddressSpace() == ADDRESS_SPACE_GLOBAL) { |
| 1348 | O << ".common " ; |
| 1349 | } else if (GVar->hasLinkOnceLinkage() || GVar->hasWeakLinkage() || |
| 1350 | GVar->hasAvailableExternallyLinkage() || |
| 1351 | GVar->hasCommonLinkage()) { |
| 1352 | O << ".weak " ; |
| 1353 | } |
| 1354 | |
| 1355 | const PTXOpaqueType OpaqueType = getPTXOpaqueType(*GVar); |
| 1356 | |
| 1357 | if (OpaqueType == PTXOpaqueType::Texture) { |
| 1358 | O << ".global .texref " ; |
| 1359 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1360 | O << ";\n" ; |
| 1361 | return; |
| 1362 | } |
| 1363 | |
| 1364 | if (OpaqueType == PTXOpaqueType::Surface) { |
| 1365 | O << ".global .surfref " ; |
| 1366 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1367 | O << ";\n" ; |
| 1368 | return; |
| 1369 | } |
| 1370 | |
| 1371 | if (GVar->isDeclaration()) { |
| 1372 | // (extern) declarations, no definition or initializer |
| 1373 | // Currently the only known declaration is for an automatic __local |
| 1374 | // (.shared) promoted to global. |
| 1375 | emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/false); |
| 1376 | O << ";\n" ; |
| 1377 | return; |
| 1378 | } |
| 1379 | |
| 1380 | if (OpaqueType == PTXOpaqueType::Sampler) { |
| 1381 | O << ".global .samplerref " ; |
| 1382 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1383 | |
| 1384 | const Constant *Initializer = nullptr; |
| 1385 | if (GVar->hasInitializer()) |
| 1386 | Initializer = GVar->getInitializer(); |
| 1387 | const ConstantInt *CI = nullptr; |
| 1388 | if (Initializer) |
| 1389 | CI = dyn_cast<ConstantInt>(Val: Initializer); |
| 1390 | if (CI) { |
| 1391 | unsigned sample = CI->getZExtValue(); |
| 1392 | |
| 1393 | O << " = { " ; |
| 1394 | |
| 1395 | for (int i = 0, |
| 1396 | addr = ((sample & __CLK_ADDRESS_MASK) >> __CLK_ADDRESS_BASE); |
| 1397 | i < 3; i++) { |
| 1398 | O << "addr_mode_" << i << " = " ; |
| 1399 | switch (addr) { |
| 1400 | case 0: |
| 1401 | O << "wrap" ; |
| 1402 | break; |
| 1403 | case 1: |
| 1404 | O << "clamp_to_border" ; |
| 1405 | break; |
| 1406 | case 2: |
| 1407 | O << "clamp_to_edge" ; |
| 1408 | break; |
| 1409 | case 3: |
| 1410 | O << "wrap" ; |
| 1411 | break; |
| 1412 | case 4: |
| 1413 | O << "mirror" ; |
| 1414 | break; |
| 1415 | } |
| 1416 | O << ", " ; |
| 1417 | } |
| 1418 | O << "filter_mode = " ; |
| 1419 | switch ((sample & __CLK_FILTER_MASK) >> __CLK_FILTER_BASE) { |
| 1420 | case 0: |
| 1421 | O << "nearest" ; |
| 1422 | break; |
| 1423 | case 1: |
| 1424 | O << "linear" ; |
| 1425 | break; |
| 1426 | case 2: |
| 1427 | llvm_unreachable("Anisotropic filtering is not supported" ); |
| 1428 | default: |
| 1429 | O << "nearest" ; |
| 1430 | break; |
| 1431 | } |
| 1432 | if (!((sample & __CLK_NORMALIZED_MASK) >> __CLK_NORMALIZED_BASE)) { |
| 1433 | O << ", force_unnormalized_coords = 1" ; |
| 1434 | } |
| 1435 | O << " }" ; |
| 1436 | } |
| 1437 | |
| 1438 | O << ";\n" ; |
| 1439 | return; |
| 1440 | } |
| 1441 | |
| 1442 | if (GVar->hasPrivateLinkage()) { |
| 1443 | if (GVar->getName().starts_with(Prefix: "unrollpragma" )) |
| 1444 | return; |
| 1445 | |
| 1446 | // FIXME - need better way (e.g. Metadata) to avoid generating this global |
| 1447 | if (GVar->getName().starts_with(Prefix: "filename" )) |
| 1448 | return; |
| 1449 | if (GVar->use_empty()) |
| 1450 | return; |
| 1451 | } |
| 1452 | |
| 1453 | const Function *DemotedFunc = nullptr; |
| 1454 | if (!ProcessDemoted && canDemoteGlobalVar(GV: GVar, f&: DemotedFunc)) { |
| 1455 | O << "// " << GVar->getName() << " has been demoted\n" ; |
| 1456 | localDecls[DemotedFunc].push_back(x: GVar); |
| 1457 | return; |
| 1458 | } |
| 1459 | |
| 1460 | emitPTXGlobalVariableDefinition(GVar, O, STI, /*EmitInitializer=*/true); |
| 1461 | O << ";\n" ; |
| 1462 | } |
| 1463 | |
| 1464 | template <typename T> |
| 1465 | void NVPTXAsmPrinter::printFunctionDirectives(const T &V, |
| 1466 | raw_ostream &O) const { |
| 1467 | const NVPTXSubtarget &STI = |
| 1468 | *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl(); |
| 1469 | |
| 1470 | if (STI.hasNoReturn() && V.doesNotReturn() && |
| 1471 | V.getFunctionType()->getReturnType()->isVoidTy()) |
| 1472 | O << ".noreturn\n" ; |
| 1473 | |
| 1474 | // Don't emit abi_preserve directives on targets where they are |
| 1475 | // unsupported. |
| 1476 | if (!STI.hasABIPreserve()) |
| 1477 | return; |
| 1478 | |
| 1479 | const ABIPreserveInfo ABI = getABIPreserve(V.getAttributes()); |
| 1480 | if (ABI.Preserve) |
| 1481 | O << ".abi_preserve " << *ABI.Preserve << "\n" ; |
| 1482 | if (ABI.PreserveControl) |
| 1483 | O << ".abi_preserve_control " << *ABI.PreserveControl << "\n" ; |
| 1484 | } |
| 1485 | |
| 1486 | void NVPTXAsmPrinter::emitPTXGlobalVariableDefinition( |
| 1487 | const GlobalVariable *GVar, raw_ostream &O, const NVPTXSubtarget &STI, |
| 1488 | bool EmitInitializer) { |
| 1489 | const DataLayout &DL = getDataLayout(); |
| 1490 | |
| 1491 | Type *ETy = GVar->getValueType(); |
| 1492 | |
| 1493 | emitPTXAddressSpace(AddressSpace: GVar->getAddressSpace(), O); |
| 1494 | |
| 1495 | if (isManaged(*GVar)) { |
| 1496 | if (!STI.hasFeature(Feature: NVPTX::PTX40) || !STI.hasFeature(Feature: NVPTX::SM30)) |
| 1497 | report_fatal_error( |
| 1498 | reason: ".attribute(.managed) requires PTX version >= 4.0 and sm_30" ); |
| 1499 | O << " .attribute(.managed)" ; |
| 1500 | } |
| 1501 | |
| 1502 | O << " .align " |
| 1503 | << GVar->getAlign().value_or(u: DL.getPrefTypeAlign(Ty: ETy)).value(); |
| 1504 | |
| 1505 | const Constant *Initializer = nullptr; |
| 1506 | if (GVar->hasInitializer()) { |
| 1507 | const Constant *Init = GVar->getInitializer(); |
| 1508 | if (!Init->isNullValue() && !isa<UndefValue>(Val: Init)) { |
| 1509 | if (GVar->getAddressSpace() != ADDRESS_SPACE_GLOBAL && |
| 1510 | GVar->getAddressSpace() != ADDRESS_SPACE_CONST) |
| 1511 | report_fatal_error(reason: "initial value of '" + GVar->getName() + |
| 1512 | "' is not allowed in addrspace(" + |
| 1513 | Twine(GVar->getAddressSpace()) + ")" ); |
| 1514 | Initializer = Init; |
| 1515 | } |
| 1516 | } |
| 1517 | |
| 1518 | if (ETy->isPointerTy() || ((ETy->isIntegerTy() || ETy->isFloatingPointTy()) && |
| 1519 | ETy->getScalarSizeInBits() <= 64)) { |
| 1520 | O << " ." << getPTXFundamentalTypeStr(Ty: ETy) << " " ; |
| 1521 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1522 | |
| 1523 | if (EmitInitializer && Initializer) { |
| 1524 | O << " = " ; |
| 1525 | printScalarConstant(CPV: Initializer, O); |
| 1526 | } |
| 1527 | return; |
| 1528 | } |
| 1529 | |
| 1530 | // Although PTX has direct support for struct type and array type and LLVM IR |
| 1531 | // is very similar to PTX, the LLVM CodeGen does not support for targets that |
| 1532 | // support these high level field accesses. Structs, arrays and vectors are |
| 1533 | // lowered into arrays of bytes. |
| 1534 | assert((ETy->isIntegerTy() || ETy->isFP128Ty() || ETy->isAggregateType() || |
| 1535 | isa<FixedVectorType>(ETy)) && |
| 1536 | "type not supported yet" ); |
| 1537 | |
| 1538 | const uint64_t ElementSize = DL.getTypeStoreSize(Ty: ETy); |
| 1539 | |
| 1540 | if (!Initializer) { |
| 1541 | O << " .b8 " ; |
| 1542 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1543 | if (ElementSize) |
| 1544 | O << "[" << ElementSize << "]" ; |
| 1545 | else if (!EmitInitializer) |
| 1546 | O << "[]" ; |
| 1547 | return; |
| 1548 | } |
| 1549 | |
| 1550 | AggBuffer aggBuffer(ElementSize, *this); |
| 1551 | bufferAggregateConstant(CV: Initializer, aggBuffer: &aggBuffer); |
| 1552 | if (aggBuffer.numSymbols()) { |
| 1553 | const unsigned int ptrSize = MAI.getCodePointerSize(); |
| 1554 | if (ElementSize % ptrSize || !aggBuffer.allSymbolsAligned(ptrSize)) { |
| 1555 | // Print in bytes and use the mask() operator for pointers. |
| 1556 | if (!STI.hasMaskOperator()) |
| 1557 | report_fatal_error(reason: "initialized packed aggregate with pointers '" + |
| 1558 | GVar->getName() + |
| 1559 | "' requires at least PTX ISA version 7.1" ); |
| 1560 | O << " .u8 " ; |
| 1561 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1562 | O << "[" << ElementSize << "]" ; |
| 1563 | if (EmitInitializer) { |
| 1564 | O << " = {" ; |
| 1565 | aggBuffer.printBytes(os&: O); |
| 1566 | O << "}" ; |
| 1567 | } |
| 1568 | } else { |
| 1569 | O << " .u" << ptrSize * 8 << " " ; |
| 1570 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1571 | O << "[" << ElementSize / ptrSize << "]" ; |
| 1572 | if (EmitInitializer) { |
| 1573 | O << " = {" ; |
| 1574 | aggBuffer.printWords(os&: O); |
| 1575 | O << "}" ; |
| 1576 | } |
| 1577 | } |
| 1578 | } else { |
| 1579 | O << " .b8 " ; |
| 1580 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1581 | O << "[" << ElementSize << "]" ; |
| 1582 | if (EmitInitializer) { |
| 1583 | O << " = {" ; |
| 1584 | aggBuffer.printBytes(os&: O); |
| 1585 | O << "}" ; |
| 1586 | } |
| 1587 | } |
| 1588 | } |
| 1589 | |
| 1590 | void NVPTXAsmPrinter::AggBuffer::printSymbol(unsigned nSym, raw_ostream &os) { |
| 1591 | const Value *v = Symbols[nSym]; |
| 1592 | const Value *v0 = SymbolsBeforeStripping[nSym]; |
| 1593 | if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: v)) { |
| 1594 | MCSymbol *Name = AP.getSymbol(GV: GVar); |
| 1595 | PointerType *PTy = dyn_cast<PointerType>(Val: v0->getType()); |
| 1596 | // Is v0 a generic pointer? |
| 1597 | bool isGenericPointer = PTy && PTy->getAddressSpace() == 0; |
| 1598 | if (EmitGeneric && isGenericPointer && !isa<Function>(Val: v)) { |
| 1599 | os << "generic(" ; |
| 1600 | Name->print(OS&: os, MAI: AP.MAI); |
| 1601 | os << ")" ; |
| 1602 | } else { |
| 1603 | Name->print(OS&: os, MAI: AP.MAI); |
| 1604 | } |
| 1605 | } else if (const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(Val: v0)) { |
| 1606 | const MCExpr *Expr = AP.lowerConstantForGV(CV: CExpr, ProcessingGeneric: false); |
| 1607 | AP.printMCExpr(Expr: *Expr, OS&: os); |
| 1608 | } else |
| 1609 | llvm_unreachable("symbol type unknown" ); |
| 1610 | } |
| 1611 | |
| 1612 | void NVPTXAsmPrinter::AggBuffer::printBytes(raw_ostream &os) { |
| 1613 | unsigned int ptrSize = AP.MAI.getCodePointerSize(); |
| 1614 | // Do not emit trailing zero initializers. They will be zero-initialized by |
| 1615 | // ptxas. This saves on both space requirements for the generated PTX and on |
| 1616 | // memory use by ptxas. (See: |
| 1617 | // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#global-state-space) |
| 1618 | unsigned int InitializerCount = Size; |
| 1619 | // TODO: symbols make this harder, but it would still be good to trim trailing |
| 1620 | // 0s for aggs with symbols as well. |
| 1621 | if (numSymbols() == 0) |
| 1622 | while (InitializerCount >= 1 && !buffer[InitializerCount - 1]) |
| 1623 | InitializerCount--; |
| 1624 | |
| 1625 | symbolPosInBuffer.push_back(Elt: InitializerCount); |
| 1626 | unsigned int nSym = 0; |
| 1627 | unsigned int nextSymbolPos = symbolPosInBuffer[nSym]; |
| 1628 | for (unsigned int pos = 0; pos < InitializerCount;) { |
| 1629 | if (pos) |
| 1630 | os << ", " ; |
| 1631 | if (pos != nextSymbolPos) { |
| 1632 | os << (unsigned int)buffer[pos]; |
| 1633 | ++pos; |
| 1634 | continue; |
| 1635 | } |
| 1636 | // Generate a per-byte mask() operator for the symbol, which looks like: |
| 1637 | // .global .u8 addr[] = {0xFF(foo), 0xFF00(foo), 0xFF0000(foo), ...}; |
| 1638 | // See https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#initializers |
| 1639 | std::string symText; |
| 1640 | llvm::raw_string_ostream oss(symText); |
| 1641 | printSymbol(nSym, os&: oss); |
| 1642 | for (unsigned i = 0; i < ptrSize; ++i) { |
| 1643 | if (i) |
| 1644 | os << ", " ; |
| 1645 | llvm::write_hex(S&: os, N: 0xFFULL << i * 8, Style: HexPrintStyle::PrefixUpper); |
| 1646 | os << "(" << symText << ")" ; |
| 1647 | } |
| 1648 | pos += ptrSize; |
| 1649 | nextSymbolPos = symbolPosInBuffer[++nSym]; |
| 1650 | assert(nextSymbolPos >= pos); |
| 1651 | } |
| 1652 | } |
| 1653 | |
| 1654 | void NVPTXAsmPrinter::AggBuffer::printWords(raw_ostream &os) { |
| 1655 | unsigned int ptrSize = AP.MAI.getCodePointerSize(); |
| 1656 | symbolPosInBuffer.push_back(Elt: Size); |
| 1657 | unsigned int nSym = 0; |
| 1658 | unsigned int nextSymbolPos = symbolPosInBuffer[nSym]; |
| 1659 | assert(nextSymbolPos % ptrSize == 0); |
| 1660 | for (unsigned int pos = 0; pos < Size; pos += ptrSize) { |
| 1661 | if (pos) |
| 1662 | os << ", " ; |
| 1663 | if (pos == nextSymbolPos) { |
| 1664 | printSymbol(nSym, os); |
| 1665 | nextSymbolPos = symbolPosInBuffer[++nSym]; |
| 1666 | assert(nextSymbolPos % ptrSize == 0); |
| 1667 | assert(nextSymbolPos >= pos + ptrSize); |
| 1668 | } else if (ptrSize == 4) |
| 1669 | os << support::endian::read32le(P: &buffer[pos]); |
| 1670 | else |
| 1671 | os << support::endian::read64le(P: &buffer[pos]); |
| 1672 | } |
| 1673 | } |
| 1674 | |
| 1675 | void NVPTXAsmPrinter::emitDemotedVars(const Function *F, raw_ostream &O) { |
| 1676 | auto It = localDecls.find(x: F); |
| 1677 | if (It == localDecls.end()) |
| 1678 | return; |
| 1679 | |
| 1680 | ArrayRef<const GlobalVariable *> GVars = It->second; |
| 1681 | |
| 1682 | const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM); |
| 1683 | const NVPTXSubtarget &STI = *NTM.getSubtargetImpl(); |
| 1684 | |
| 1685 | for (const GlobalVariable *GV : GVars) { |
| 1686 | O << "\t// demoted variable\n\t" ; |
| 1687 | printModuleLevelGV(GVar: GV, O, /*processDemoted=*/ProcessDemoted: true, STI); |
| 1688 | } |
| 1689 | } |
| 1690 | |
| 1691 | /// The PTX state space directive for \p AddressSpace, or an empty string if it |
| 1692 | /// does not name one, as is the case for the generic address space. |
| 1693 | static StringRef getPTXAddressSpaceName(unsigned AddressSpace) { |
| 1694 | switch (AddressSpace) { |
| 1695 | case ADDRESS_SPACE_LOCAL: |
| 1696 | return ".local" ; |
| 1697 | case ADDRESS_SPACE_GLOBAL: |
| 1698 | return ".global" ; |
| 1699 | case ADDRESS_SPACE_CONST: |
| 1700 | return ".const" ; |
| 1701 | case ADDRESS_SPACE_SHARED: |
| 1702 | return ".shared" ; |
| 1703 | default: |
| 1704 | return {}; |
| 1705 | } |
| 1706 | } |
| 1707 | |
| 1708 | /// The PTX opaque type directive for an image or sampler handle, or an empty |
| 1709 | /// string for PTXOpaqueType::None. |
| 1710 | static StringRef getPTXOpaqueTypeName(PTXOpaqueType OpaqueType) { |
| 1711 | switch (OpaqueType) { |
| 1712 | case PTXOpaqueType::Sampler: |
| 1713 | return ".samplerref" ; |
| 1714 | case PTXOpaqueType::Texture: |
| 1715 | return ".texref" ; |
| 1716 | case PTXOpaqueType::Surface: |
| 1717 | return ".surfref" ; |
| 1718 | case PTXOpaqueType::None: |
| 1719 | return {}; |
| 1720 | } |
| 1721 | llvm_unreachable("unexpected PTXOpaqueType" ); |
| 1722 | } |
| 1723 | |
| 1724 | void NVPTXAsmPrinter::emitPTXAddressSpace(unsigned int AddressSpace, |
| 1725 | raw_ostream &O) const { |
| 1726 | const StringRef Name = getPTXAddressSpaceName(AddressSpace); |
| 1727 | if (Name.empty()) |
| 1728 | report_fatal_error(reason: "Bad address space found while emitting PTX: " + |
| 1729 | llvm::Twine(AddressSpace)); |
| 1730 | O << Name; |
| 1731 | } |
| 1732 | |
| 1733 | std::string NVPTXAsmPrinter::getPTXFundamentalTypeStr(Type *Ty) const { |
| 1734 | switch (Ty->getTypeID()) { |
| 1735 | case Type::IntegerTyID: |
| 1736 | case Type::PointerTyID: { |
| 1737 | const uint64_t NumBits = getDataLayout().getTypeStoreSizeInBits(Ty); |
| 1738 | assert(NumBits <= 64 && "type too large" ); |
| 1739 | return "u" + utostr(X: promoteScalarKernelArgumentSize(Size: NumBits)); |
| 1740 | } |
| 1741 | case Type::BFloatTyID: |
| 1742 | case Type::HalfTyID: |
| 1743 | case Type::FloatTyID: |
| 1744 | case Type::DoubleTyID: |
| 1745 | return "b" + utostr(X: Ty->getScalarSizeInBits()); |
| 1746 | default: |
| 1747 | break; |
| 1748 | } |
| 1749 | llvm_unreachable("unexpected type" ); |
| 1750 | } |
| 1751 | |
| 1752 | void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) { |
| 1753 | const DataLayout &DL = getDataLayout(); |
| 1754 | const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(F: *F); |
| 1755 | const auto *TLI = cast<NVPTXTargetLowering>(Val: STI.getTargetLowering()); |
| 1756 | const NVPTXMachineFunctionInfo *MFI = |
| 1757 | MF ? MF->getInfo<NVPTXMachineFunctionInfo>() : nullptr; |
| 1758 | |
| 1759 | const bool IsKernelFunc = isKernelFunction(F: *F); |
| 1760 | |
| 1761 | // Zero-sized arguments (e.g. empty structs) do not produce a parameter. |
| 1762 | // Number the emitted parameters contiguously, skipping the zero-sized ones, |
| 1763 | // so that the names match those used in LowerFormalArguments and the |
| 1764 | // contiguous numbering used by callers (see LowerCall). |
| 1765 | const auto NonEmptyArgs = |
| 1766 | make_filter_range(Range: F->args(), Pred: [](const Argument &Arg) { |
| 1767 | return !Arg.getType()->isEmptyTy(); |
| 1768 | }); |
| 1769 | |
| 1770 | if (NonEmptyArgs.empty() && !F->isVarArg()) { |
| 1771 | O << "()" ; |
| 1772 | return; |
| 1773 | } |
| 1774 | |
| 1775 | O << "(\n" ; |
| 1776 | |
| 1777 | auto MakeParam = [&](const auto &IndexedArg) { |
| 1778 | const auto &[ParamIndex, Arg] = IndexedArg; |
| 1779 | Type *Ty = Arg.getType(); |
| 1780 | MCSymbol *const ParamSym = TLI->getParamSymbol(Ctx&: OutContext, F, Idx: ParamIndex); |
| 1781 | |
| 1782 | O << "\t" ; |
| 1783 | |
| 1784 | // A byval param is passed as a copy of the pointee and an aggregate is |
| 1785 | // passed as a blob of bytes; both are declared as a byte array. |
| 1786 | const bool IsByVal = Arg.hasByValAttr(); |
| 1787 | const bool AsArray = IsByVal || shouldPassAsArray(Ty); |
| 1788 | |
| 1789 | // Kernels declare image/sampler handles and the address space of a |
| 1790 | // pointee. Both of those are scalar handles, so a byte-array param is |
| 1791 | // neither. |
| 1792 | if (IsKernelFunc && !AsArray) { |
| 1793 | const StringRef OpaqueType = getPTXOpaqueTypeName(getPTXOpaqueType(Arg)); |
| 1794 | if (!OpaqueType.empty()) { |
| 1795 | O << ".param " ; |
| 1796 | if (!MFI || !MFI->checkImageHandleSymbol(Symbol: ParamSym)) |
| 1797 | O << ".u64 .ptr " ; |
| 1798 | |
| 1799 | O << OpaqueType << " " << *ParamSym; |
| 1800 | return; |
| 1801 | } |
| 1802 | |
| 1803 | if (auto *PTy = dyn_cast<PointerType>(Val: Ty)) { |
| 1804 | const unsigned AS = PTy->getAddressSpace(); |
| 1805 | O << ".param .u" << DL.getPointerSizeInBits(AS) << " .ptr" ; |
| 1806 | |
| 1807 | const StringRef Space = getPTXAddressSpaceName(AddressSpace: AS); |
| 1808 | if (!Space.empty()) |
| 1809 | O << " " << Space; |
| 1810 | |
| 1811 | O << " .align " << Arg.getParamAlign().valueOrOne().value() << " " |
| 1812 | << *ParamSym; |
| 1813 | return; |
| 1814 | } |
| 1815 | } |
| 1816 | |
| 1817 | printParam(F, IsByVal ? Arg.getParamByValType() : Ty, |
| 1818 | Arg.getArgNo() + AttributeList::FirstArgIndex, IsByVal, |
| 1819 | IsKernelFunc, ParamSym->getName(), DL, O); |
| 1820 | }; |
| 1821 | |
| 1822 | interleave(c: enumerate(First: NonEmptyArgs), os&: O, each_fn: MakeParam, separator: ",\n" ); |
| 1823 | |
| 1824 | if (F->isVarArg()) |
| 1825 | O << (NonEmptyArgs.empty() ? "" : ",\n" ) << "\t.param .align " |
| 1826 | << STI.getMaxRequiredAlignment() << " .b8 " |
| 1827 | << *TLI->getParamSymbol(Ctx&: OutContext, F, /* vararg */ Idx: -1) << "[]" ; |
| 1828 | |
| 1829 | O << "\n)" ; |
| 1830 | } |
| 1831 | |
| 1832 | void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters( |
| 1833 | const MachineFunction &MF) { |
| 1834 | auto *TS = getTargetStreamer(); |
| 1835 | |
| 1836 | // Emit the Fake Stack Object |
| 1837 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 1838 | if (const int64_t NumBytes = MFI.getStackSize()) { |
| 1839 | TS->emitLocalDirective(Alignment: MFI.getMaxAlign(), Name: getFunctionFrameSymbol(), |
| 1840 | Size: NumBytes); |
| 1841 | |
| 1842 | // Declare the frame pointers that NVPTXFrameLowering's prologue defines. |
| 1843 | const NVPTXRegisterInfo *NRI = |
| 1844 | MF.getSubtarget<NVPTXSubtarget>().getRegisterInfo(); |
| 1845 | for (const Register FrameReg : |
| 1846 | {NRI->getFrameRegister(MF), NRI->getFrameLocalRegister(MF)}) |
| 1847 | TS->emitRegDirective( |
| 1848 | SizeInBits: NRI->getRegSizeInBits(Reg: FrameReg, MRI: *MRI).getFixedValue(), |
| 1849 | Name: NVPTXInstPrinter::getRegisterName(Reg: FrameReg)); |
| 1850 | } |
| 1851 | |
| 1852 | // Go through all virtual registers to establish the mapping between the |
| 1853 | // global virtual |
| 1854 | // register number and the per class virtual register number. |
| 1855 | // We use the per class virtual register number in the ptx output. |
| 1856 | for (unsigned I : llvm::seq(Size: MRI->getNumVirtRegs())) { |
| 1857 | Register VR = Register::index2VirtReg(Index: I); |
| 1858 | if (MRI->use_empty(RegNo: VR) && MRI->def_empty(RegNo: VR)) |
| 1859 | continue; |
| 1860 | auto &RCRegMap = VRegMapping[MRI->getRegClass(Reg: VR)]; |
| 1861 | RCRegMap[VR] = RCRegMap.size() + 1; |
| 1862 | } |
| 1863 | |
| 1864 | // Emit declaration of the virtual registers or 'physical' registers for |
| 1865 | // each register class |
| 1866 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
| 1867 | for (const TargetRegisterClass &RC : TRI->regclasses()) { |
| 1868 | // Only declare those registers that may be used. |
| 1869 | const auto It = VRegMapping.find(Val: &RC); |
| 1870 | if (It == VRegMapping.end() || It->second.empty()) |
| 1871 | continue; |
| 1872 | |
| 1873 | TS->emitRegDirective( |
| 1874 | SizeInBits: TRI->getRegSizeInBits(RC).getFixedValue(), |
| 1875 | Name: NVPTX::getVirtualRegisterPrefix(Kind: getVirtualRegisterKind(RC: &RC)), |
| 1876 | Count: It->second.size() + 1); |
| 1877 | } |
| 1878 | } |
| 1879 | |
| 1880 | /// Translate virtual register numbers in DebugInfo locations to their printed |
| 1881 | /// encodings, as used by CUDA-GDB. |
| 1882 | void NVPTXAsmPrinter::encodeDebugInfoRegisterNumbers( |
| 1883 | const MachineFunction &MF) { |
| 1884 | const NVPTXSubtarget &STI = MF.getSubtarget<NVPTXSubtarget>(); |
| 1885 | const NVPTXRegisterInfo *NRI = STI.getRegisterInfo(); |
| 1886 | |
| 1887 | // Clear the old mapping, and add the new one. This mapping is used after the |
| 1888 | // printing of the current function is complete, but before the next function |
| 1889 | // is printed. |
| 1890 | NRI->clearDebugRegisterMap(); |
| 1891 | |
| 1892 | for (const VRegMap &RegMap : make_second_range(c&: VRegMapping)) |
| 1893 | for (const Register Reg : make_first_range(c: RegMap)) |
| 1894 | NRI->addToDebugRegisterMap(VirtReg: Reg, RegisterName: getVirtualRegisterName(Reg)); |
| 1895 | } |
| 1896 | |
| 1897 | void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp, |
| 1898 | raw_ostream &O) const { |
| 1899 | if (Fp->getType()->isFloatTy()) |
| 1900 | O << "0f" ; |
| 1901 | else if (Fp->getType()->isDoubleTy()) |
| 1902 | O << "0d" ; |
| 1903 | else |
| 1904 | llvm_unreachable("unsupported fp type" ); |
| 1905 | |
| 1906 | const APInt API = Fp->getValueAPF().bitcastToAPInt(); |
| 1907 | O << format_hex_no_prefix(N: API.getZExtValue(), Width: API.getBitWidth() / 4, |
| 1908 | /*Upper=*/true); |
| 1909 | } |
| 1910 | |
| 1911 | void NVPTXAsmPrinter::printScalarConstant(const Constant *CPV, raw_ostream &O) { |
| 1912 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) { |
| 1913 | O << CI->getValue(); |
| 1914 | return; |
| 1915 | } |
| 1916 | if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) { |
| 1917 | const APInt API = CFP->getValueAPF().bitcastToAPInt(); |
| 1918 | O << "0x" |
| 1919 | << format_hex_no_prefix(N: API.getZExtValue(), Width: API.getBitWidth() / 4, |
| 1920 | /*Upper=*/true); |
| 1921 | return; |
| 1922 | } |
| 1923 | if (isa<ConstantPointerNull>(Val: CPV)) { |
| 1924 | O << "0" ; |
| 1925 | return; |
| 1926 | } |
| 1927 | if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) { |
| 1928 | const bool IsNonGenericPointer = GVar->getAddressSpace() != 0; |
| 1929 | if (EmitGeneric && !isa<Function>(Val: CPV) && !IsNonGenericPointer) { |
| 1930 | O << "generic(" ; |
| 1931 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1932 | O << ")" ; |
| 1933 | } else { |
| 1934 | getSymbol(GV: GVar)->print(OS&: O, MAI); |
| 1935 | } |
| 1936 | return; |
| 1937 | } |
| 1938 | if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) { |
| 1939 | const MCExpr *E = lowerConstantForGV(CV: cast<Constant>(Val: Cexpr), ProcessingGeneric: false); |
| 1940 | printMCExpr(Expr: *E, OS&: O); |
| 1941 | return; |
| 1942 | } |
| 1943 | llvm_unreachable("Not scalar type found in printScalarConstant()" ); |
| 1944 | } |
| 1945 | |
| 1946 | void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes, |
| 1947 | AggBuffer *AggBuffer) { |
| 1948 | const DataLayout &DL = getDataLayout(); |
| 1949 | int AllocSize = DL.getTypeAllocSize(Ty: CPV->getType()); |
| 1950 | if (isa<UndefValue>(Val: CPV) || CPV->isNullValue()) { |
| 1951 | // Non-zero Bytes indicates that we need to zero-fill everything. Otherwise, |
| 1952 | // only the space allocated by CPV. |
| 1953 | AggBuffer->addZeros(Num: Bytes ? Bytes : AllocSize); |
| 1954 | return; |
| 1955 | } |
| 1956 | |
| 1957 | // Helper for filling AggBuffer with APInts. |
| 1958 | auto AddIntToBuffer = [AggBuffer, Bytes](const APInt &Val) { |
| 1959 | size_t NumBytes = (Val.getBitWidth() + 7) / 8; |
| 1960 | SmallVector<unsigned char, 16> Buf(NumBytes); |
| 1961 | // `extractBitsAsZExtValue` does not allow the extraction of bits beyond the |
| 1962 | // input's bit width, and i1 arrays may not have a length that is a multuple |
| 1963 | // of 8. We handle the last byte separately, so we never request out of |
| 1964 | // bounds bits. |
| 1965 | for (unsigned I = 0; I < NumBytes - 1; ++I) { |
| 1966 | Buf[I] = Val.extractBitsAsZExtValue(numBits: 8, bitPosition: I * 8); |
| 1967 | } |
| 1968 | size_t LastBytePosition = (NumBytes - 1) * 8; |
| 1969 | size_t LastByteBits = Val.getBitWidth() - LastBytePosition; |
| 1970 | Buf[NumBytes - 1] = |
| 1971 | Val.extractBitsAsZExtValue(numBits: LastByteBits, bitPosition: LastBytePosition); |
| 1972 | AggBuffer->addBytes(Ptr: Buf.data(), Num: NumBytes, Bytes); |
| 1973 | }; |
| 1974 | |
| 1975 | switch (CPV->getType()->getTypeID()) { |
| 1976 | case Type::IntegerTyID: |
| 1977 | if (const auto *CI = dyn_cast<ConstantInt>(Val: CPV)) { |
| 1978 | AddIntToBuffer(CI->getValue()); |
| 1979 | break; |
| 1980 | } |
| 1981 | if (const auto *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) { |
| 1982 | if (const auto *CI = |
| 1983 | dyn_cast<ConstantInt>(Val: ConstantFoldConstant(C: Cexpr, DL))) { |
| 1984 | AddIntToBuffer(CI->getValue()); |
| 1985 | break; |
| 1986 | } |
| 1987 | if (Cexpr->getOpcode() == Instruction::PtrToInt) { |
| 1988 | Value *V = Cexpr->getOperand(i_nocapture: 0)->stripPointerCasts(); |
| 1989 | AggBuffer->addSymbol(GVar: V, GVarBeforeStripping: Cexpr->getOperand(i_nocapture: 0)); |
| 1990 | AggBuffer->addZeros(Num: AllocSize); |
| 1991 | break; |
| 1992 | } |
| 1993 | // A symbol-relative integer whose offset is applied outside the |
| 1994 | // ptrtoint, e.g. add(ptrtoint(@g), C). It can't fold to a ConstantInt |
| 1995 | // because it references a symbol; emit it through lowerConstantForGV, the |
| 1996 | // same path scalar symbol-relative integer globals use. |
| 1997 | AggBuffer->addSymbol(GVar: Cexpr, GVarBeforeStripping: Cexpr); |
| 1998 | AggBuffer->addZeros(Num: AllocSize); |
| 1999 | break; |
| 2000 | } |
| 2001 | llvm_unreachable("unsupported integer const type" ); |
| 2002 | break; |
| 2003 | |
| 2004 | case Type::HalfTyID: |
| 2005 | case Type::BFloatTyID: |
| 2006 | case Type::FloatTyID: |
| 2007 | case Type::DoubleTyID: |
| 2008 | case Type::FP128TyID: |
| 2009 | AddIntToBuffer(cast<ConstantFP>(Val: CPV)->getValueAPF().bitcastToAPInt()); |
| 2010 | break; |
| 2011 | |
| 2012 | case Type::PointerTyID: { |
| 2013 | if (const GlobalValue *GVar = dyn_cast<GlobalValue>(Val: CPV)) { |
| 2014 | AggBuffer->addSymbol(GVar, GVarBeforeStripping: GVar); |
| 2015 | } else if (const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(Val: CPV)) { |
| 2016 | const Value *v = Cexpr->stripPointerCasts(); |
| 2017 | AggBuffer->addSymbol(GVar: v, GVarBeforeStripping: Cexpr); |
| 2018 | } |
| 2019 | AggBuffer->addZeros(Num: AllocSize); |
| 2020 | break; |
| 2021 | } |
| 2022 | |
| 2023 | case Type::ArrayTyID: |
| 2024 | case Type::FixedVectorTyID: |
| 2025 | case Type::StructTyID: { |
| 2026 | if (isa<ConstantAggregate, ConstantDataSequential, ConstantInt, ConstantFP>( |
| 2027 | Val: CPV)) { |
| 2028 | // bufferAggregateConstant doesn't emit tail-padding, i.e. it writes |
| 2029 | // `store_size` bytes, not `alloc_size` bytes. Do it ourselves here. |
| 2030 | unsigned StartPos = AggBuffer->getCurpos(); |
| 2031 | bufferAggregateConstant(CV: CPV, aggBuffer: AggBuffer); |
| 2032 | unsigned Written = AggBuffer->getCurpos() - StartPos; |
| 2033 | unsigned SlotSize = std::max<int>(a: Bytes, b: AllocSize); |
| 2034 | assert(Written <= SlotSize && "Constant overflows its slot" ); |
| 2035 | if (SlotSize > Written) |
| 2036 | AggBuffer->addZeros(Num: SlotSize - Written); |
| 2037 | } else if (isa<ConstantAggregateZero>(Val: CPV)) |
| 2038 | AggBuffer->addZeros(Num: Bytes); |
| 2039 | else |
| 2040 | llvm_unreachable("Unexpected Constant type" ); |
| 2041 | break; |
| 2042 | } |
| 2043 | |
| 2044 | default: |
| 2045 | llvm_unreachable("unsupported type" ); |
| 2046 | } |
| 2047 | } |
| 2048 | |
| 2049 | void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV, |
| 2050 | AggBuffer *aggBuffer) { |
| 2051 | const DataLayout &DL = getDataLayout(); |
| 2052 | |
| 2053 | auto ExtendBuffer = [](APInt Val, AggBuffer *Buffer) { |
| 2054 | unsigned NumBytes = divideCeil(Numerator: Val.getBitWidth(), Denominator: 8); |
| 2055 | for (unsigned I : llvm::seq(Size: NumBytes)) { |
| 2056 | unsigned NumBits = std::min(a: 8u, b: Val.getBitWidth() - I * 8); |
| 2057 | Buffer->addByte(Byte: Val.extractBitsAsZExtValue(numBits: NumBits, bitPosition: I * 8)); |
| 2058 | } |
| 2059 | }; |
| 2060 | |
| 2061 | if (auto *VTy = dyn_cast<FixedVectorType>(Val: CPV->getType())) { |
| 2062 | // Elements that aren't a whole number of bytes are bit-packed, so buffer |
| 2063 | // the vector as one integer. |
| 2064 | if (!DL.typeSizeEqualsStoreSize(Ty: VTy->getElementType())) { |
| 2065 | Type *IntTy = |
| 2066 | IntegerType::get(C&: CPV->getContext(), NumBits: VTy->getPrimitiveSizeInBits()); |
| 2067 | auto *Packed = dyn_cast<ConstantInt>(Val: ConstantFoldConstant( |
| 2068 | C: ConstantExpr::getBitCast(C: const_cast<Constant *>(CPV), Ty: IntTy), DL)); |
| 2069 | if (!Packed) |
| 2070 | report_fatal_error( |
| 2071 | reason: "Cannot lower vector global with unusual element type" ); |
| 2072 | ExtendBuffer(Packed->getValue(), aggBuffer); |
| 2073 | return; |
| 2074 | } |
| 2075 | |
| 2076 | // Integer or floating point vector splats. |
| 2077 | if (isa<ConstantInt, ConstantFP>(Val: CPV)) { |
| 2078 | Constant *Splat = CPV->getSplatValue(); |
| 2079 | for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) |
| 2080 | bufferLEByte(CPV: Splat, Bytes: 0, AggBuffer: aggBuffer); |
| 2081 | return; |
| 2082 | } |
| 2083 | } |
| 2084 | |
| 2085 | // Integers of arbitrary width |
| 2086 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CPV)) { |
| 2087 | assert(CI->getType()->isIntegerTy() && "Expected integer constant!" ); |
| 2088 | ExtendBuffer(CI->getValue(), aggBuffer); |
| 2089 | return; |
| 2090 | } |
| 2091 | |
| 2092 | // f128 |
| 2093 | if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CPV)) { |
| 2094 | assert(CFP->getType()->isFloatingPointTy() && "Expected fp constant!" ); |
| 2095 | if (CFP->getType()->isFP128Ty()) { |
| 2096 | ExtendBuffer(CFP->getValueAPF().bitcastToAPInt(), aggBuffer); |
| 2097 | return; |
| 2098 | } |
| 2099 | } |
| 2100 | |
| 2101 | // Buffer arrays and vectors one element at a time. |
| 2102 | if (isa<ConstantArray, ConstantVector>(Val: CPV)) { |
| 2103 | for (const auto &Op : CPV->operands()) |
| 2104 | bufferLEByte(CPV: cast<Constant>(Val: Op), Bytes: 0, AggBuffer: aggBuffer); |
| 2105 | return; |
| 2106 | } |
| 2107 | |
| 2108 | if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: CPV)) { |
| 2109 | for (unsigned I : llvm::seq(Size: CDS->getNumElements())) |
| 2110 | bufferLEByte(CPV: cast<Constant>(Val: CDS->getElementAsConstant(i: I)), Bytes: 0, AggBuffer: aggBuffer); |
| 2111 | return; |
| 2112 | } |
| 2113 | |
| 2114 | if (isa<ConstantStruct>(Val: CPV)) { |
| 2115 | if (CPV->getNumOperands()) { |
| 2116 | StructType *ST = cast<StructType>(Val: CPV->getType()); |
| 2117 | for (unsigned I : llvm::seq(Size: CPV->getNumOperands())) { |
| 2118 | int EndOffset = (I + 1 == CPV->getNumOperands()) |
| 2119 | ? DL.getStructLayout(Ty: ST)->getElementOffset(Idx: 0) + |
| 2120 | DL.getTypeAllocSize(Ty: ST) |
| 2121 | : DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I + 1); |
| 2122 | int Bytes = EndOffset - DL.getStructLayout(Ty: ST)->getElementOffset(Idx: I); |
| 2123 | bufferLEByte(CPV: cast<Constant>(Val: CPV->getOperand(i: I)), Bytes, AggBuffer: aggBuffer); |
| 2124 | } |
| 2125 | } |
| 2126 | return; |
| 2127 | } |
| 2128 | llvm_unreachable("unsupported constant type in printAggregateConstant()" ); |
| 2129 | } |
| 2130 | |
| 2131 | /// lowerConstantForGV - Return an MCExpr for the given Constant. This is mostly |
| 2132 | /// a copy from AsmPrinter::lowerConstant, except customized to only handle |
| 2133 | /// expressions that are representable in PTX and create |
| 2134 | /// NVPTXGenericMCSymbolRefExpr nodes for addrspacecast instructions. |
| 2135 | const MCExpr * |
| 2136 | NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV, |
| 2137 | bool ProcessingGeneric) const { |
| 2138 | MCContext &Ctx = OutContext; |
| 2139 | |
| 2140 | if (CV->isNullValue() || isa<UndefValue>(Val: CV)) |
| 2141 | return MCConstantExpr::create(Value: 0, Ctx); |
| 2142 | |
| 2143 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: CV)) |
| 2144 | return MCConstantExpr::create(Value: CI->getZExtValue(), Ctx); |
| 2145 | |
| 2146 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) { |
| 2147 | const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Symbol: getSymbol(GV), Ctx); |
| 2148 | if (ProcessingGeneric) |
| 2149 | return NVPTXGenericMCSymbolRefExpr::create(SymExpr: Expr, Ctx); |
| 2150 | return Expr; |
| 2151 | } |
| 2152 | |
| 2153 | const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: CV); |
| 2154 | if (!CE) { |
| 2155 | llvm_unreachable("Unknown constant value to lower!" ); |
| 2156 | } |
| 2157 | |
| 2158 | switch (CE->getOpcode()) { |
| 2159 | default: |
| 2160 | break; // Error |
| 2161 | |
| 2162 | case Instruction::AddrSpaceCast: { |
| 2163 | // Strip the addrspacecast and pass along the operand |
| 2164 | PointerType *DstTy = cast<PointerType>(Val: CE->getType()); |
| 2165 | if (DstTy->getAddressSpace() == 0) |
| 2166 | return lowerConstantForGV(CV: cast<const Constant>(Val: CE->getOperand(i_nocapture: 0)), ProcessingGeneric: true); |
| 2167 | |
| 2168 | break; // Error |
| 2169 | } |
| 2170 | |
| 2171 | case Instruction::GetElementPtr: { |
| 2172 | const DataLayout &DL = getDataLayout(); |
| 2173 | |
| 2174 | // Generate a symbolic expression for the byte address |
| 2175 | APInt OffsetAI(DL.getPointerTypeSizeInBits(CE->getType()), 0); |
| 2176 | cast<GEPOperator>(Val: CE)->accumulateConstantOffset(DL, Offset&: OffsetAI); |
| 2177 | |
| 2178 | const MCExpr *Base = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), |
| 2179 | ProcessingGeneric); |
| 2180 | if (!OffsetAI) |
| 2181 | return Base; |
| 2182 | |
| 2183 | int64_t Offset = OffsetAI.getSExtValue(); |
| 2184 | return MCBinaryExpr::createAdd(LHS: Base, RHS: MCConstantExpr::create(Value: Offset, Ctx), |
| 2185 | Ctx); |
| 2186 | } |
| 2187 | |
| 2188 | case Instruction::Trunc: |
| 2189 | // We emit the value and depend on the assembler to truncate the generated |
| 2190 | // expression properly. This is important for differences between |
| 2191 | // blockaddress labels. Since the two labels are in the same function, it |
| 2192 | // is reasonable to treat their delta as a 32-bit value. |
| 2193 | [[fallthrough]]; |
| 2194 | case Instruction::BitCast: |
| 2195 | return lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric); |
| 2196 | |
| 2197 | case Instruction::IntToPtr: { |
| 2198 | const DataLayout &DL = getDataLayout(); |
| 2199 | |
| 2200 | // Handle casts to pointers by changing them into casts to the appropriate |
| 2201 | // integer type. This promotes constant folding and simplifies this code. |
| 2202 | Constant *Op = CE->getOperand(i_nocapture: 0); |
| 2203 | Op = ConstantFoldIntegerCast(C: Op, DestTy: DL.getIntPtrType(CV->getType()), |
| 2204 | /*IsSigned*/ false, DL); |
| 2205 | if (Op) |
| 2206 | return lowerConstantForGV(CV: Op, ProcessingGeneric); |
| 2207 | |
| 2208 | break; // Error |
| 2209 | } |
| 2210 | |
| 2211 | case Instruction::PtrToInt: { |
| 2212 | const DataLayout &DL = getDataLayout(); |
| 2213 | |
| 2214 | // Support only foldable casts to/from pointers that can be eliminated by |
| 2215 | // changing the pointer to the appropriately sized integer type. |
| 2216 | Constant *Op = CE->getOperand(i_nocapture: 0); |
| 2217 | Type *Ty = CE->getType(); |
| 2218 | |
| 2219 | const MCExpr *OpExpr = lowerConstantForGV(CV: Op, ProcessingGeneric); |
| 2220 | |
| 2221 | // We can emit the pointer value into this slot if the slot is an |
| 2222 | // integer slot equal to the size of the pointer. |
| 2223 | if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Ty: Op->getType())) |
| 2224 | return OpExpr; |
| 2225 | |
| 2226 | // Otherwise the pointer is smaller than the resultant integer, mask off |
| 2227 | // the high bits so we are sure to get a proper truncation if the input is |
| 2228 | // a constant expr. |
| 2229 | unsigned InBits = DL.getTypeAllocSizeInBits(Ty: Op->getType()); |
| 2230 | const MCExpr *MaskExpr = MCConstantExpr::create(Value: ~0ULL >> (64-InBits), Ctx); |
| 2231 | return MCBinaryExpr::createAnd(LHS: OpExpr, RHS: MaskExpr, Ctx); |
| 2232 | } |
| 2233 | |
| 2234 | // The MC library also has a right-shift operator, but it isn't consistently |
| 2235 | // signed or unsigned between different targets. |
| 2236 | case Instruction::Add: { |
| 2237 | const MCExpr *LHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 0), ProcessingGeneric); |
| 2238 | const MCExpr *RHS = lowerConstantForGV(CV: CE->getOperand(i_nocapture: 1), ProcessingGeneric); |
| 2239 | switch (CE->getOpcode()) { |
| 2240 | default: llvm_unreachable("Unknown binary operator constant cast expr" ); |
| 2241 | case Instruction::Add: return MCBinaryExpr::createAdd(LHS, RHS, Ctx); |
| 2242 | } |
| 2243 | } |
| 2244 | } |
| 2245 | |
| 2246 | // If the code isn't optimized, there may be outstanding folding |
| 2247 | // opportunities. Attempt to fold the expression using DataLayout as a |
| 2248 | // last resort before giving up. |
| 2249 | Constant *C = ConstantFoldConstant(C: CE, DL: getDataLayout()); |
| 2250 | if (C != CE) |
| 2251 | return lowerConstantForGV(CV: C, ProcessingGeneric); |
| 2252 | |
| 2253 | // Otherwise report the problem to the user. |
| 2254 | std::string S; |
| 2255 | raw_string_ostream OS(S); |
| 2256 | OS << "Unsupported expression in static initializer: " ; |
| 2257 | CE->printAsOperand(O&: OS, /*PrintType=*/false, |
| 2258 | M: !MF ? nullptr : MF->getFunction().getParent()); |
| 2259 | report_fatal_error(reason: Twine(OS.str())); |
| 2260 | } |
| 2261 | |
| 2262 | void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) const { |
| 2263 | OutContext.getAsmInfo().printExpr(OS, Expr); |
| 2264 | } |
| 2265 | |
| 2266 | /// PrintAsmOperand - Print out an operand for an inline asm expression. |
| 2267 | /// |
| 2268 | bool NVPTXAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, |
| 2269 | const char *, raw_ostream &O) { |
| 2270 | if (ExtraCode && ExtraCode[0]) { |
| 2271 | if (ExtraCode[1] != 0) |
| 2272 | return true; // Unknown modifier. |
| 2273 | |
| 2274 | switch (ExtraCode[0]) { |
| 2275 | default: |
| 2276 | // See if this is a generic print operand |
| 2277 | return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O); |
| 2278 | case 'r': |
| 2279 | break; |
| 2280 | } |
| 2281 | } |
| 2282 | |
| 2283 | printOperand(MI, OpNum: OpNo, O); |
| 2284 | |
| 2285 | return false; |
| 2286 | } |
| 2287 | |
| 2288 | bool NVPTXAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, |
| 2289 | unsigned OpNo, |
| 2290 | const char *, |
| 2291 | raw_ostream &O) { |
| 2292 | if (ExtraCode && ExtraCode[0]) |
| 2293 | return true; // Unknown modifier |
| 2294 | |
| 2295 | O << '['; |
| 2296 | printMemOperand(MI, OpNum: OpNo, O); |
| 2297 | O << ']'; |
| 2298 | |
| 2299 | return false; |
| 2300 | } |
| 2301 | |
| 2302 | void NVPTXAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, |
| 2303 | raw_ostream &O) { |
| 2304 | const MachineOperand &MO = MI->getOperand(i: OpNum); |
| 2305 | switch (MO.getType()) { |
| 2306 | case MachineOperand::MO_Register: |
| 2307 | if (MO.getReg().isPhysical()) { |
| 2308 | if (MO.getReg() == NVPTX::VRDepot) |
| 2309 | getFunctionFrameSymbol()->print(OS&: O, MAI); |
| 2310 | else |
| 2311 | O << NVPTXInstPrinter::getRegisterName(Reg: MO.getReg()); |
| 2312 | } else { |
| 2313 | O << getVirtualRegisterName(Reg: MO.getReg()); |
| 2314 | } |
| 2315 | break; |
| 2316 | |
| 2317 | case MachineOperand::MO_Immediate: |
| 2318 | O << MO.getImm(); |
| 2319 | break; |
| 2320 | |
| 2321 | case MachineOperand::MO_FPImmediate: |
| 2322 | printFPConstant(Fp: MO.getFPImm(), O); |
| 2323 | break; |
| 2324 | |
| 2325 | case MachineOperand::MO_GlobalAddress: |
| 2326 | PrintSymbolOperand(MO, OS&: O); |
| 2327 | break; |
| 2328 | |
| 2329 | case MachineOperand::MO_MCSymbol: |
| 2330 | MO.getMCSymbol()->print(OS&: O, MAI); |
| 2331 | break; |
| 2332 | |
| 2333 | case MachineOperand::MO_MachineBasicBlock: |
| 2334 | MO.getMBB()->getSymbol()->print(OS&: O, MAI); |
| 2335 | break; |
| 2336 | |
| 2337 | default: |
| 2338 | llvm_unreachable("Operand type not supported." ); |
| 2339 | } |
| 2340 | } |
| 2341 | |
| 2342 | void NVPTXAsmPrinter::printMemOperand(const MachineInstr *MI, unsigned OpNum, |
| 2343 | raw_ostream &O, const char *Modifier) { |
| 2344 | printOperand(MI, OpNum, O); |
| 2345 | |
| 2346 | if (Modifier && strcmp(s1: Modifier, s2: "add" ) == 0) { |
| 2347 | O << ", " ; |
| 2348 | printOperand(MI, OpNum: OpNum + 1, O); |
| 2349 | } else { |
| 2350 | if (MI->getOperand(i: OpNum + 1).isImm() && |
| 2351 | MI->getOperand(i: OpNum + 1).getImm() == 0) |
| 2352 | return; // don't print ',0' or '+0' |
| 2353 | O << "+" ; |
| 2354 | printOperand(MI, OpNum: OpNum + 1, O); |
| 2355 | } |
| 2356 | } |
| 2357 | |
| 2358 | /// Returns true if \p Line begins with an alphabetic character or underscore, |
| 2359 | /// indicating it is a PTX instruction that should receive a .loc directive. |
| 2360 | static bool isPTXInstruction(StringRef Line) { |
| 2361 | StringRef Trimmed = Line.ltrim(); |
| 2362 | return !Trimmed.empty() && |
| 2363 | (std::isalpha(static_cast<unsigned char>(Trimmed[0])) || |
| 2364 | Trimmed[0] == '_'); |
| 2365 | } |
| 2366 | |
| 2367 | /// Returns the DILocation for an inline asm MachineInstr if debug line info |
| 2368 | /// should be emitted, or nullptr otherwise. |
| 2369 | static const DILocation *getInlineAsmDebugLoc(const MachineInstr *MI) { |
| 2370 | if (!MI || !MI->getDebugLoc()) |
| 2371 | return nullptr; |
| 2372 | const DISubprogram *SP = MI->getMF()->getFunction().getSubprogram(); |
| 2373 | if (!SP || SP->getUnit()->getEmissionKind() == DICompileUnit::NoDebug) |
| 2374 | return nullptr; |
| 2375 | const DILocation *DL = MI->getDebugLoc(); |
| 2376 | if (!DL->getFile() || !DL->getLine() || DL->isImplicitCode()) |
| 2377 | return nullptr; |
| 2378 | return DL; |
| 2379 | } |
| 2380 | |
| 2381 | namespace { |
| 2382 | struct InlineAsmInliningContext { |
| 2383 | MCSymbol *FuncNameSym = nullptr; |
| 2384 | unsigned FileIA = 0; |
| 2385 | unsigned LineIA = 0; |
| 2386 | unsigned ColIA = 0; |
| 2387 | |
| 2388 | bool hasInlinedAt() const { return FuncNameSym != nullptr; } |
| 2389 | }; |
| 2390 | } // namespace |
| 2391 | |
| 2392 | /// Resolves the enhanced-lineinfo inlining context for an inline asm debug |
| 2393 | /// location. Returns a default (empty) context if inlining info is unavailable. |
| 2394 | static InlineAsmInliningContext |
| 2395 | getInlineAsmInliningContext(const DILocation *DL, const MachineFunction &MF, |
| 2396 | NVPTXDwarfDebug *NVDD, MCStreamer &Streamer, |
| 2397 | unsigned CUID) { |
| 2398 | InlineAsmInliningContext Ctx; |
| 2399 | const DILocation *InlinedAt = DL->getInlinedAt(); |
| 2400 | if (!InlinedAt || !InlinedAt->getFile() || !NVDD || |
| 2401 | !NVDD->isEnhancedLineinfo(MF)) |
| 2402 | return Ctx; |
| 2403 | const auto *SubProg = getDISubprogram(Scope: DL->getScope()); |
| 2404 | if (!SubProg) |
| 2405 | return Ctx; |
| 2406 | Ctx.FuncNameSym = NVDD->getOrCreateFuncNameSymbol(LinkageName: SubProg->getLinkageName()); |
| 2407 | Ctx.FileIA = Streamer.emitDwarfFileDirective( |
| 2408 | FileNo: 0, Directory: InlinedAt->getFile()->getDirectory(), |
| 2409 | Filename: InlinedAt->getFile()->getFilename(), Checksum: std::nullopt, Source: std::nullopt, CUID); |
| 2410 | Ctx.LineIA = InlinedAt->getLine(); |
| 2411 | Ctx.ColIA = InlinedAt->getColumn(); |
| 2412 | return Ctx; |
| 2413 | } |
| 2414 | |
| 2415 | void NVPTXAsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, |
| 2416 | const MCTargetOptions &MCOptions, |
| 2417 | const MDNode *LocMDNode, |
| 2418 | InlineAsm::AsmDialect Dialect, |
| 2419 | const MachineInstr *MI) { |
| 2420 | assert(!Str.empty() && "Can't emit empty inline asm block" ); |
| 2421 | if (Str.back() == 0) |
| 2422 | Str = Str.substr(Start: 0, N: Str.size() - 1); |
| 2423 | |
| 2424 | auto emitAsmStr = [&](StringRef AsmStr) { |
| 2425 | emitInlineAsmStart(); |
| 2426 | OutStreamer->emitRawText(String: AsmStr); |
| 2427 | emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI); |
| 2428 | }; |
| 2429 | |
| 2430 | const DILocation *DL = getInlineAsmDebugLoc(MI); |
| 2431 | if (!DL) { |
| 2432 | emitAsmStr(Str); |
| 2433 | return; |
| 2434 | } |
| 2435 | |
| 2436 | const DIFile *File = DL->getFile(); |
| 2437 | unsigned Line = DL->getLine(); |
| 2438 | const unsigned Column = DL->getColumn(); |
| 2439 | const unsigned CUID = OutStreamer->getContext().getDwarfCompileUnitID(); |
| 2440 | const unsigned FileNumber = OutStreamer->emitDwarfFileDirective( |
| 2441 | FileNo: 0, Directory: File->getDirectory(), Filename: File->getFilename(), Checksum: std::nullopt, Source: std::nullopt, |
| 2442 | CUID); |
| 2443 | |
| 2444 | auto *NVDD = static_cast<NVPTXDwarfDebug *>(getDwarfDebug()); |
| 2445 | InlineAsmInliningContext InlineCtx = |
| 2446 | getInlineAsmInliningContext(DL, MF: *MI->getMF(), NVDD, Streamer&: *OutStreamer, CUID); |
| 2447 | |
| 2448 | SmallVector<StringRef, 16> Lines; |
| 2449 | Str.split(A&: Lines, Separator: '\n'); |
| 2450 | emitInlineAsmStart(); |
| 2451 | for (const StringRef &L : Lines) { |
| 2452 | StringRef RTrimmed = L.rtrim(Char: '\r'); |
| 2453 | if (isPTXInstruction(Line: L)) { |
| 2454 | if (InlineCtx.hasInlinedAt()) { |
| 2455 | OutStreamer->emitDwarfLocDirectiveWithInlinedAt( |
| 2456 | FileNo: FileNumber, Line, Column, FileIA: InlineCtx.FileIA, LineIA: InlineCtx.LineIA, |
| 2457 | ColumnIA: InlineCtx.ColIA, Sym: InlineCtx.FuncNameSym, DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0, |
| 2458 | FileName: File->getFilename()); |
| 2459 | } else { |
| 2460 | OutStreamer->emitDwarfLocDirective(FileNo: FileNumber, Line, Column, |
| 2461 | DWARF2_FLAG_IS_STMT, Isa: 0, Discriminator: 0, |
| 2462 | FileName: File->getFilename()); |
| 2463 | } |
| 2464 | } |
| 2465 | OutStreamer->emitRawText(String: RTrimmed); |
| 2466 | ++Line; |
| 2467 | } |
| 2468 | emitInlineAsmEnd(StartInfo: STI, EndInfo: nullptr, MI); |
| 2469 | } |
| 2470 | |
| 2471 | char NVPTXAsmPrinter::ID = 0; |
| 2472 | |
| 2473 | INITIALIZE_PASS(NVPTXAsmPrinter, "nvptx-asm-printer" , "NVPTX Assembly Printer" , |
| 2474 | false, false) |
| 2475 | |
| 2476 | // Force static initialization. |
| 2477 | extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void |
| 2478 | LLVMInitializeNVPTXAsmPrinter() { |
| 2479 | RegisterAsmPrinter<NVPTXAsmPrinter> X(getTheNVPTXTarget32()); |
| 2480 | RegisterAsmPrinter<NVPTXAsmPrinter> Y(getTheNVPTXTarget64()); |
| 2481 | } |
| 2482 | |
| 2483 | PreservedAnalyses NVPTXAsmPrinterBeginPass::run(Module &M, |
| 2484 | ModuleAnalysisManager &MAM) { |
| 2485 | AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter(); |
| 2486 | setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer); |
| 2487 | Printer.doInitialization(M); |
| 2488 | return PreservedAnalyses::all(); |
| 2489 | } |
| 2490 | |
| 2491 | PreservedAnalyses |
| 2492 | NVPTXAsmPrinterPass::run(MachineFunction &MF, |
| 2493 | MachineFunctionAnalysisManager &MFAM) { |
| 2494 | AsmPrinter &Printer = |
| 2495 | MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF) |
| 2496 | .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent()) |
| 2497 | ->getPrinter(); |
| 2498 | setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter&: Printer); |
| 2499 | Printer.runOnMachineFunction(MF); |
| 2500 | return PreservedAnalyses::all(); |
| 2501 | } |
| 2502 | |
| 2503 | PreservedAnalyses NVPTXAsmPrinterEndPass::run(Module &M, |
| 2504 | ModuleAnalysisManager &MAM) { |
| 2505 | AsmPrinter &Printer = MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter(); |
| 2506 | setupModuleAsmPrinter(M, MAM, AsmPrinter&: Printer); |
| 2507 | Printer.doFinalization(M); |
| 2508 | return PreservedAnalyses::all(); |
| 2509 | } |
| 2510 | |