1//===-- SPIRVAsmPrinter.cpp - SPIR-V LLVM assembly writer ------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the SPIR-V assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/SPIRVInstPrinter.h"
15#include "SPIRV.h"
16#include "SPIRVAuxDataHandler.h"
17#include "SPIRVInstrInfo.h"
18#include "SPIRVMCInstLower.h"
19#include "SPIRVModuleAnalysis.h"
20#include "SPIRVNonSemanticDebugHandler.h"
21#include "SPIRVSubtarget.h"
22#include "SPIRVTargetMachine.h"
23#include "SPIRVUtils.h"
24#include "TargetInfo/SPIRVTargetInfo.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/CodeGen/AsmPrinter.h"
28#include "llvm/CodeGen/MachineConstantPool.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineModuleInfo.h"
31#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
32#include "llvm/MC/MCAsmInfo.h"
33#include "llvm/MC/MCAssembler.h"
34#include "llvm/MC/MCInst.h"
35#include "llvm/MC/MCObjectStreamer.h"
36#include "llvm/MC/MCSPIRVObjectWriter.h"
37#include "llvm/MC/MCStreamer.h"
38#include "llvm/MC/MCSymbol.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Compiler.h"
42#include "llvm/Support/raw_ostream.h"
43
44using namespace llvm;
45
46#define DEBUG_TYPE "asm-printer"
47
48namespace {
49enum class SPIRVFPContractMode { On, Off, Fast };
50
51static cl::opt<SPIRVFPContractMode> SPIRVFPContract(
52 "spirv-fp-contract",
53 cl::desc("Override FP contraction policy for SPIR-V kernel entry points"),
54 cl::values(
55 clEnumValN(SPIRVFPContractMode::On, "on",
56 "Follow IR metadata (default)"),
57 clEnumValN(SPIRVFPContractMode::Off, "off",
58 "Force ContractionOff on all kernel entry points"),
59 clEnumValN(SPIRVFPContractMode::Fast, "fast",
60 "Suppress ContractionOff on all kernel entry points")),
61 cl::init(Val: SPIRVFPContractMode::On));
62
63class SPIRVAsmPrinter : public AsmPrinter {
64 unsigned NLabels = 0;
65 SmallPtrSet<const MachineBasicBlock *, 8> LabeledMBB;
66
67public:
68 explicit SPIRVAsmPrinter(TargetMachine &TM,
69 std::unique_ptr<MCStreamer> Streamer)
70 : AsmPrinter(TM, std::move(Streamer), ID), ModuleSectionsEmitted(false),
71 ST(nullptr), TII(nullptr), MAI(nullptr) {}
72 static char ID;
73 bool ModuleSectionsEmitted;
74 const SPIRVSubtarget *ST;
75 const SPIRVInstrInfo *TII;
76
77 StringRef getPassName() const override { return "SPIRV Assembly Printer"; }
78 void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
79 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
80 const char *ExtraCode, raw_ostream &O) override;
81
82 void outputMCInst(MCInst &Inst);
83 void outputInstruction(const MachineInstr *MI);
84 void outputModuleSection(SPIRV::ModuleSectionType MSType);
85 void outputGlobalRequirements();
86 void outputEntryPoints();
87 void outputDebugSourceAndStrings(const Module &M);
88 void outputOpExtInstImports(const Module &M);
89 void outputOpMemoryModel();
90 void outputOpFunctionEnd();
91 void outputExtFuncDecls();
92 void outputExecutionModeFromMDNode(MCRegister Reg, MDNode *Node,
93 SPIRV::ExecutionMode::ExecutionMode EM,
94 unsigned ExpectMDOps, int64_t DefVal);
95 void outputExecutionModeFromNumthreadsAttribute(
96 const MCRegister &Reg, const Attribute &Attr,
97 SPIRV::ExecutionMode::ExecutionMode EM);
98 void outputExecutionModeFromEnableMaximalReconvergenceAttr(
99 const MCRegister &Reg, const SPIRVSubtarget &ST);
100 void emitSimpleExecutionMode(MCRegister Reg,
101 SPIRV::ExecutionMode::ExecutionMode EM);
102 void outputExecutionMode(const Module &M);
103 void outputAnnotations(const Module &M);
104 void outputModuleSections();
105 void outputFPFastMathDefaultInfo();
106 bool isHidden() {
107 return MF->getFunction()
108 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
109 .isValid();
110 }
111
112 void emitInstruction(const MachineInstr *MI) override;
113 void emitFunctionEntryLabel() override {}
114 void emitFunctionHeader() override;
115 void emitFunctionBodyStart() override {}
116 void emitFunctionBodyEnd() override;
117 void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
118 void emitBasicBlockEnd(const MachineBasicBlock &MBB) override {}
119 void emitGlobalVariable(const GlobalVariable *GV) override {}
120 void emitOpLabel(const MachineBasicBlock &MBB);
121 void emitEndOfAsmFile(Module &M) override;
122 bool doInitialization(Module &M) override;
123
124 void getAnalysisUsage(AnalysisUsage &AU) const override;
125 SPIRV::ModuleAnalysisInfo *MAI;
126
127 // Non-owning pointer to the NSDI handler registered via addAsmPrinterHandler.
128 // The handler's lifetime is managed by AsmPrinter (the base class of this
129 // object), so this pointer cannot dangle.
130 SPIRVNonSemanticDebugHandler *NSDebugHandler = nullptr;
131
132 std::unique_ptr<SPIRVAuxDataHandler> AuxDataHandler;
133
134protected:
135 void cleanUp(Module &M);
136};
137} // namespace
138
139void SPIRVAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
140 AU.addRequired<SPIRVModuleAnalysis>();
141 AU.addPreserved<SPIRVModuleAnalysis>();
142 AsmPrinter::getAnalysisUsage(AU);
143}
144
145// If the module has no functions, we need output global info anyway.
146void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) {
147 if (!ModuleSectionsEmitted) {
148 outputModuleSections();
149 ModuleSectionsEmitted = true;
150 }
151
152 ST = static_cast<const SPIRVTargetMachine &>(TM).getSubtargetImpl();
153 // SPIRVModuleAnalysis sets GR->Bound = MAI->MaxID before printing. Any IDs
154 // allocated by AsmPrinter handlers (e.g. SPIRVNonSemanticDebugHandler) during
155 // outputModuleSections() are not counted. Refresh the bound here so the
156 // formula below sees the final allocation count.
157 if (MAI)
158 ST->getSPIRVGlobalRegistry()->setBound(MAI->MaxID);
159 VersionTuple SPIRVVersion = ST->getSPIRVVersion();
160 uint32_t Major = SPIRVVersion.getMajor();
161 uint32_t Minor = SPIRVVersion.getMinor().value_or(u: 0);
162 // Bound is an approximation that accounts for the maximum used register
163 // number and number of generated OpLabels
164 unsigned Bound = 2 * (ST->getBound() + 1) + NLabels;
165 if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
166 static_cast<SPIRVObjectWriter &>(Asm->getWriter())
167 .setBuildVersion(Major, Minor, Bound);
168
169 cleanUp(M);
170}
171
172// Any cleanup actions with the Module after we don't care about its content
173// anymore.
174void SPIRVAsmPrinter::cleanUp(Module &M) {
175 // Verifier disallows uses of intrinsic global variables.
176 for (StringRef GVName :
177 {"llvm.global_ctors", "llvm.global_dtors", "llvm.used"}) {
178 if (GlobalVariable *GV = M.getNamedGlobal(Name: GVName))
179 GV->setName("");
180 }
181}
182
183void SPIRVAsmPrinter::emitFunctionHeader() {
184 if (!ModuleSectionsEmitted) {
185 outputModuleSections();
186 ModuleSectionsEmitted = true;
187 }
188 // Get the subtarget from the current MachineFunction.
189 ST = &MF->getSubtarget<SPIRVSubtarget>();
190 TII = ST->getInstrInfo();
191 const Function &F = MF->getFunction();
192
193 if (isVerbose() && !isHidden()) {
194 OutStreamer->getCommentOS()
195 << "-- Begin function "
196 << GlobalValue::dropLLVMManglingEscape(Name: F.getName()) << '\n';
197 }
198
199 auto Section = getObjFileLowering().SectionForGlobal(GO: &F, TM);
200 MF->setSection(Section);
201
202 // SPIRVAsmPrinter::emitFunctionHeader() does not call the base class,
203 // so handlers never receive beginFunction() from the normal path. Drive the
204 // per-function lifecycle here, matching what AsmPrinter::emitFunctionHeader()
205 // does for other targets.
206 for (auto &Handler : Handlers) {
207 Handler->beginFunction(MF);
208 Handler->beginBasicBlockSection(MBB: MF->front());
209 }
210}
211
212void SPIRVAsmPrinter::outputOpFunctionEnd() {
213 MCInst FunctionEndInst;
214 FunctionEndInst.setOpcode(SPIRV::OpFunctionEnd);
215 outputMCInst(Inst&: FunctionEndInst);
216}
217
218void SPIRVAsmPrinter::emitFunctionBodyEnd() {
219 if (!isHidden())
220 outputOpFunctionEnd();
221}
222
223void SPIRVAsmPrinter::emitOpLabel(const MachineBasicBlock &MBB) {
224 // Do not emit anything if it's an internal service function.
225 if (isHidden())
226 return;
227
228 MCInst LabelInst;
229 LabelInst.setOpcode(SPIRV::OpLabel);
230 LabelInst.addOperand(Op: MCOperand::createReg(Reg: MAI->getOrCreateMBBRegister(MBB)));
231 outputMCInst(Inst&: LabelInst);
232 ++NLabels;
233 LabeledMBB.insert(Ptr: &MBB);
234}
235
236void SPIRVAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
237 // Do not emit anything if it's an internal service function.
238 if (MBB.empty() || isHidden())
239 return;
240
241 // If it's the first MBB in MF, it has OpFunction and OpFunctionParameter, so
242 // OpLabel should be output after them.
243 if (MBB.getNumber() == MF->front().getNumber()) {
244 for (const MachineInstr &MI : MBB)
245 if (MI.getOpcode() == SPIRV::OpFunction)
246 return;
247 // TODO: this case should be checked by the verifier.
248 report_fatal_error(reason: "OpFunction is expected in the front MBB of MF");
249 }
250 emitOpLabel(MBB);
251}
252
253void SPIRVAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
254 raw_ostream &O) {
255 const MachineOperand &MO = MI->getOperand(i: OpNum);
256
257 switch (MO.getType()) {
258 case MachineOperand::MO_Register:
259 O << SPIRVInstPrinter::getRegisterName(Reg: MO.getReg());
260 break;
261
262 case MachineOperand::MO_Immediate:
263 O << MO.getImm();
264 break;
265
266 case MachineOperand::MO_FPImmediate:
267 O << MO.getFPImm();
268 break;
269
270 case MachineOperand::MO_MachineBasicBlock:
271 O << *MO.getMBB()->getSymbol();
272 break;
273
274 case MachineOperand::MO_GlobalAddress:
275 O << *getSymbol(GV: MO.getGlobal());
276 break;
277
278 case MachineOperand::MO_BlockAddress: {
279 MCSymbol *BA = GetBlockAddressSymbol(BA: MO.getBlockAddress());
280 O << BA->getName();
281 break;
282 }
283
284 case MachineOperand::MO_ExternalSymbol:
285 O << *GetExternalSymbolSymbol(Sym: MO.getSymbolName());
286 break;
287
288 case MachineOperand::MO_JumpTableIndex:
289 case MachineOperand::MO_ConstantPoolIndex:
290 default:
291 llvm_unreachable("<unknown operand type>");
292 }
293}
294
295bool SPIRVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
296 const char *ExtraCode, raw_ostream &O) {
297 if (ExtraCode && ExtraCode[0])
298 return true; // Invalid instruction - SPIR-V does not have special modifiers
299
300 printOperand(MI, OpNum: OpNo, O);
301 return false;
302}
303
304static bool isFuncOrHeaderInstr(const MachineInstr *MI,
305 const SPIRVInstrInfo *TII) {
306 return TII->isHeaderInstr(MI: *MI) || MI->getOpcode() == SPIRV::OpFunction ||
307 MI->getOpcode() == SPIRV::OpFunctionParameter;
308}
309
310void SPIRVAsmPrinter::outputMCInst(MCInst &Inst) {
311 OutStreamer->emitInstruction(Inst, STI: *OutContext.getSubtargetInfo());
312}
313
314void SPIRVAsmPrinter::outputInstruction(const MachineInstr *MI) {
315 SPIRVMCInstLower MCInstLowering;
316 MCInst TmpInst;
317 MCInstLowering.lower(MI, OutMI&: TmpInst, MAI);
318 outputMCInst(Inst&: TmpInst);
319}
320
321void SPIRVAsmPrinter::emitInstruction(const MachineInstr *MI) {
322 SPIRV_MC::verifyInstructionPredicates(Opcode: MI->getOpcode(),
323 Features: getSubtargetInfo().getFeatureBits());
324
325 if (!MAI->getSkipEmission(MI))
326 outputInstruction(MI);
327
328 // Output OpLabel after OpFunction and OpFunctionParameter in the first MBB.
329 const MachineInstr *NextMI = MI->getNextNode();
330 if (!LabeledMBB.contains(Ptr: MI->getParent()) && isFuncOrHeaderInstr(MI, TII) &&
331 (!NextMI || !isFuncOrHeaderInstr(MI: NextMI, TII))) {
332 assert(MI->getParent()->getNumber() == MF->front().getNumber() &&
333 "OpFunction is not in the front MBB of MF");
334 emitOpLabel(MBB: *MI->getParent());
335 }
336}
337
338void SPIRVAsmPrinter::outputModuleSection(SPIRV::ModuleSectionType MSType) {
339 for (const MachineInstr *MI : MAI->getMSInstrs(MSType))
340 outputInstruction(MI);
341}
342
343void SPIRVAsmPrinter::outputDebugSourceAndStrings(const Module &M) {
344 // Output OpSourceExtensions.
345 for (auto &Str : MAI->SrcExt) {
346 MCInst Inst;
347 Inst.setOpcode(SPIRV::OpSourceExtension);
348 addStringImm(Str: Str.first(), Inst);
349 outputMCInst(Inst);
350 }
351 // Output OpString.
352 outputModuleSection(MSType: SPIRV::MB_DebugStrings);
353 // Output OpSource.
354 MCInst Inst;
355 Inst.setOpcode(SPIRV::OpSource);
356 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(MAI->SrcLang)));
357 Inst.addOperand(
358 Op: MCOperand::createImm(Val: static_cast<unsigned>(MAI->SrcLangVersion)));
359 outputMCInst(Inst);
360 // Emit OpString instructions for NSDI file paths and type names here, in
361 // section 7. OpString must precede type/constant declarations per the SPIR-V
362 // module layout (section 2.4). The OpExtInst instructions that reference
363 // these strings are emitted later at section 10 by
364 // emitNonSemanticGlobalDebugInfo().
365 if (NSDebugHandler)
366 NSDebugHandler->emitNonSemanticDebugStrings(MAI&: *MAI);
367 if (AuxDataHandler)
368 AuxDataHandler->emitAuxDataStrings(MAI&: *MAI);
369}
370
371void SPIRVAsmPrinter::outputOpExtInstImports(const Module &M) {
372 for (auto &CU : MAI->ExtInstSetMap) {
373 unsigned Set = CU.first;
374 MCRegister Reg = CU.second;
375 MCInst Inst;
376 Inst.setOpcode(SPIRV::OpExtInstImport);
377 Inst.addOperand(Op: MCOperand::createReg(Reg));
378 addStringImm(Str: getExtInstSetName(
379 Set: static_cast<SPIRV::InstructionSet::InstructionSet>(Set)),
380 Inst);
381 outputMCInst(Inst);
382 }
383}
384
385void SPIRVAsmPrinter::outputOpMemoryModel() {
386 MCInst Inst;
387 Inst.setOpcode(SPIRV::OpMemoryModel);
388 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(MAI->Addr)));
389 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(MAI->Mem)));
390 outputMCInst(Inst);
391}
392
393// Before the OpEntryPoints' output, we need to add the entry point's
394// interfaces. The interface is a list of IDs of global OpVariable instructions.
395// These declare the set of global variables from a module that form
396// the interface of this entry point.
397void SPIRVAsmPrinter::outputEntryPoints() {
398 // Find all OpVariable IDs with required StorageClass.
399 DenseSet<MCRegister> InterfaceIDs;
400 for (const MachineInstr *MI : MAI->GlobalVarList) {
401 assert(MI->getOpcode() == SPIRV::OpVariable);
402 auto SC = static_cast<SPIRV::StorageClass::StorageClass>(
403 MI->getOperand(i: 2).getImm());
404 // Before version 1.4, the interface's storage classes are limited to
405 // the Input and Output storage classes. Starting with version 1.4,
406 // the interface's storage classes are all storage classes used in
407 // declaring all global variables referenced by the entry point call tree.
408 if (ST->isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)) ||
409 SC == SPIRV::StorageClass::Input || SC == SPIRV::StorageClass::Output) {
410 const MachineFunction *MF = MI->getMF();
411 MCRegister Reg = MAI->getRegisterAlias(MF, Reg: MI->getOperand(i: 0).getReg());
412 InterfaceIDs.insert(V: Reg);
413 }
414 }
415
416 // Output OpEntryPoints adding interface args to all of them.
417 for (const MachineInstr *MI : MAI->getMSInstrs(MSType: SPIRV::MB_EntryPoints)) {
418 SPIRVMCInstLower MCInstLowering;
419 MCInst TmpInst;
420 MCInstLowering.lower(MI, OutMI&: TmpInst, MAI);
421 for (MCRegister Reg : InterfaceIDs) {
422 assert(Reg.isValid());
423 TmpInst.addOperand(Op: MCOperand::createReg(Reg));
424 }
425 outputMCInst(Inst&: TmpInst);
426 }
427}
428
429// Create global OpCapability instructions for the required capabilities.
430void SPIRVAsmPrinter::outputGlobalRequirements() {
431 // Abort here if not all requirements can be satisfied.
432 MAI->Reqs.checkSatisfiable(ST: *ST);
433
434 for (const auto &Cap : MAI->Reqs.getMinimalCapabilities()) {
435 MCInst Inst;
436 Inst.setOpcode(SPIRV::OpCapability);
437 Inst.addOperand(Op: MCOperand::createImm(Val: Cap));
438 outputMCInst(Inst);
439 }
440
441 // Generate the final OpExtensions with strings instead of enums.
442 for (const auto &Ext : MAI->Reqs.getExtensions()) {
443 MCInst Inst;
444 Inst.setOpcode(SPIRV::OpExtension);
445 addStringImm(Str: getSymbolicOperandMnemonic(
446 Category: SPIRV::OperandCategory::ExtensionOperand, Value: Ext),
447 Inst);
448 outputMCInst(Inst);
449 }
450 // TODO add a pseudo instr for version number.
451}
452
453void SPIRVAsmPrinter::outputExtFuncDecls() {
454 // Insert OpFunctionEnd after each declaration.
455 auto I = MAI->getMSInstrs(MSType: SPIRV::MB_ExtFuncDecls).begin(),
456 E = MAI->getMSInstrs(MSType: SPIRV::MB_ExtFuncDecls).end();
457 for (; I != E; ++I) {
458 outputInstruction(MI: *I);
459 if ((I + 1) == E || (*(I + 1))->getOpcode() == SPIRV::OpFunction)
460 outputOpFunctionEnd();
461 }
462}
463
464// Encode LLVM type by SPIR-V execution mode VecTypeHint.
465static unsigned encodeVecTypeHint(Type *Ty) {
466 if (Ty->isHalfTy())
467 return 4;
468 if (Ty->isFloatTy())
469 return 5;
470 if (Ty->isDoubleTy())
471 return 6;
472 if (IntegerType *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
473 switch (IntTy->getIntegerBitWidth()) {
474 case 8:
475 return 0;
476 case 16:
477 return 1;
478 case 32:
479 return 2;
480 case 64:
481 return 3;
482 default:
483 llvm_unreachable("invalid integer type");
484 }
485 }
486 if (FixedVectorType *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
487 Type *EleTy = VecTy->getElementType();
488 unsigned Size = VecTy->getNumElements();
489 return Size << 16 | encodeVecTypeHint(Ty: EleTy);
490 }
491 llvm_unreachable("invalid type");
492}
493
494static void addOpsFromMDNode(MDNode *MDN, MCInst &Inst,
495 SPIRV::ModuleAnalysisInfo *MAI) {
496 for (const MDOperand &MDOp : MDN->operands()) {
497 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
498 Constant *C = CMeta->getValue();
499 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
500 Inst.addOperand(Op: MCOperand::createImm(Val: Const->getZExtValue()));
501 } else if (auto *CE = dyn_cast<Function>(Val: C)) {
502 MCRegister FuncReg = MAI->getGlobalObjReg(GO: CE);
503 assert(FuncReg.isValid());
504 Inst.addOperand(Op: MCOperand::createReg(Reg: FuncReg));
505 }
506 }
507 }
508}
509
510void SPIRVAsmPrinter::outputExecutionModeFromMDNode(
511 MCRegister Reg, MDNode *Node, SPIRV::ExecutionMode::ExecutionMode EM,
512 unsigned ExpectMDOps, int64_t DefVal) {
513 MCInst Inst;
514 Inst.setOpcode(SPIRV::OpExecutionMode);
515 Inst.addOperand(Op: MCOperand::createReg(Reg));
516 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(EM)));
517 addOpsFromMDNode(MDN: Node, Inst, MAI);
518 // reqd_work_group_size and work_group_size_hint require 3 operands,
519 // if metadata contains less operands, just add a default value
520 unsigned NodeSz = Node->getNumOperands();
521 if (ExpectMDOps > 0 && NodeSz < ExpectMDOps)
522 for (unsigned i = NodeSz; i < ExpectMDOps; ++i)
523 Inst.addOperand(Op: MCOperand::createImm(Val: DefVal));
524 outputMCInst(Inst);
525}
526
527void SPIRVAsmPrinter::outputExecutionModeFromNumthreadsAttribute(
528 const MCRegister &Reg, const Attribute &Attr,
529 SPIRV::ExecutionMode::ExecutionMode EM) {
530 assert(Attr.isValid() && "Function called with an invalid attribute.");
531
532 MCInst Inst;
533 Inst.setOpcode(SPIRV::OpExecutionMode);
534 Inst.addOperand(Op: MCOperand::createReg(Reg));
535 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(EM)));
536
537 SmallVector<StringRef> NumThreads;
538 Attr.getValueAsString().split(A&: NumThreads, Separator: ',');
539 assert(NumThreads.size() == 3 && "invalid numthreads");
540 for (uint32_t i = 0; i < 3; ++i) {
541 uint32_t V;
542 [[maybe_unused]] bool Result = NumThreads[i].getAsInteger(Radix: 10, Result&: V);
543 assert(!Result && "Failed to parse numthreads");
544 Inst.addOperand(Op: MCOperand::createImm(Val: V));
545 }
546
547 outputMCInst(Inst);
548}
549
550void SPIRVAsmPrinter::emitSimpleExecutionMode(
551 MCRegister Reg, SPIRV::ExecutionMode::ExecutionMode EM) {
552 MCInst Inst;
553 Inst.setOpcode(SPIRV::OpExecutionMode);
554 Inst.addOperand(Op: MCOperand::createReg(Reg));
555 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<unsigned>(EM)));
556 outputMCInst(Inst);
557}
558
559void SPIRVAsmPrinter::outputExecutionModeFromEnableMaximalReconvergenceAttr(
560 const MCRegister &Reg, const SPIRVSubtarget &ST) {
561 assert(ST.canUseExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence) &&
562 "Function called when SPV_KHR_maximal_reconvergence is not enabled.");
563
564 emitSimpleExecutionMode(Reg, EM: SPIRV::ExecutionMode::MaximallyReconvergesKHR);
565}
566
567void SPIRVAsmPrinter::outputExecutionMode(const Module &M) {
568 NamedMDNode *Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
569 if (Node) {
570 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
571 const auto EM =
572 cast<ConstantInt>(
573 Val: cast<ConstantAsMetadata>(Val: (Node->getOperand(i))->getOperand(I: 1))
574 ->getValue())
575 ->getZExtValue();
576 // Skip ArithmeticPoisonKHR to avoid a duplicate.
577 if (EM == SPIRV::ExecutionMode::ArithmeticPoisonKHR)
578 continue;
579 // If SPV_KHR_float_controls2 is enabled and we find any of
580 // FPFastMathDefault, ContractionOff or SignedZeroInfNanPreserve execution
581 // modes, skip it, it'll be done somewhere else.
582 if (ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
583 if (EM == SPIRV::ExecutionMode::FPFastMathDefault ||
584 EM == SPIRV::ExecutionMode::ContractionOff ||
585 EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve)
586 continue;
587 }
588
589 MCInst Inst;
590 Inst.setOpcode(SPIRV::OpExecutionMode);
591 addOpsFromMDNode(MDN: cast<MDNode>(Val: Node->getOperand(i)), Inst, MAI);
592 outputMCInst(Inst);
593 }
594 outputFPFastMathDefaultInfo();
595 }
596 for (auto FI = M.begin(), E = M.end(); FI != E; ++FI) {
597 const Function &F = *FI;
598 // Only operands of OpEntryPoint instructions are allowed to be
599 // <Entry Point> operands of OpExecutionMode
600 if (F.isDeclaration() || !isEntryPoint(F))
601 continue;
602 MCRegister FReg = MAI->getGlobalObjReg(GO: &F);
603 assert(FReg.isValid());
604
605 if (Attribute Attr = F.getFnAttribute(Kind: "hlsl.shader"); Attr.isValid()) {
606 // SPIR-V common validation: Fragment requires OriginUpperLeft or
607 // OriginLowerLeft.
608 // VUID-StandaloneSpirv-OriginLowerLeft-04653: Fragment must declare
609 // OriginUpperLeft.
610 if (Attr.getValueAsString() == "pixel") {
611 emitSimpleExecutionMode(Reg: FReg, EM: SPIRV::ExecutionMode::OriginUpperLeft);
612 }
613 }
614 if (MDNode *Node = F.getMetadata(Kind: "reqd_work_group_size"))
615 outputExecutionModeFromMDNode(Reg: FReg, Node, EM: SPIRV::ExecutionMode::LocalSize,
616 ExpectMDOps: 3, DefVal: 1);
617 if (Attribute Attr = F.getFnAttribute(Kind: "hlsl.numthreads"); Attr.isValid())
618 outputExecutionModeFromNumthreadsAttribute(
619 Reg: FReg, Attr, EM: SPIRV::ExecutionMode::LocalSize);
620 if (Attribute Attr = F.getFnAttribute(Kind: "enable-maximal-reconvergence");
621 Attr.getValueAsBool()) {
622 outputExecutionModeFromEnableMaximalReconvergenceAttr(Reg: FReg, ST: *ST);
623 }
624 if (MDNode *Node = F.getMetadata(Kind: "work_group_size_hint"))
625 outputExecutionModeFromMDNode(Reg: FReg, Node,
626 EM: SPIRV::ExecutionMode::LocalSizeHint, ExpectMDOps: 3, DefVal: 1);
627 if (MDNode *Node = F.getMetadata(Kind: "reqd_sub_group_size"))
628 outputExecutionModeFromMDNode(Reg: FReg, Node,
629 EM: SPIRV::ExecutionMode::SubgroupSize, ExpectMDOps: 0, DefVal: 0);
630 if (MDNode *Node = F.getMetadata(Kind: "intel_reqd_sub_group_size"))
631 outputExecutionModeFromMDNode(Reg: FReg, Node,
632 EM: SPIRV::ExecutionMode::SubgroupSize, ExpectMDOps: 0, DefVal: 0);
633 if (MDNode *Node = F.getMetadata(Kind: "max_work_group_size")) {
634 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_kernel_attributes))
635 outputExecutionModeFromMDNode(
636 Reg: FReg, Node, EM: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ExpectMDOps: 3, DefVal: 1);
637 }
638 if (MDNode *Node = F.getMetadata(Kind: "vec_type_hint")) {
639 MCInst Inst;
640 Inst.setOpcode(SPIRV::OpExecutionMode);
641 Inst.addOperand(Op: MCOperand::createReg(Reg: FReg));
642 unsigned EM = static_cast<unsigned>(SPIRV::ExecutionMode::VecTypeHint);
643 Inst.addOperand(Op: MCOperand::createImm(Val: EM));
644 unsigned TypeCode = encodeVecTypeHint(Ty: getMDOperandAsType(N: Node, I: 0));
645 Inst.addOperand(Op: MCOperand::createImm(Val: TypeCode));
646 outputMCInst(Inst);
647 }
648 // Per SPV_KHR_poison_freeze description of PoisonFreezeKHR "If declared,
649 // all entry points must use the ArithmeticPoisonKHR execution mode".
650 if (llvm::is_contained(Range: MAI->Reqs.getMinimalCapabilities(),
651 Element: SPIRV::Capability::PoisonFreezeKHR)) {
652 emitSimpleExecutionMode(Reg: FReg, EM: SPIRV::ExecutionMode::ArithmeticPoisonKHR);
653 }
654 // --spirv-fp-contract=off forces to emit ContractionOff for this kernel
655 // entry point, --spirv-fp-contract=fast suppresses it.
656 bool EmitContractionOff =
657 ST->isKernel() && !M.getNamedMetadata(Name: "spirv.ExecutionMode") &&
658 SPIRVFPContract != SPIRVFPContractMode::Fast &&
659 (SPIRVFPContract == SPIRVFPContractMode::Off ||
660 !M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT"));
661 if (EmitContractionOff) {
662 if (ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
663 // When SPV_KHR_float_controls2 is enabled, ContractionOff is
664 // deprecated. We need to use FPFastMathDefault with the appropriate
665 // flags instead. Since FPFastMathDefault takes a target type, we need
666 // to emit it for each floating-point type that exists in the module
667 // to match the effect of ContractionOff. As of now, there are 3 FP
668 // types: fp16, fp32 and fp64.
669
670 // We only end up here because there is no "spirv.ExecutionMode"
671 // metadata, so that means no FPFastMathDefault. Therefore, we only
672 // need to make sure AllowContract is set to 0, as the rest of flags.
673 // We still need to emit the OpExecutionMode instruction, otherwise
674 // it's up to the client API to define the flags. Therefore, we need
675 // to find the constant with 0 value.
676
677 // Collect the SPIRVTypes for fp16, fp32, and fp64 and the constant of
678 // type int32 with 0 value to represent the FP Fast Math Mode.
679 std::vector<const MachineInstr *> SPIRVFloatTypes;
680 const MachineInstr *ConstZeroInt32 = nullptr;
681 for (const MachineInstr *MI :
682 MAI->getMSInstrs(MSType: SPIRV::MB_TypeConstVars)) {
683 unsigned OpCode = MI->getOpcode();
684
685 // Collect the SPIRV type if it's a float.
686 if (OpCode == SPIRV::OpTypeFloat) {
687 // Skip if the target type is not fp16, fp32, fp64.
688 const unsigned OpTypeFloatSize = MI->getOperand(i: 1).getImm();
689 if (OpTypeFloatSize != 16 && OpTypeFloatSize != 32 &&
690 OpTypeFloatSize != 64) {
691 continue;
692 }
693 SPIRVFloatTypes.push_back(x: MI);
694 continue;
695 }
696
697 if (OpCode == SPIRV::OpConstantNull) {
698 // Check if the constant is int32, if not skip it.
699 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
700 MachineInstr *TypeMI = MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg());
701 bool IsInt32Ty = TypeMI &&
702 TypeMI->getOpcode() == SPIRV::OpTypeInt &&
703 TypeMI->getOperand(i: 1).getImm() == 32;
704 if (IsInt32Ty)
705 ConstZeroInt32 = MI;
706 }
707 }
708
709 // When SPV_KHR_float_controls2 is enabled, ContractionOff is
710 // deprecated. We need to use FPFastMathDefault with the appropriate
711 // flags instead. Since FPFastMathDefault takes a target type, we need
712 // to emit it for each floating-point type that exists in the module
713 // to match the effect of ContractionOff. As of now, there are 3 FP
714 // types: fp16, fp32 and fp64.
715 for (const MachineInstr *MI : SPIRVFloatTypes) {
716 MCInst Inst;
717 Inst.setOpcode(SPIRV::OpExecutionModeId);
718 Inst.addOperand(Op: MCOperand::createReg(Reg: FReg));
719 unsigned EM =
720 static_cast<unsigned>(SPIRV::ExecutionMode::FPFastMathDefault);
721 Inst.addOperand(Op: MCOperand::createImm(Val: EM));
722 const MachineFunction *MF = MI->getMF();
723 MCRegister TypeReg =
724 MAI->getRegisterAlias(MF, Reg: MI->getOperand(i: 0).getReg());
725 Inst.addOperand(Op: MCOperand::createReg(Reg: TypeReg));
726 assert(ConstZeroInt32 && "There should be a constant zero.");
727 MCRegister ConstReg = MAI->getRegisterAlias(
728 MF: ConstZeroInt32->getMF(), Reg: ConstZeroInt32->getOperand(i: 0).getReg());
729 Inst.addOperand(Op: MCOperand::createReg(Reg: ConstReg));
730 outputMCInst(Inst);
731 }
732 } else {
733 emitSimpleExecutionMode(Reg: FReg, EM: SPIRV::ExecutionMode::ContractionOff);
734 }
735 }
736 }
737}
738
739void SPIRVAsmPrinter::outputAnnotations(const Module &M) {
740 outputModuleSection(MSType: SPIRV::MB_Annotations);
741 // Process llvm.global.annotations special global variable.
742 if (const GlobalVariable *V = M.getNamedGlobal(Name: "llvm.global.annotations")) {
743 const ConstantArray *CA = cast<ConstantArray>(Val: V->getOperand(i_nocapture: 0));
744 for (Value *Op : CA->operands()) {
745 ConstantStruct *CS = cast<ConstantStruct>(Val: Op);
746 // The first field of the struct contains a pointer to
747 // the annotated variable.
748 Value *AnnotatedVar = CS->getOperand(i_nocapture: 0)->stripPointerCasts();
749 auto *GO = dyn_cast<GlobalObject>(Val: AnnotatedVar);
750 MCRegister Reg = GO ? MAI->getGlobalObjReg(GO) : MCRegister();
751 if (!Reg.isValid()) {
752 std::string DiagMsg;
753 raw_string_ostream OS(DiagMsg);
754 AnnotatedVar->print(O&: OS);
755 DiagMsg = "Unsupported value in llvm.global.annotations: " + DiagMsg;
756 report_fatal_error(reason: DiagMsg.c_str());
757 }
758
759 // The second field contains a pointer to a global annotation string.
760 GlobalVariable *GV =
761 cast<GlobalVariable>(Val: CS->getOperand(i_nocapture: 1)->stripPointerCasts());
762
763 StringRef AnnotationString;
764 [[maybe_unused]] bool Success =
765 getConstantStringInfo(V: GV, Str&: AnnotationString);
766 assert(Success && "Failed to get annotation string");
767 MCInst Inst;
768 Inst.setOpcode(SPIRV::OpDecorate);
769 Inst.addOperand(Op: MCOperand::createReg(Reg));
770 unsigned Dec = static_cast<unsigned>(SPIRV::Decoration::UserSemantic);
771 Inst.addOperand(Op: MCOperand::createImm(Val: Dec));
772 addStringImm(Str: AnnotationString, Inst);
773 outputMCInst(Inst);
774 }
775 }
776}
777
778void SPIRVAsmPrinter::outputFPFastMathDefaultInfo() {
779 // Collect the SPIRVTypes that are OpTypeFloat and the constants of type
780 // int32, that might be used as FP Fast Math Mode.
781 std::vector<const MachineInstr *> SPIRVFloatTypes;
782 // Hashtable to associate immediate values with the constant holding them.
783 DenseMap<int, const MachineInstr *> ConstMap;
784 for (const MachineInstr *MI : MAI->getMSInstrs(MSType: SPIRV::MB_TypeConstVars)) {
785 // Skip if the instruction is not OpTypeFloat or OpConstant.
786 unsigned OpCode = MI->getOpcode();
787 if (OpCode != SPIRV::OpTypeFloat && OpCode != SPIRV::OpConstantI &&
788 OpCode != SPIRV::OpConstantNull)
789 continue;
790
791 // Collect the SPIRV type if it's a float.
792 if (OpCode == SPIRV::OpTypeFloat) {
793 SPIRVFloatTypes.push_back(x: MI);
794 } else {
795 // Check if the constant is int32, if not skip it.
796 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
797 MachineInstr *TypeMI = MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg());
798 if (!TypeMI || TypeMI->getOpcode() != SPIRV::OpTypeInt ||
799 TypeMI->getOperand(i: 1).getImm() != 32)
800 continue;
801
802 if (OpCode == SPIRV::OpConstantI)
803 ConstMap[MI->getOperand(i: 2).getImm()] = MI;
804 else
805 ConstMap[0] = MI;
806 }
807 }
808
809 for (const auto &[Func, FPFastMathDefaultInfoVec] :
810 MAI->FPFastMathDefaultInfoMap) {
811 if (FPFastMathDefaultInfoVec.empty())
812 continue;
813
814 for (const MachineInstr *MI : SPIRVFloatTypes) {
815 unsigned OpTypeFloatSize = MI->getOperand(i: 1).getImm();
816 unsigned Index = SPIRV::FPFastMathDefaultInfoVector::
817 computeFPFastMathDefaultInfoVecIndex(BitWidth: OpTypeFloatSize);
818 assert(Index < FPFastMathDefaultInfoVec.size() &&
819 "Index out of bounds for FPFastMathDefaultInfoVec");
820 const auto &FPFastMathDefaultInfo = FPFastMathDefaultInfoVec[Index];
821 assert(FPFastMathDefaultInfo.Ty &&
822 "Expected target type for FPFastMathDefaultInfo");
823 assert(FPFastMathDefaultInfo.Ty->getScalarSizeInBits() ==
824 OpTypeFloatSize &&
825 "Mismatched float type size");
826 MCInst Inst;
827 Inst.setOpcode(SPIRV::OpExecutionModeId);
828 MCRegister FuncReg = MAI->getGlobalObjReg(GO: Func);
829 assert(FuncReg.isValid());
830 Inst.addOperand(Op: MCOperand::createReg(Reg: FuncReg));
831 Inst.addOperand(
832 Op: MCOperand::createImm(Val: SPIRV::ExecutionMode::FPFastMathDefault));
833 MCRegister TypeReg =
834 MAI->getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
835 Inst.addOperand(Op: MCOperand::createReg(Reg: TypeReg));
836 unsigned Flags = FPFastMathDefaultInfo.FastMathFlags;
837 if (FPFastMathDefaultInfo.ContractionOff &&
838 (Flags & SPIRV::FPFastMathMode::AllowContract))
839 report_fatal_error(
840 reason: "Conflicting FPFastMathFlags: ContractionOff and AllowContract");
841
842 if (FPFastMathDefaultInfo.SignedZeroInfNanPreserve &&
843 !(Flags &
844 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
845 SPIRV::FPFastMathMode::NSZ))) {
846 if (FPFastMathDefaultInfo.FPFastMathDefault)
847 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
848 "SignedZeroInfNanPreserve but at least one of "
849 "NotNaN/NotInf/NSZ is enabled.");
850 }
851
852 // Don't emit if none of the execution modes was used.
853 if (Flags == SPIRV::FPFastMathMode::None &&
854 !FPFastMathDefaultInfo.ContractionOff &&
855 !FPFastMathDefaultInfo.SignedZeroInfNanPreserve &&
856 !FPFastMathDefaultInfo.FPFastMathDefault)
857 continue;
858
859 // Retrieve the constant instruction for the immediate value.
860 auto It = ConstMap.find(Val: Flags);
861 if (It == ConstMap.end())
862 report_fatal_error(reason: "Expected constant instruction for FP Fast Math "
863 "Mode operand of FPFastMathDefault execution mode.");
864 const MachineInstr *ConstMI = It->second;
865 MCRegister ConstReg = MAI->getRegisterAlias(
866 MF: ConstMI->getMF(), Reg: ConstMI->getOperand(i: 0).getReg());
867 Inst.addOperand(Op: MCOperand::createReg(Reg: ConstReg));
868 outputMCInst(Inst);
869 }
870 }
871}
872
873void SPIRVAsmPrinter::outputModuleSections() {
874 const Module *M = MMI->getModule();
875 // Get the global subtarget to output module-level info.
876 ST = static_cast<const SPIRVTargetMachine &>(TM).getSubtargetImpl();
877 TII = ST->getInstrInfo();
878 MAI = &getAnalysis<SPIRVModuleAnalysis>().MAI;
879 assert(ST && TII && MAI && M && "Module analysis is required");
880
881 if (!AuxDataHandler) {
882 auto Handler = std::make_unique<SPIRVAuxDataHandler>(args&: *this, args: *M);
883 if (Handler->hasWork())
884 AuxDataHandler = std::move(Handler);
885 }
886
887 // Let the NSDI handler add its extension and ext inst import entry to MAI
888 // before the module header sections are emitted.
889 if (NSDebugHandler)
890 NSDebugHandler->prepareModuleOutput(ST: *ST, MAI&: *MAI);
891 if (AuxDataHandler)
892 AuxDataHandler->prepareModuleOutput(ST: *ST, MAI&: *MAI);
893
894 // Output instructions according to the Logical Layout of a Module:
895 // 1,2. All OpCapability instructions, then optional OpExtension
896 // instructions.
897 outputGlobalRequirements();
898 // 3. Optional OpExtInstImport instructions.
899 outputOpExtInstImports(M: *M);
900 // 4. The single required OpMemoryModel instruction.
901 outputOpMemoryModel();
902 // 5. All entry point declarations, using OpEntryPoint.
903 outputEntryPoints();
904 // 6. Execution-mode declarations, using OpExecutionMode or
905 // OpExecutionModeId.
906 outputExecutionMode(M: *M);
907 // 7a. Debug: all OpString, OpSourceExtension, OpSource, and
908 // OpSourceContinued, without forward references.
909 outputDebugSourceAndStrings(M: *M);
910 // 7b. Debug: all OpName and all OpMemberName.
911 outputModuleSection(MSType: SPIRV::MB_DebugNames);
912 // 7c. Debug: all OpModuleProcessed instructions.
913 outputModuleSection(MSType: SPIRV::MB_DebugModuleProcessed);
914 // xxx. SPV_INTEL_memory_access_aliasing instructions go before 8.
915 // "All annotation instructions"
916 outputModuleSection(MSType: SPIRV::MB_AliasingInsts);
917 // 8. All annotation instructions (all decorations).
918 outputAnnotations(M: *M);
919 // 9. All type declarations (OpTypeXXX instructions), all constant
920 // instructions, and all global variable declarations. This section is
921 // the first section to allow use of: OpLine and OpNoLine debug information;
922 // non-semantic instructions with OpExtInst.
923 outputModuleSection(MSType: SPIRV::MB_TypeConstVars);
924 // 10. All global NonSemantic.Shader.DebugInfo.100 instructions. The
925 // SPIRVNonSemanticDebugHandler emits these directly as MCInsts; the
926 // MB_NonSemanticGlobalDI section in MAI is intentionally left empty.
927 if (NSDebugHandler)
928 NSDebugHandler->emitNonSemanticGlobalDebugInfo(MAI&: *MAI);
929 if (AuxDataHandler)
930 AuxDataHandler->emitAuxData(MAI&: *MAI);
931 // 11. All function declarations (functions without a body).
932 outputExtFuncDecls();
933 // 12. All function definitions (functions with a body).
934 // This is done in regular function output.
935}
936
937bool SPIRVAsmPrinter::doInitialization(Module &M) {
938 ModuleSectionsEmitted = false;
939 if (!M.getModuleInlineAsm().empty()) {
940 M.getContext().emitError(
941 ErrorStr: "SPIR-V does not support module-level inline assembly");
942 M.removeModuleInlineAsm();
943 }
944
945 // Register the NSDI handler before calling the base class so that
946 // AsmPrinter::doInitialization() calls Handler->beginModule(M) for it.
947 if (M.getNamedMetadata(Name: "llvm.dbg.cu")) {
948 auto Handler = std::make_unique<SPIRVNonSemanticDebugHandler>(args&: *this);
949 NSDebugHandler = Handler.get();
950 addAsmPrinterHandler(Handler: std::move(Handler));
951 }
952 // We need to call the parent's one explicitly.
953 return AsmPrinter::doInitialization(M);
954}
955
956char SPIRVAsmPrinter::ID = 0;
957
958INITIALIZE_PASS(SPIRVAsmPrinter, "spirv-asm-printer", "SPIRV Assembly Printer",
959 false, false)
960
961// Force static initialization.
962extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
963LLVMInitializeSPIRVAsmPrinter() {
964 RegisterAsmPrinter<SPIRVAsmPrinter> X(getTheSPIRV32Target());
965 RegisterAsmPrinter<SPIRVAsmPrinter> Y(getTheSPIRV64Target());
966 RegisterAsmPrinter<SPIRVAsmPrinter> Z(getTheSPIRVLogicalTarget());
967}
968