1//===-- SPIRVAuxDataHandler.cpp - NonSemantic.AuxData emitter -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SPIRVAuxDataHandler.h"
10#include "MCTargetDesc/SPIRVMCTargetDesc.h"
11#include "SPIRVSubtarget.h"
12#include "SPIRVUtils.h"
13#include "llvm/CodeGen/AsmPrinter.h"
14#include "llvm/IR/Attributes.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/GlobalObject.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/LLVMContext.h"
20#include "llvm/IR/Metadata.h"
21#include "llvm/IR/Module.h"
22#include "llvm/MC/MCInst.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/MathExtras.h"
27
28using namespace llvm;
29
30static cl::opt<bool> SPVPreserveAuxData(
31 "spirv-preserve-auxdata",
32 cl::desc("Preserve LLVM attributes and metadata as "
33 "NonSemantic.AuxData ExtInst annotations (requires "
34 "SPV_KHR_non_semantic_info)"),
35 cl::Hidden, cl::init(Val: false));
36
37namespace {
38enum AuxDataLinkageType : uint32_t {
39 AvailableExternally = 0,
40};
41
42constexpr unsigned NonSemanticAuxDataSet =
43 static_cast<unsigned>(SPIRV::InstructionSet::NonSemantic_AuxData);
44
45AttributeSet getGOAttrs(const GlobalObject *GO) {
46 if (const auto *F = dyn_cast<Function>(Val: GO))
47 return F->getAttributes().getFnAttrs();
48 return cast<GlobalVariable>(Val: GO)->getAttributes();
49}
50} // namespace
51
52static bool wasAvailableExternally(const GlobalObject *GO) {
53 if (const auto *F = dyn_cast<Function>(Val: GO))
54 return F->hasFnAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
55 return cast<GlobalVariable>(Val: GO)->getAttributes().hasAttribute(
56 SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
57}
58
59// Return true if any function definition may get an AuxData record.
60static bool hasFunctionAuxData(const Module &M) {
61 return any_of(Range: M.functions(), P: [](const Function &F) {
62 return !F.isDeclaration() &&
63 (F.getAttributes().hasFnAttrs() || F.hasMetadata());
64 });
65}
66
67static bool canUseForwardRefs(const SPIRV::ModuleAnalysisInfo &MAI) {
68 return MAI.Reqs.getExtensions().contains(
69 V: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction);
70}
71
72SPIRVAuxDataHandler::SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M)
73 : Asm(AP), Mod(M) {
74 assert(spirvPreserveAuxData(M.getTargetTriple()) &&
75 "AuxData handler created with AuxData preservation disabled");
76 for (const GlobalObject &GO : M.global_objects())
77 if (wasAvailableExternally(GO: &GO))
78 LinkagePreservedGOs.push_back(Elt: &GO);
79}
80
81bool llvm::spirvPreserveAuxData(const Triple &TT) {
82 return SPVPreserveAuxData || TT.getVendor() == Triple::VendorType::AMD;
83}
84
85void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
86 SPIRV::ModuleAnalysisInfo &MAI) {
87 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_non_semantic_info))
88 reportFatalUsageError(reason: "-spirv-preserve-auxdata requires the "
89 "SPV_KHR_non_semantic_info extension to be enabled.");
90 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
91 if (!MAI.ExtInstSetMap.contains(Val: NonSemanticAuxDataSet))
92 MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
93
94 // Drop NonSemantic forward references if
95 // SPV_KHR_relaxed_extended_instruction not enabled.
96 if (MAI.InstrAuxDataRecords.empty() && !hasFunctionAuxData(M: Mod))
97 return;
98 if (ST.canUseExtension(
99 E: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction))
100 MAI.Reqs.addExtension(
101 ToAdd: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction);
102 else
103 MAI.InstrAuxDataRecords.clear();
104}
105
106MCRegister
107SPIRVAuxDataHandler::getOrEmitString(StringRef S,
108 SPIRV::ModuleAnalysisInfo &MAI) {
109 auto [It, Inserted] = StringRegs.try_emplace(Key: S);
110 if (!Inserted)
111 return It->second;
112 MCRegister Reg = MAI.getNextIDRegister();
113 It->second = Reg;
114 MCInst Inst;
115 Inst.setOpcode(SPIRV::OpString);
116 Inst.addOperand(Op: MCOperand::createReg(Reg));
117 addStringImm(Str: S, Inst);
118 emitMCInst(Inst);
119 return Reg;
120}
121
122void SPIRVAuxDataHandler::collectAttributesFor(const GlobalObject *GO,
123 SPIRV::ModuleAnalysisInfo &MAI) {
124 AuxDataOpcode Opcode = isa<Function>(Val: GO) ? FunctionAttributeOpcode
125 : GlobalVariableAttributeOpcode;
126 for (const Attribute &A : getGOAttrs(GO)) {
127 if (A.isStringAttribute() &&
128 A.getKindAsString() == SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR)
129 continue;
130 ExtInstRecord Rec;
131 Rec.Opcode = Opcode;
132 Rec.Target = GO;
133 if (A.isStringAttribute()) {
134 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: A.getKindAsString(), MAI)});
135 StringRef Val = A.getValueAsString();
136 if (!Val.empty())
137 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: Val, MAI)});
138 } else {
139 Rec.Operands.push_back(
140 Elt: {.Reg: getOrEmitString(S: StringPool.save(S: A.getAsString()), MAI)});
141 }
142 PendingRecords.push_back(Elt: std::move(Rec));
143 }
144}
145
146void SPIRVAuxDataHandler::collectMetadataFor(const GlobalObject *GO,
147 ArrayRef<StringRef> MDNames,
148 SPIRV::ModuleAnalysisInfo &MAI) {
149 SmallVector<std::pair<unsigned, MDNode *>> AllMD;
150 GO->getAllMetadata(MDs&: AllMD);
151 if (AllMD.empty())
152 return;
153 AuxDataOpcode Opcode =
154 isa<Function>(Val: GO) ? FunctionMetadataOpcode : GlobalVariableMetadataOpcode;
155 // MDString operands become OpStrings; ValueAsMetadata constants (e.g.
156 // !{i32 5}) become OpConstants emitted at section 10. Any other operand
157 // kind would need full value translation, so skip the whole node.
158 auto CollectOperands =
159 [&](MDNode *MD) -> std::optional<SmallVector<Operand, 4>> {
160 SmallVector<Operand, 4> Out;
161 for (const MDOperand &MdOp : MD->operands()) {
162 Metadata *Md = MdOp.get();
163 if (auto *MDStr = dyn_cast_or_null<MDString>(Val: Md)) {
164 Out.push_back(Elt: {.Reg: getOrEmitString(S: MDStr->getString(), MAI)});
165 } else if (auto *VAM = dyn_cast_or_null<ValueAsMetadata>(Val: Md)) {
166 auto *C = dyn_cast<Constant>(Val: VAM->getValue());
167 if (!C || !(isa<ConstantInt>(Val: C) || isa<ConstantFP>(Val: C)))
168 return std::nullopt;
169 Out.push_back(Elt: {.Reg: MCRegister(), .Const: C});
170 } else {
171 return std::nullopt;
172 }
173 }
174 return Out;
175 };
176 for (const auto &MD : AllMD) {
177 if (MD.first == LLVMContext::MD_dbg)
178 continue;
179 StringRef MDName = MDNames[MD.first];
180 if (MDName == "spirv.Decorations" || MDName == "spirv.ParameterDecorations")
181 continue;
182 auto Operands = CollectOperands(MD.second);
183 if (!Operands)
184 continue;
185 ExtInstRecord Rec;
186 Rec.Opcode = Opcode;
187 Rec.Target = GO;
188 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: MDName, MAI)});
189 Rec.Operands.append(in_start: Operands->begin(), in_end: Operands->end());
190 PendingRecords.push_back(Elt: std::move(Rec));
191 }
192}
193
194void SPIRVAuxDataHandler::emitAuxDataStrings(SPIRV::ModuleAnalysisInfo &MAI) {
195 if (!MAI.getExtInstSetReg(SetNum: NonSemanticAuxDataSet).isValid())
196 return;
197 // One OpString per metadata kind; stop scanning once every kind is seen.
198 constexpr unsigned AllMDKindsSeen =
199 (1u << (static_cast<unsigned>(
200 SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind::Last) +
201 1)) -
202 1;
203 unsigned SeenMask = 0;
204 for (const auto &Rec : MAI.InstrAuxDataRecords) {
205 unsigned Bit = 1u << static_cast<unsigned>(Rec.Kind);
206 if (SeenMask & Bit)
207 continue;
208 SeenMask |= Bit;
209 getOrEmitString(S: MAI.getAMDGPUAtomicMDName(Kind: Rec.Kind), MAI);
210 if (SeenMask == AllMDKindsSeen)
211 break;
212 }
213 SmallVector<StringRef> MDNames;
214 Mod.getContext().getMDKindNames(Result&: MDNames);
215 bool CanUseForwardRefs = canUseForwardRefs(MAI);
216 for (const GlobalObject &GO : Mod.global_objects()) {
217 if (GO.isDeclaration() || (isa<Function>(Val: GO) && !CanUseForwardRefs))
218 continue;
219 collectAttributesFor(GO: &GO, MAI);
220 collectMetadataFor(GO: &GO, MDNames, MAI);
221 }
222}
223
224void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
225 MCRegister ExtSetReg = MAI.getExtInstSetReg(SetNum: NonSemanticAuxDataSet);
226 if (!ExtSetReg.isValid())
227 return;
228
229 MCRegister VoidTypeReg = findOrEmitOpTypeVoid(MAI);
230
231 for (const ExtInstRecord &Rec : PendingRecords) {
232 MCRegister TargetReg = MAI.getGlobalObjReg(GO: Rec.Target);
233 if (!TargetReg.isValid())
234 continue;
235 SmallVector<MCRegister, 5> Operands;
236 Operands.push_back(Elt: TargetReg);
237 for (const Operand &Op : Rec.Operands)
238 Operands.push_back(Elt: Op.Const ? emitConstant(C: Op.Const, MAI) : Op.Reg);
239 emitAuxDataExtInst(Opcode: Rec.Opcode, VoidTypeReg, ExtSetReg, Operands, MAI,
240 /*UseForwardRefs=*/isa<Function>(Val: Rec.Target));
241 }
242
243 for (const auto &Rec : MAI.InstrAuxDataRecords) {
244 MCRegister TargetReg = MAI.getRegisterAlias(MF: Rec.MF, Reg: Rec.TargetReg);
245 if (!TargetReg.isValid())
246 continue;
247 MCRegister MDNameReg =
248 getOrEmitString(S: MAI.getAMDGPUAtomicMDName(Kind: Rec.Kind), MAI);
249 // TargetReg is defined in a function body, emitted after this section, so
250 // it is always a forward reference.
251 emitAuxDataExtInst(Opcode: InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
252 Operands: {TargetReg, MDNameReg}, MAI, /*UseForwardRefs=*/true);
253 }
254
255 if (LinkagePreservedGOs.empty())
256 return;
257
258 MCRegister UInt32TypeReg = findOrEmitOpTypeUInt32(MAI);
259 MCRegister AEConstReg;
260 bool CanUseForwardRefs = canUseForwardRefs(MAI);
261 for (const GlobalObject *GO : LinkagePreservedGOs) {
262 if (isa<Function>(Val: GO) && !CanUseForwardRefs)
263 continue;
264 MCRegister TargetReg = MAI.getGlobalObjReg(GO);
265 if (!TargetReg.isValid())
266 continue;
267 if (!AEConstReg.isValid())
268 AEConstReg =
269 emitOpConstantUInt32(Value: AvailableExternally, UInt32TypeReg, MAI);
270 emitAuxDataExtInst(Opcode: LinkageOpcode, VoidTypeReg, ExtSetReg,
271 Operands: {TargetReg, AEConstReg}, MAI,
272 /*UseForwardRefs=*/isa<Function>(Val: GO));
273 }
274}
275
276void SPIRVAuxDataHandler::emitAuxDataExtInst(AuxDataOpcode Opcode,
277 MCRegister VoidTypeReg,
278 MCRegister ExtSetReg,
279 ArrayRef<MCRegister> Operands,
280 SPIRV::ModuleAnalysisInfo &MAI,
281 bool UseForwardRefs) {
282 MCInst Inst;
283 Inst.setOpcode(UseForwardRefs ? SPIRV::OpExtInstWithForwardRefsKHR
284 : SPIRV::OpExtInst);
285 Inst.addOperand(Op: MCOperand::createReg(Reg: MAI.getNextIDRegister()));
286 Inst.addOperand(Op: MCOperand::createReg(Reg: VoidTypeReg));
287 Inst.addOperand(Op: MCOperand::createReg(Reg: ExtSetReg));
288 Inst.addOperand(Op: MCOperand::createImm(Val: Opcode));
289 for (MCRegister R : Operands)
290 Inst.addOperand(Op: MCOperand::createReg(Reg: R));
291 emitMCInst(Inst);
292}
293
294void SPIRVAuxDataHandler::emitMCInst(MCInst &Inst) {
295 Asm.OutStreamer->emitInstruction(Inst, STI: Asm.getSubtargetInfo());
296}
297
298MCRegister
299SPIRVAuxDataHandler::findOrEmitOpTypeVoid(SPIRV::ModuleAnalysisInfo &MAI) {
300 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
301 if (MI->getOpcode() == SPIRV::OpTypeVoid)
302 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
303 MCRegister Reg = MAI.getNextIDRegister();
304 MCInst Inst;
305 Inst.setOpcode(SPIRV::OpTypeVoid);
306 Inst.addOperand(Op: MCOperand::createReg(Reg));
307 emitMCInst(Inst);
308 return Reg;
309}
310
311MCRegister
312SPIRVAuxDataHandler::findOrEmitOpTypeInt(unsigned BitWidth,
313 SPIRV::ModuleAnalysisInfo &MAI) {
314 // SPIR-V OpTypeInt: <width>, <signedness>. Signedness 0 = unsigned, 1 =
315 // signed; we always emit unsigned.
316 constexpr int64_t UnsignedSignedness = 0;
317 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
318 if (MI->getOpcode() == SPIRV::OpTypeInt &&
319 MI->getOperand(i: 1).getImm() == static_cast<int64_t>(BitWidth) &&
320 MI->getOperand(i: 2).getImm() == UnsignedSignedness)
321 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
322 MCRegister Reg = MAI.getNextIDRegister();
323 MCInst Inst;
324 Inst.setOpcode(SPIRV::OpTypeInt);
325 Inst.addOperand(Op: MCOperand::createReg(Reg));
326 Inst.addOperand(Op: MCOperand::createImm(Val: BitWidth));
327 Inst.addOperand(Op: MCOperand::createImm(Val: UnsignedSignedness));
328 emitMCInst(Inst);
329 return Reg;
330}
331
332MCRegister
333SPIRVAuxDataHandler::findOrEmitOpTypeUInt32(SPIRV::ModuleAnalysisInfo &MAI) {
334 return findOrEmitOpTypeInt(BitWidth: 32, MAI);
335}
336
337MCRegister
338SPIRVAuxDataHandler::findOrEmitOpTypeFloat(unsigned BitWidth,
339 SPIRV::ModuleAnalysisInfo &MAI) {
340 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
341 if (MI->getOpcode() == SPIRV::OpTypeFloat &&
342 MI->getOperand(i: 1).getImm() == static_cast<int64_t>(BitWidth))
343 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
344 MCRegister Reg = MAI.getNextIDRegister();
345 MCInst Inst;
346 Inst.setOpcode(SPIRV::OpTypeFloat);
347 Inst.addOperand(Op: MCOperand::createReg(Reg));
348 Inst.addOperand(Op: MCOperand::createImm(Val: BitWidth));
349 emitMCInst(Inst);
350 return Reg;
351}
352
353MCRegister SPIRVAuxDataHandler::emitConstant(const Constant *C,
354 SPIRV::ModuleAnalysisInfo &MAI) {
355 auto [It, Inserted] = ConstantRegs.try_emplace(Key: C);
356 if (!Inserted)
357 return It->second;
358
359 APInt Bits;
360 unsigned Opcode;
361 MCRegister TypeReg;
362 if (const auto *CI = dyn_cast<ConstantInt>(Val: C)) {
363 Bits = CI->getValue();
364 Opcode = SPIRV::OpConstantI;
365 TypeReg = findOrEmitOpTypeInt(BitWidth: Bits.getBitWidth(), MAI);
366 } else {
367 const auto *CF = cast<ConstantFP>(Val: C);
368 Bits = CF->getValueAPF().bitcastToAPInt();
369 Opcode = SPIRV::OpConstantF;
370 TypeReg = findOrEmitOpTypeFloat(BitWidth: Bits.getBitWidth(), MAI);
371 }
372
373 MCRegister Reg = MAI.getNextIDRegister();
374 It->second = Reg;
375 MCInst Inst;
376 Inst.setOpcode(Opcode);
377 Inst.addOperand(Op: MCOperand::createReg(Reg));
378 Inst.addOperand(Op: MCOperand::createReg(Reg: TypeReg));
379 // SPIR-V encodes the literal as ceil(width/32) little-endian 32-bit words.
380 unsigned NumWords = divideCeil(Numerator: Bits.getBitWidth(), Denominator: 32);
381 for (unsigned I = 0; I < NumWords; ++I)
382 Inst.addOperand(Op: MCOperand::createImm(Val: Bits.extractBitsAsZExtValue(
383 numBits: std::min(a: 32u, b: Bits.getBitWidth() - I * 32), bitPosition: I * 32)));
384 // The asm printer needs this hint to render an f16 literal correctly.
385 if (Opcode == SPIRV::OpConstantF && Bits.getBitWidth() == 16)
386 Inst.setFlags(SPIRV::INST_PRINTER_WIDTH16);
387 emitMCInst(Inst);
388 return Reg;
389}
390
391MCRegister SPIRVAuxDataHandler::emitOpConstantUInt32(
392 uint32_t Value, MCRegister UInt32TypeReg, SPIRV::ModuleAnalysisInfo &MAI) {
393 MCRegister Reg = MAI.getNextIDRegister();
394 MCInst Inst;
395 Inst.setOpcode(SPIRV::OpConstantI);
396 Inst.addOperand(Op: MCOperand::createReg(Reg));
397 Inst.addOperand(Op: MCOperand::createReg(Reg: UInt32TypeReg));
398 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<int64_t>(Value)));
399 emitMCInst(Inst);
400 return Reg;
401}
402