1//===-- SPIRVAuxDataHandler.cpp - NonSemantic.AuxData emitter -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SPIRVAuxDataHandler.h"
10#include "MCTargetDesc/SPIRVMCTargetDesc.h"
11#include "SPIRVSubtarget.h"
12#include "SPIRVUtils.h"
13#include "llvm/CodeGen/AsmPrinter.h"
14#include "llvm/IR/Attributes.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/GlobalObject.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/LLVMContext.h"
20#include "llvm/IR/Metadata.h"
21#include "llvm/IR/Module.h"
22#include "llvm/MC/MCInst.h"
23#include "llvm/MC/MCStreamer.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/MathExtras.h"
26
27using namespace llvm;
28
29namespace {
30enum AuxDataLinkageType : uint32_t {
31 AvailableExternally = 0,
32};
33
34constexpr unsigned NonSemanticAuxDataSet =
35 static_cast<unsigned>(SPIRV::InstructionSet::NonSemantic_AuxData);
36
37AttributeSet getGOAttrs(const GlobalObject *GO) {
38 if (const auto *F = dyn_cast<Function>(Val: GO))
39 return F->getAttributes().getFnAttrs();
40 return cast<GlobalVariable>(Val: GO)->getAttributes();
41}
42} // namespace
43
44static bool wasAvailableExternally(const GlobalObject *GO) {
45 if (const auto *F = dyn_cast<Function>(Val: GO))
46 return F->hasFnAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
47 return cast<GlobalVariable>(Val: GO)->getAttributes().hasAttribute(
48 SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
49}
50
51// Return true if any function definition may get an AuxData record.
52static bool hasFunctionAuxData(const Module &M) {
53 return any_of(Range: M.functions(), P: [](const Function &F) {
54 return !F.isDeclaration() &&
55 (F.getAttributes().hasFnAttrs() || F.hasMetadata());
56 });
57}
58
59static bool canUseForwardRefs(const SPIRV::ModuleAnalysisInfo &MAI) {
60 return MAI.Reqs.getExtensions().contains(
61 V: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction);
62}
63
64SPIRVAuxDataHandler::SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M)
65 : Asm(AP), Mod(M) {
66 for (const GlobalObject &GO : M.global_objects())
67 if (wasAvailableExternally(GO: &GO))
68 LinkagePreservedGOs.push_back(Elt: &GO);
69}
70
71bool llvm::spirvPreserveAuxData(const SPIRVOptions &CLOpts, const Triple &TT) {
72 return CLOpts.preserve_auxdata || TT.getVendor() == Triple::VendorType::AMD;
73}
74
75void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
76 SPIRV::ModuleAnalysisInfo &MAI) {
77 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_non_semantic_info))
78 reportFatalUsageError(reason: "-spirv-preserve-auxdata requires the "
79 "SPV_KHR_non_semantic_info extension to be enabled.");
80 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
81 if (!MAI.ExtInstSetMap.contains(Val: NonSemanticAuxDataSet))
82 MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
83
84 // Drop NonSemantic forward references if
85 // SPV_KHR_relaxed_extended_instruction not enabled.
86 if (MAI.InstrAuxDataRecords.empty() && !hasFunctionAuxData(M: Mod))
87 return;
88 if (ST.canUseExtension(
89 E: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction))
90 MAI.Reqs.addExtension(
91 ToAdd: SPIRV::Extension::SPV_KHR_relaxed_extended_instruction);
92 else
93 MAI.InstrAuxDataRecords.clear();
94}
95
96MCRegister
97SPIRVAuxDataHandler::getOrEmitString(StringRef S,
98 SPIRV::ModuleAnalysisInfo &MAI) {
99 auto [It, Inserted] = StringRegs.try_emplace(Key: S);
100 if (!Inserted)
101 return It->second;
102 MCRegister Reg = MAI.getNextIDRegister();
103 It->second = Reg;
104 MCInst Inst;
105 Inst.setOpcode(SPIRV::OpString);
106 Inst.addOperand(Op: MCOperand::createReg(Reg));
107 addStringImm(Str: S, Inst);
108 emitMCInst(Inst);
109 return Reg;
110}
111
112void SPIRVAuxDataHandler::collectAttributesFor(const GlobalObject *GO,
113 SPIRV::ModuleAnalysisInfo &MAI) {
114 AuxDataOpcode Opcode = isa<Function>(Val: GO) ? FunctionAttributeOpcode
115 : GlobalVariableAttributeOpcode;
116 for (const Attribute &A : getGOAttrs(GO)) {
117 if (A.isStringAttribute() &&
118 A.getKindAsString() == SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR)
119 continue;
120 ExtInstRecord Rec;
121 Rec.Opcode = Opcode;
122 Rec.Target = GO;
123 if (A.isStringAttribute()) {
124 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: A.getKindAsString(), MAI)});
125 StringRef Val = A.getValueAsString();
126 if (!Val.empty())
127 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: Val, MAI)});
128 } else {
129 Rec.Operands.push_back(
130 Elt: {.Reg: getOrEmitString(S: StringPool.save(S: A.getAsString()), MAI)});
131 }
132 PendingRecords.push_back(Elt: std::move(Rec));
133 }
134}
135
136void SPIRVAuxDataHandler::collectMetadataFor(const GlobalObject *GO,
137 ArrayRef<StringRef> MDNames,
138 SPIRV::ModuleAnalysisInfo &MAI) {
139 SmallVector<std::pair<unsigned, MDNode *>> AllMD;
140 GO->getAllMetadata(MDs&: AllMD);
141 if (AllMD.empty())
142 return;
143 AuxDataOpcode Opcode =
144 isa<Function>(Val: GO) ? FunctionMetadataOpcode : GlobalVariableMetadataOpcode;
145 // MDString operands become OpStrings; ValueAsMetadata constants (e.g.
146 // !{i32 5}) become OpConstants emitted at section 10. Any other operand
147 // kind would need full value translation, so skip the whole node.
148 auto CollectOperands =
149 [&](MDNode *MD) -> std::optional<SmallVector<Operand, 4>> {
150 SmallVector<Operand, 4> Out;
151 for (const MDOperand &MdOp : MD->operands()) {
152 Metadata *Md = MdOp.get();
153 if (auto *MDStr = dyn_cast_or_null<MDString>(Val: Md)) {
154 Out.push_back(Elt: {.Reg: getOrEmitString(S: MDStr->getString(), MAI)});
155 } else if (auto *VAM = dyn_cast_or_null<ValueAsMetadata>(Val: Md)) {
156 auto *C = dyn_cast<Constant>(Val: VAM->getValue());
157 if (!C || !(isa<ConstantInt>(Val: C) || isa<ConstantFP>(Val: C)))
158 return std::nullopt;
159 Out.push_back(Elt: {.Reg: MCRegister(), .Const: C});
160 } else {
161 return std::nullopt;
162 }
163 }
164 return Out;
165 };
166 for (const auto &MD : AllMD) {
167 if (MD.first == LLVMContext::MD_dbg)
168 continue;
169 StringRef MDName = MDNames[MD.first];
170 if (MDName == "spirv.Decorations" || MDName == "spirv.ParameterDecorations")
171 continue;
172 auto Operands = CollectOperands(MD.second);
173 if (!Operands)
174 continue;
175 ExtInstRecord Rec;
176 Rec.Opcode = Opcode;
177 Rec.Target = GO;
178 Rec.Operands.push_back(Elt: {.Reg: getOrEmitString(S: MDName, MAI)});
179 Rec.Operands.append(in_start: Operands->begin(), in_end: Operands->end());
180 PendingRecords.push_back(Elt: std::move(Rec));
181 }
182}
183
184void SPIRVAuxDataHandler::emitAuxDataStrings(SPIRV::ModuleAnalysisInfo &MAI) {
185 if (!MAI.getExtInstSetReg(SetNum: NonSemanticAuxDataSet).isValid())
186 return;
187 // One OpString per metadata kind; stop scanning once every kind is seen.
188 constexpr unsigned AllMDKindsSeen =
189 (1u << (static_cast<unsigned>(
190 SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind::Last) +
191 1)) -
192 1;
193 unsigned SeenMask = 0;
194 for (const auto &Rec : MAI.InstrAuxDataRecords) {
195 unsigned Bit = 1u << static_cast<unsigned>(Rec.Kind);
196 if (SeenMask & Bit)
197 continue;
198 SeenMask |= Bit;
199 getOrEmitString(S: MAI.getAMDGPUAtomicMDName(Kind: Rec.Kind), MAI);
200 if (SeenMask == AllMDKindsSeen)
201 break;
202 }
203 SmallVector<StringRef> MDNames;
204 Mod.getContext().getMDKindNames(Result&: MDNames);
205 bool CanUseForwardRefs = canUseForwardRefs(MAI);
206 for (const GlobalObject &GO : Mod.global_objects()) {
207 if (GO.isDeclaration() || (isa<Function>(Val: GO) && !CanUseForwardRefs))
208 continue;
209 collectAttributesFor(GO: &GO, MAI);
210 collectMetadataFor(GO: &GO, MDNames, MAI);
211 }
212}
213
214void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
215 MCRegister ExtSetReg = MAI.getExtInstSetReg(SetNum: NonSemanticAuxDataSet);
216 if (!ExtSetReg.isValid())
217 return;
218
219 MCRegister VoidTypeReg = findOrEmitOpTypeVoid(MAI);
220
221 for (const ExtInstRecord &Rec : PendingRecords) {
222 MCRegister TargetReg = MAI.getGlobalObjReg(GO: Rec.Target);
223 if (!TargetReg.isValid())
224 continue;
225 SmallVector<MCRegister, 5> Operands;
226 Operands.push_back(Elt: TargetReg);
227 for (const Operand &Op : Rec.Operands)
228 Operands.push_back(Elt: Op.Const ? emitConstant(C: Op.Const, MAI) : Op.Reg);
229 emitAuxDataExtInst(Opcode: Rec.Opcode, VoidTypeReg, ExtSetReg, Operands, MAI,
230 /*UseForwardRefs=*/isa<Function>(Val: Rec.Target));
231 }
232
233 for (const auto &Rec : MAI.InstrAuxDataRecords) {
234 MCRegister TargetReg = MAI.getRegisterAlias(MF: Rec.MF, Reg: Rec.TargetReg);
235 if (!TargetReg.isValid())
236 continue;
237 MCRegister MDNameReg =
238 getOrEmitString(S: MAI.getAMDGPUAtomicMDName(Kind: Rec.Kind), MAI);
239 // TargetReg is defined in a function body, emitted after this section, so
240 // it is always a forward reference.
241 emitAuxDataExtInst(Opcode: InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
242 Operands: {TargetReg, MDNameReg}, MAI, /*UseForwardRefs=*/true);
243 }
244
245 if (LinkagePreservedGOs.empty())
246 return;
247
248 MCRegister UInt32TypeReg = findOrEmitOpTypeUInt32(MAI);
249 MCRegister AEConstReg;
250 bool CanUseForwardRefs = canUseForwardRefs(MAI);
251 for (const GlobalObject *GO : LinkagePreservedGOs) {
252 if (isa<Function>(Val: GO) && !CanUseForwardRefs)
253 continue;
254 MCRegister TargetReg = MAI.getGlobalObjReg(GO);
255 if (!TargetReg.isValid())
256 continue;
257 if (!AEConstReg.isValid())
258 AEConstReg =
259 emitOpConstantUInt32(Value: AvailableExternally, UInt32TypeReg, MAI);
260 emitAuxDataExtInst(Opcode: LinkageOpcode, VoidTypeReg, ExtSetReg,
261 Operands: {TargetReg, AEConstReg}, MAI,
262 /*UseForwardRefs=*/isa<Function>(Val: GO));
263 }
264}
265
266void SPIRVAuxDataHandler::emitAuxDataExtInst(AuxDataOpcode Opcode,
267 MCRegister VoidTypeReg,
268 MCRegister ExtSetReg,
269 ArrayRef<MCRegister> Operands,
270 SPIRV::ModuleAnalysisInfo &MAI,
271 bool UseForwardRefs) {
272 MCInst Inst;
273 Inst.setOpcode(UseForwardRefs ? SPIRV::OpExtInstWithForwardRefsKHR
274 : SPIRV::OpExtInst);
275 Inst.addOperand(Op: MCOperand::createReg(Reg: MAI.getNextIDRegister()));
276 Inst.addOperand(Op: MCOperand::createReg(Reg: VoidTypeReg));
277 Inst.addOperand(Op: MCOperand::createReg(Reg: ExtSetReg));
278 Inst.addOperand(Op: MCOperand::createImm(Val: Opcode));
279 for (MCRegister R : Operands)
280 Inst.addOperand(Op: MCOperand::createReg(Reg: R));
281 emitMCInst(Inst);
282}
283
284void SPIRVAuxDataHandler::emitMCInst(MCInst &Inst) {
285 Asm.OutStreamer->emitInstruction(Inst, STI: Asm.getSubtargetInfo());
286}
287
288MCRegister
289SPIRVAuxDataHandler::findOrEmitOpTypeVoid(SPIRV::ModuleAnalysisInfo &MAI) {
290 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
291 if (MI->getOpcode() == SPIRV::OpTypeVoid)
292 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
293 MCRegister Reg = MAI.getNextIDRegister();
294 MCInst Inst;
295 Inst.setOpcode(SPIRV::OpTypeVoid);
296 Inst.addOperand(Op: MCOperand::createReg(Reg));
297 emitMCInst(Inst);
298 return Reg;
299}
300
301MCRegister
302SPIRVAuxDataHandler::findOrEmitOpTypeInt(unsigned BitWidth,
303 SPIRV::ModuleAnalysisInfo &MAI) {
304 // SPIR-V OpTypeInt: <width>, <signedness>. Signedness 0 = unsigned, 1 =
305 // signed; we always emit unsigned.
306 constexpr int64_t UnsignedSignedness = 0;
307 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
308 if (MI->getOpcode() == SPIRV::OpTypeInt &&
309 MI->getOperand(i: 1).getImm() == static_cast<int64_t>(BitWidth) &&
310 MI->getOperand(i: 2).getImm() == UnsignedSignedness)
311 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
312 MCRegister Reg = MAI.getNextIDRegister();
313 MCInst Inst;
314 Inst.setOpcode(SPIRV::OpTypeInt);
315 Inst.addOperand(Op: MCOperand::createReg(Reg));
316 Inst.addOperand(Op: MCOperand::createImm(Val: BitWidth));
317 Inst.addOperand(Op: MCOperand::createImm(Val: UnsignedSignedness));
318 emitMCInst(Inst);
319 return Reg;
320}
321
322MCRegister
323SPIRVAuxDataHandler::findOrEmitOpTypeUInt32(SPIRV::ModuleAnalysisInfo &MAI) {
324 return findOrEmitOpTypeInt(BitWidth: 32, MAI);
325}
326
327MCRegister
328SPIRVAuxDataHandler::findOrEmitOpTypeFloat(unsigned BitWidth,
329 SPIRV::ModuleAnalysisInfo &MAI) {
330 for (const MachineInstr *MI : MAI.getMSInstrs(MSType: SPIRV::MB_TypeConstVars))
331 if (MI->getOpcode() == SPIRV::OpTypeFloat &&
332 MI->getOperand(i: 1).getImm() == static_cast<int64_t>(BitWidth))
333 return MAI.getRegisterAlias(MF: MI->getMF(), Reg: MI->getOperand(i: 0).getReg());
334 MCRegister Reg = MAI.getNextIDRegister();
335 MCInst Inst;
336 Inst.setOpcode(SPIRV::OpTypeFloat);
337 Inst.addOperand(Op: MCOperand::createReg(Reg));
338 Inst.addOperand(Op: MCOperand::createImm(Val: BitWidth));
339 emitMCInst(Inst);
340 return Reg;
341}
342
343MCRegister SPIRVAuxDataHandler::emitConstant(const Constant *C,
344 SPIRV::ModuleAnalysisInfo &MAI) {
345 auto [It, Inserted] = ConstantRegs.try_emplace(Key: C);
346 if (!Inserted)
347 return It->second;
348
349 APInt Bits;
350 unsigned Opcode;
351 MCRegister TypeReg;
352 if (const auto *CI = dyn_cast<ConstantInt>(Val: C)) {
353 Bits = CI->getValue();
354 Opcode = SPIRV::OpConstantI;
355 TypeReg = findOrEmitOpTypeInt(BitWidth: Bits.getBitWidth(), MAI);
356 } else {
357 const auto *CF = cast<ConstantFP>(Val: C);
358 Bits = CF->getValueAPF().bitcastToAPInt();
359 Opcode = SPIRV::OpConstantF;
360 TypeReg = findOrEmitOpTypeFloat(BitWidth: Bits.getBitWidth(), MAI);
361 }
362
363 MCRegister Reg = MAI.getNextIDRegister();
364 It->second = Reg;
365 MCInst Inst;
366 Inst.setOpcode(Opcode);
367 Inst.addOperand(Op: MCOperand::createReg(Reg));
368 Inst.addOperand(Op: MCOperand::createReg(Reg: TypeReg));
369 // SPIR-V encodes the literal as ceil(width/32) little-endian 32-bit words.
370 unsigned NumWords = divideCeil(Numerator: Bits.getBitWidth(), Denominator: 32);
371 for (unsigned I = 0; I < NumWords; ++I)
372 Inst.addOperand(Op: MCOperand::createImm(Val: Bits.extractBitsAsZExtValue(
373 numBits: std::min(a: 32u, b: Bits.getBitWidth() - I * 32), bitPosition: I * 32)));
374 // The asm printer needs this hint to render an f16 literal correctly.
375 if (Opcode == SPIRV::OpConstantF && Bits.getBitWidth() == 16)
376 Inst.setFlags(SPIRV::INST_PRINTER_WIDTH16);
377 emitMCInst(Inst);
378 return Reg;
379}
380
381MCRegister SPIRVAuxDataHandler::emitOpConstantUInt32(
382 uint32_t Value, MCRegister UInt32TypeReg, SPIRV::ModuleAnalysisInfo &MAI) {
383 MCRegister Reg = MAI.getNextIDRegister();
384 MCInst Inst;
385 Inst.setOpcode(SPIRV::OpConstantI);
386 Inst.addOperand(Op: MCOperand::createReg(Reg));
387 Inst.addOperand(Op: MCOperand::createReg(Reg: UInt32TypeReg));
388 Inst.addOperand(Op: MCOperand::createImm(Val: static_cast<int64_t>(Value)));
389 emitMCInst(Inst);
390 return Reg;
391}
392