1//===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the lowering of LLVM calls to machine code calls for
10// GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVCallLowering.h"
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "SPIRV.h"
17#include "SPIRVBuiltins.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVISelLowering.h"
20#include "SPIRVMetadata.h"
21#include "SPIRVRegisterInfo.h"
22#include "SPIRVSubtarget.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/CodeGen/FunctionLoweringInfo.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/IntrinsicsSPIRV.h"
28#include "llvm/Support/ModRef.h"
29
30using namespace llvm;
31
32SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI,
33 SPIRVGlobalRegistry *GR)
34 : CallLowering(&TLI), GR(GR) {}
35
36bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
37 const Value *Val, ArrayRef<Register> VRegs,
38 FunctionLoweringInfo &FLI,
39 Register SwiftErrorVReg) const {
40 // Ignore if called from the internal service function
41 if (MIRBuilder.getMF()
42 .getFunction()
43 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
44 .isValid())
45 return true;
46
47 // Currently all return types should use a single register.
48 // TODO: handle the case of multiple registers.
49 if (VRegs.size() > 1)
50 return false;
51
52 if (Val) {
53 const auto &STI = MIRBuilder.getMF().getSubtarget();
54 MIRBuilder.buildInstr(Opcode: SPIRV::OpReturnValue)
55 .addUse(RegNo: VRegs[0])
56 .constrainAllUses(TII: MIRBuilder.getTII(), TRI: *STI.getRegisterInfo(),
57 RBI: *STI.getRegBankInfo());
58 return true;
59 }
60 MIRBuilder.buildInstr(Opcode: SPIRV::OpReturn);
61 return true;
62}
63
64// Based on the LLVM function attributes, get a SPIR-V FunctionControl.
65static uint32_t getFunctionControl(const Function &F,
66 const SPIRVSubtarget *ST) {
67 MemoryEffects MemEffects = F.getMemoryEffects();
68
69 uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None);
70
71 if (F.hasFnAttribute(Kind: Attribute::AttrKind::NoInline))
72 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline);
73 else if (F.hasFnAttribute(Kind: Attribute::AttrKind::AlwaysInline))
74 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline);
75
76 if (MemEffects.doesNotAccessMemory())
77 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure);
78 else if (MemEffects.onlyReadsMemory())
79 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const);
80
81 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone) ||
82 ST->canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone))
83 if (F.hasFnAttribute(Kind: Attribute::OptimizeNone))
84 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::OptNoneEXT);
85
86 return FuncControl;
87}
88
89// If the function has pointer arguments, we are forced to re-create this
90// function type from the very beginning, changing PointerType by
91// TypedPointerType for each pointer argument. Otherwise, the same `Type*`
92// potentially corresponds to different SPIR-V function type, effectively
93// invalidating logic behind global registry and duplicates tracker.
94static FunctionType *
95fixFunctionTypeIfPtrArgs(SPIRVGlobalRegistry *GR, const Function &F,
96 FunctionType *FTy, SPIRVTypeInst SRetTy,
97 const SmallVector<SPIRVTypeInst, 4> &SArgTys) {
98 bool hasArgPtrs = any_of(Range: F.args(), P: [](const Argument &Arg) {
99 // check if it's an instance of a non-typed PointerType
100 return Arg.getType()->isPointerTy();
101 });
102 if (!hasArgPtrs) {
103 Type *RetTy = FTy->getReturnType();
104 // check if it's an instance of a non-typed PointerType
105 if (!RetTy->isPointerTy())
106 return FTy;
107 }
108
109 // re-create function type, using TypedPointerType instead of PointerType to
110 // properly trace argument types
111 const Type *RetTy = GR->getTypeForSPIRVType(Ty: SRetTy);
112 SmallVector<Type *, 4> ArgTys;
113 for (auto SArgTy : SArgTys)
114 ArgTys.push_back(Elt: const_cast<Type *>(GR->getTypeForSPIRVType(Ty: SArgTy)));
115 return FunctionType::get(Result: const_cast<Type *>(RetTy), Params: ArgTys, isVarArg: false);
116}
117
118static SPIRV::AccessQualifier::AccessQualifier
119getArgAccessQual(const Function &F, unsigned ArgIdx) {
120 if (F.getCallingConv() != CallingConv::SPIR_KERNEL)
121 return SPIRV::AccessQualifier::ReadWrite;
122
123 MDString *ArgAttribute = getOCLKernelArgAccessQual(F, ArgIdx);
124 if (!ArgAttribute)
125 return SPIRV::AccessQualifier::ReadWrite;
126
127 if (ArgAttribute->getString() == "read_only")
128 return SPIRV::AccessQualifier::ReadOnly;
129 if (ArgAttribute->getString() == "write_only")
130 return SPIRV::AccessQualifier::WriteOnly;
131 return SPIRV::AccessQualifier::ReadWrite;
132}
133
134static SPIRVTypeInst getArgSPIRVType(const Function &F, unsigned ArgIdx,
135 SPIRVGlobalRegistry *GR,
136 MachineIRBuilder &MIRBuilder,
137 const SPIRVSubtarget &ST) {
138 // Read argument's access qualifier from metadata or default.
139 SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
140 getArgAccessQual(F, ArgIdx);
141
142 Type *OriginalArgType =
143 SPIRV::getOriginalFunctionType(F)->getParamType(i: ArgIdx);
144
145 // Vector of untyped pointers: build with the deduced pointee instead of
146 // the default i8 (mismatches typed uses downstream).
147 Argument *Arg = F.getArg(i: ArgIdx);
148 if (auto *VTy = dyn_cast<FixedVectorType>(Val: OriginalArgType);
149 VTy && isUntypedPointerTy(T: VTy->getElementType()))
150 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg))
151 return GR->getOrCreateSPIRVVectorType(
152 BaseType: GR->getOrCreateSPIRVPointerType(
153 BaseType: ElemTy, MIRBuilder,
154 SC: addressSpaceToStorageClass(
155 AddrSpace: getPointerAddressSpace(T: OriginalArgType), STI: ST)),
156 NumElements: VTy->getNumElements(), MIRBuilder, EmitIR: true);
157
158 // If OriginalArgType is non-pointer, use the OriginalArgType (the type cannot
159 // be legally reassigned later).
160 if (!isPointerTy(T: OriginalArgType))
161 return GR->getOrCreateSPIRVType(Type: OriginalArgType, MIRBuilder, AQ: ArgAccessQual,
162 EmitIR: true);
163
164 Type *ArgType = Arg->getType();
165
166 // In case OriginalArgType is of untyped pointer type, there are three
167 // possibilities:
168 // 1) This is a pointer of an LLVM IR element type, passed byval/byref.
169 // 2) This is an OpenCL/SPIR-V builtin type if there is spv_assign_type
170 // intrinsic assigning a TargetExtType.
171 // 3) This is a pointer, try to retrieve pointer element type from a
172 // spv_assign_ptr_type intrinsic or otherwise use default pointer element
173 // type.
174 if (hasPointeeTypeAttr(Arg)) {
175 // byval/byref/sret carry the aggregate layout in the pointee type, so keep
176 // a typed pointer here. An untyped one drops the type and breaks the
177 // argument ABI on the way back from SPIR-V.
178 return GR->getOrCreateSPIRVTypedPointerType(
179 BaseType: getPointeeTypeByAttr(Arg), MIRBuilder,
180 SC: addressSpaceToStorageClass(AddrSpace: getPointerAddressSpace(T: ArgType), STI: ST));
181 }
182
183 for (auto User : Arg->users()) {
184 auto *II = dyn_cast<IntrinsicInst>(Val: User);
185 // Check if this is spv_assign_type assigning OpenCL/SPIR-V builtin type.
186 if (II && II->getIntrinsicID() == Intrinsic::spv_assign_type) {
187 MetadataAsValue *VMD = cast<MetadataAsValue>(Val: II->getOperand(i_nocapture: 1));
188 Type *BuiltinType =
189 cast<ConstantAsMetadata>(Val: VMD->getMetadata())->getType();
190 assert(BuiltinType->isTargetExtTy() && "Expected TargetExtType");
191 return GR->getOrCreateSPIRVType(Type: BuiltinType, MIRBuilder, AQ: ArgAccessQual,
192 EmitIR: true);
193 }
194
195 // Check if this is spv_assign_ptr_type assigning pointer element type.
196 if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type)
197 continue;
198
199 MetadataAsValue *VMD = cast<MetadataAsValue>(Val: II->getOperand(i_nocapture: 1));
200 Type *ElementTy =
201 toTypedPointer(Ty: cast<ConstantAsMetadata>(Val: VMD->getMetadata())->getType());
202 return GR->getOrCreateSPIRVPointerType(
203 BaseType: ElementTy, MIRBuilder,
204 SC: addressSpaceToStorageClass(
205 AddrSpace: cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getZExtValue(), STI: ST));
206 }
207
208 // Use the recorded pointee type for declarations.
209 if (F.isDeclaration())
210 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg))
211 return GR->getOrCreateSPIRVPointerType(
212 BaseType: ElemTy, MIRBuilder,
213 SC: addressSpaceToStorageClass(AddrSpace: getPointerAddressSpace(T: ArgType), STI: ST));
214
215 // Replace PointerType with TypedPointerType to be able to map SPIR-V types to
216 // LLVM types in a consistent manner
217 return GR->getOrCreateSPIRVType(Type: toTypedPointer(Ty: OriginalArgType), MIRBuilder,
218 AQ: ArgAccessQual, EmitIR: true);
219}
220
221static SPIRV::ExecutionModel::ExecutionModel
222getExecutionModel(const SPIRVSubtarget &STI, const Function &F) {
223 assert(STI.getEnv() != SPIRVSubtarget::Unknown &&
224 "Environment must be resolved before lowering entry points.");
225
226 if (STI.isKernel())
227 return SPIRV::ExecutionModel::Kernel;
228
229 auto attribute = F.getFnAttribute(Kind: "hlsl.shader");
230 if (!attribute.isValid()) {
231 report_fatal_error(
232 reason: "This entry point lacks mandatory hlsl.shader attribute.");
233 }
234
235 const auto value = attribute.getValueAsString();
236 if (value == "compute")
237 return SPIRV::ExecutionModel::GLCompute;
238 if (value == "vertex")
239 return SPIRV::ExecutionModel::Vertex;
240 if (value == "pixel")
241 return SPIRV::ExecutionModel::Fragment;
242
243 report_fatal_error(reason: "This HLSL entry point is not supported by this backend.");
244}
245
246bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
247 const Function &F,
248 ArrayRef<ArrayRef<Register>> VRegs,
249 FunctionLoweringInfo &FLI) const {
250 // Discard the internal service function
251 if (F.getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME).isValid())
252 return true;
253
254 assert(GR && "Must initialize the SPIRV type registry before lowering args.");
255 GR->setCurrentFunc(MIRBuilder.getMF());
256
257 // Get access to information about available extensions
258 const SPIRVSubtarget *ST =
259 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
260
261 // Assign types and names to all args, and store their types for later.
262 SmallVector<SPIRVTypeInst, 4> ArgTypeVRegs;
263 if (VRegs.size() > 0) {
264 unsigned i = 0;
265 for (const auto &Arg : F.args()) {
266 // Currently formal args should use single registers.
267 // TODO: handle the case of multiple registers.
268 if (VRegs[i].size() > 1)
269 return false;
270 SPIRVTypeInst SpirvTy = getArgSPIRVType(F, ArgIdx: i, GR, MIRBuilder, ST: *ST);
271 GR->assignSPIRVTypeToVReg(Type: SpirvTy, VReg: VRegs[i][0], MF: MIRBuilder.getMF());
272 ArgTypeVRegs.push_back(Elt: SpirvTy);
273
274 if (Arg.hasName())
275 buildOpName(Target: VRegs[i][0], Name: Arg.getName(), MIRBuilder);
276 if (isPointerTyOrWrapper(Ty: Arg.getType()) && !ST->isShader()) {
277 auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes());
278 if (DerefBytes != 0 && ST->isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)))
279 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
280 Dec: SPIRV::Decoration::MaxByteOffset, DecArgs: {DerefBytes});
281 }
282 if (Arg.hasAttribute(Kind: Attribute::Alignment) && !ST->isShader()) {
283 auto Alignment = static_cast<unsigned>(
284 Arg.getAttribute(Kind: Attribute::Alignment).getValueAsInt());
285 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder, Dec: SPIRV::Decoration::Alignment,
286 DecArgs: {Alignment});
287 }
288 if (ST->isKernel()) {
289 if (Arg.hasAttribute(Kind: Attribute::ReadOnly)) {
290 auto Attr =
291 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite);
292 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
293 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
294 }
295 if (Arg.hasAttribute(Kind: Attribute::ReadNone)) {
296 auto Attr = static_cast<unsigned>(
297 SPIRV::FunctionParameterAttribute::NoReadWrite);
298 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
299 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
300 }
301 if (Arg.hasAttribute(Kind: Attribute::ZExt)) {
302 auto Attr =
303 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext);
304 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
305 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
306 }
307 if (Arg.hasAttribute(Kind: Attribute::SExt)) {
308 auto Attr =
309 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sext);
310 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
311 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
312 }
313 if (Arg.hasAttribute(Kind: Attribute::NoAlias)) {
314 auto Attr =
315 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias);
316 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
317 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
318 }
319 if (Arg.hasNoCaptureAttr()) {
320 auto Attr = static_cast<unsigned>(
321 SPIRV::FunctionParameterAttribute::NoCapture);
322 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
323 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
324 }
325 // TODO: the AMDGPU BE only supports ByRef argument passing, thus for
326 // AMDGCN flavoured SPIRV we CodeGen for ByRef, but lower it to
327 // ByVal, handling the impedance mismatch during reverse
328 // translation from SPIRV to LLVM IR; the vendor check should be
329 // removed once / if SPIRV adds ByRef support.
330 if (Arg.hasAttribute(Kind: Attribute::ByVal) ||
331 (Arg.hasAttribute(Kind: Attribute::ByRef) &&
332 F.getParent()->getTargetTriple().getVendor() ==
333 Triple::VendorType::AMD)) {
334 auto Attr =
335 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::ByVal);
336 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
337 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
338 }
339 if (Arg.hasAttribute(Kind: Attribute::StructRet)) {
340 auto Attr =
341 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sret);
342 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
343 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
344 }
345 }
346
347 MDNode *Node = F.getMetadata(Kind: "spirv.ParameterDecorations");
348 if (Node && i < Node->getNumOperands() &&
349 isa<MDNode>(Val: Node->getOperand(I: i))) {
350 MDNode *MD = cast<MDNode>(Val: Node->getOperand(I: i));
351 for (const MDOperand &MDOp : MD->operands()) {
352 MDNode *MD2 = dyn_cast<MDNode>(Val: MDOp);
353 assert(MD2 && "Metadata operand is expected");
354 ConstantInt *Const = getMDOperandAsConstInt(N: MD2, I: 0);
355 assert(Const && "MDOperand should be ConstantInt");
356 auto Dec =
357 static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue());
358 std::vector<uint32_t> DecVec;
359 for (unsigned j = 1; j < MD2->getNumOperands(); j++) {
360 ConstantInt *Const = getMDOperandAsConstInt(N: MD2, I: j);
361 assert(Const && "MDOperand should be ConstantInt");
362 DecVec.push_back(x: static_cast<uint32_t>(Const->getZExtValue()));
363 }
364 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder, Dec, DecArgs: DecVec);
365 }
366 }
367 ++i;
368 }
369 }
370
371 auto MRI = MIRBuilder.getMRI();
372 Register FuncVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
373 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::iIDRegClass);
374 FunctionType *FTy = SPIRV::getOriginalFunctionType(F);
375 Type *FRetTy = FTy->getReturnType();
376 if (isUntypedPointerTy(T: FRetTy)) {
377 if (Type *FRetElemTy = GR->findDeducedElementType(Val: &F)) {
378 TypedPointerType *DerivedTy = TypedPointerType::get(
379 ElementType: toTypedPointer(Ty: FRetElemTy), AddressSpace: getPointerAddressSpace(T: FRetTy));
380 GR->addReturnType(ArgF: &F, DerivedTy);
381 FRetTy = DerivedTy;
382 }
383 }
384 SPIRVTypeInst RetTy = GR->getOrCreateSPIRVType(
385 Type: FRetTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
386 FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, SRetTy: RetTy, SArgTys: ArgTypeVRegs);
387 SPIRVTypeInst FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
388 Ty: FTy, RetType: RetTy, ArgTypes: ArgTypeVRegs, MIRBuilder);
389 uint32_t FuncControl = getFunctionControl(F, ST);
390
391 // Add OpFunction instruction
392 MachineInstrBuilder MB = MIRBuilder.buildInstr(Opcode: SPIRV::OpFunction)
393 .addDef(RegNo: FuncVReg)
394 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetTy))
395 .addImm(Val: FuncControl)
396 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncTy));
397 GR->recordFunctionDefinition(F: &F, MO: &MB.getInstr()->getOperand(i: 0));
398 GR->addGlobalObject(V: &F, MF: &MIRBuilder.getMF(), R: FuncVReg);
399 if (F.isDeclaration())
400 GR->add(V: &F, MI: MB);
401
402 // Add OpFunctionParameter instructions
403 int i = 0;
404 for (const auto &Arg : F.args()) {
405 assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
406 Register ArgReg = VRegs[i][0];
407 MRI->setRegClass(Reg: ArgReg, RC: GR->getRegClass(SpvType: ArgTypeVRegs[i]));
408 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpFunctionParameter)
409 .addDef(RegNo: ArgReg)
410 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ArgTypeVRegs[i]));
411 if (F.isDeclaration())
412 GR->add(V: &Arg, MI: MIB);
413 GR->addGlobalObject(V: &Arg, MF: &MIRBuilder.getMF(), R: ArgReg);
414 i++;
415 }
416 if (!ST->isShader()) {
417 if (F.hasRetAttribute(Kind: Attribute::ZExt)) {
418 auto Attr =
419 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext);
420 buildOpDecorate(Reg: FuncVReg, MIRBuilder, Dec: SPIRV::Decoration::FuncParamAttr,
421 DecArgs: {Attr});
422 }
423 if (F.hasRetAttribute(Kind: Attribute::SExt)) {
424 auto Attr =
425 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sext);
426 buildOpDecorate(Reg: FuncVReg, MIRBuilder, Dec: SPIRV::Decoration::FuncParamAttr,
427 DecArgs: {Attr});
428 }
429 }
430
431 // Name the function.
432 if (F.hasName())
433 buildOpName(Target: FuncVReg, Name: F.getName(), MIRBuilder);
434
435 // Handle entry points and function linkage.
436 if (isEntryPoint(F)) {
437 if (F.getName().empty())
438 report_fatal_error(reason: "SPIR-V entry point function must have a name");
439 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpEntryPoint)
440 .addImm(Val: static_cast<uint32_t>(getExecutionModel(STI: *ST, F)))
441 .addUse(RegNo: FuncVReg);
442 addStringImm(Str: F.getName(), MIB);
443 } else if (const auto LnkTy = getSpirvLinkageTypeFor(ST: *ST, GV: F);
444 LnkTy && !F.getName().empty()) {
445 buildOpDecorate(Reg: FuncVReg, MIRBuilder, Dec: SPIRV::Decoration::LinkageAttributes,
446 DecArgs: {static_cast<uint32_t>(*LnkTy)}, StrImm: F.getName());
447 }
448
449 // Handle function pointers decoration
450 bool hasFunctionPointers =
451 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
452 if (hasFunctionPointers) {
453 if (F.hasFnAttribute(Kind: "referenced-indirectly")) {
454 assert((F.getCallingConv() != CallingConv::SPIR_KERNEL) &&
455 "Unexpected 'referenced-indirectly' attribute of the kernel "
456 "function");
457 buildOpDecorate(Reg: FuncVReg, MIRBuilder,
458 Dec: SPIRV::Decoration::ReferencedIndirectlyINTEL, DecArgs: {});
459 }
460 }
461
462 if (MDNode *FuncMD = F.getMetadata(Kind: "spirv.Decorations"))
463 buildOpSpirvDecorations(Reg: FuncVReg, MIRBuilder, GVarMD: FuncMD, ST: *ST);
464
465 return true;
466}
467
468// TODO:
469// - add a topological sort of IndirectCalls to ensure the best types knowledge
470// - we may need to fix function formal parameter types if they are opaque
471// pointers used as function pointers in these indirect calls
472// - defaulting to StorageClass::Function in the absence of the
473// SPV_INTEL_function_pointers extension seems wrong, as that might not be
474// able to hold a full width pointer to function, and it also does not model
475// the semantics of a pointer to function in a generic fashion.
476void SPIRVCallLowering::produceIndirectPtrType(
477 MachineIRBuilder &MIRBuilder,
478 const SPIRVCallLowering::SPIRVIndirectCall &IC) const {
479 // Create indirect call data type if any
480 MachineFunction &MF = MIRBuilder.getMF();
481 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
482 SPIRVTypeInst SpirvRetTy = GR->getOrCreateSPIRVType(
483 Type: IC.RetTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
484 SmallVector<SPIRVTypeInst, 4> SpirvArgTypes;
485 for (size_t i = 0; i < IC.ArgTys.size(); ++i) {
486 SPIRVTypeInst SPIRVTy = GR->getOrCreateSPIRVType(
487 Type: IC.ArgTys[i], MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
488 SpirvArgTypes.push_back(Elt: SPIRVTy);
489 if (!GR->getSPIRVTypeForVReg(VReg: IC.ArgRegs[i]))
490 GR->assignSPIRVTypeToVReg(Type: SPIRVTy, VReg: IC.ArgRegs[i], MF);
491 }
492 // SPIR-V function type:
493 FunctionType *FTy =
494 FunctionType::get(Result: const_cast<Type *>(IC.RetTy), Params: IC.ArgTys, isVarArg: false);
495 SPIRVTypeInst SpirvFuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
496 Ty: FTy, RetType: SpirvRetTy, ArgTypes: SpirvArgTypes, MIRBuilder);
497 // SPIR-V pointer to function type:
498 auto SC = ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
499 ? SPIRV::StorageClass::CodeSectionINTEL
500 : SPIRV::StorageClass::Function;
501 SPIRVTypeInst IndirectFuncPtrTy =
502 GR->getOrCreateSPIRVPointerType(BaseType: SpirvFuncTy, MIRBuilder, SC);
503 // Correct the Callee type
504 GR->assignSPIRVTypeToVReg(Type: IndirectFuncPtrTy, VReg: IC.Callee, MF);
505}
506
507bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
508 CallLoweringInfo &Info) const {
509 // Currently call returns should have single vregs.
510 // TODO: handle the case of multiple registers.
511 if (Info.OrigRet.Regs.size() > 1)
512 return false;
513 MachineFunction &MF = MIRBuilder.getMF();
514 GR->setCurrentFunc(MF);
515 const Function *CF = nullptr;
516 std::string DemangledName;
517 const Type *OrigRetTy = Info.OrigRet.Ty;
518
519 // Emit a regular OpFunctionCall. If it's an externally declared function,
520 // be sure to emit its type and function declaration here. It will be hoisted
521 // globally later.
522 if (Info.Callee.isGlobal()) {
523 std::string FuncName = Info.Callee.getGlobal()->getName().str();
524 DemangledName = getOclOrSpirvBuiltinDemangledName(Name: FuncName);
525 CF = dyn_cast_or_null<const Function>(Val: Info.Callee.getGlobal());
526 // TODO: support constexpr casts and indirect calls.
527 if (CF == nullptr)
528 return false;
529
530 FunctionType *FTy = SPIRV::getOriginalFunctionType(F: *CF);
531 OrigRetTy = FTy->getReturnType();
532 if (isUntypedPointerTy(T: OrigRetTy)) {
533 if (auto *DerivedRetTy = GR->findReturnType(ArgF: CF))
534 OrigRetTy = DerivedRetTy;
535 }
536 }
537
538 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
539 Register ResVReg =
540 Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0];
541 const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
542
543 bool isFunctionDecl = CF && CF->isDeclaration();
544 if (isFunctionDecl && !DemangledName.empty()) {
545 if (ResVReg.isValid()) {
546 if (!GR->getSPIRVTypeForVReg(VReg: ResVReg)) {
547 const Type *RetTy = OrigRetTy;
548 if (auto *PtrRetTy = dyn_cast<PointerType>(Val: OrigRetTy)) {
549 const Value *OrigValue = Info.OrigRet.OrigValue;
550 if (!OrigValue)
551 OrigValue = Info.CB;
552 if (OrigValue)
553 if (Type *ElemTy = GR->findDeducedElementType(Val: OrigValue))
554 RetTy =
555 TypedPointerType::get(ElementType: ElemTy, AddressSpace: PtrRetTy->getAddressSpace());
556 }
557 setRegClassType(Reg: ResVReg, Ty: RetTy, GR, MIRBuilder,
558 AccessQual: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
559 }
560 } else {
561 ResVReg = createVirtualRegister(Ty: OrigRetTy, GR, MIRBuilder,
562 AccessQual: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
563 }
564 SmallVector<Register, 8> ArgVRegs;
565 for (auto Arg : Info.OrigArgs) {
566 assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
567 Register ArgReg = Arg.Regs[0];
568 ArgVRegs.push_back(Elt: ArgReg);
569 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: ArgReg);
570 if (!SpvType) {
571 Type *ArgTy = nullptr;
572 if (auto *PtrArgTy = dyn_cast<PointerType>(Val: Arg.Ty)) {
573 // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
574 // don't have access to original value in LLVM IR or info about
575 // deduced pointee type, then we should wait with setting the type for
576 // the virtual register until pre-legalizer step when we access
577 // @llvm.spv.assign.ptr.type.p...(...)'s info.
578 if (Arg.OrigValue)
579 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg.OrigValue))
580 ArgTy =
581 TypedPointerType::get(ElementType: ElemTy, AddressSpace: PtrArgTy->getAddressSpace());
582 } else {
583 ArgTy = Arg.Ty;
584 }
585 if (ArgTy) {
586 SpvType = GR->getOrCreateSPIRVType(
587 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
588 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ArgReg, MF);
589 }
590 }
591 if (!MRI->getRegClassOrNull(Reg: ArgReg)) {
592 // Either we have SpvType created, or Arg.Ty is an untyped pointer and
593 // we know its virtual register's class and type even if we don't know
594 // pointee type.
595 MRI->setRegClass(Reg: ArgReg, RC: SpvType ? GR->getRegClass(SpvType)
596 : &SPIRV::pIDRegClass);
597 MRI->setType(
598 VReg: ArgReg,
599 Ty: SpvType ? GR->getRegType(SpvType)
600 : LLT::pointer(AddressSpace: cast<PointerType>(Val: Arg.Ty)->getAddressSpace(),
601 SizeInBits: GR->getPointerSize()));
602 }
603 }
604 if (auto Res = SPIRV::lowerBuiltin(
605 DemangledCall: DemangledName, Set: ST->getPreferredInstructionSet(), MIRBuilder,
606 OrigRet: ResVReg, OrigRetTy, Args: ArgVRegs, GR, CB: *Info.CB))
607 return *Res;
608 }
609
610 if (isFunctionDecl && !GR->find(V: CF, MF: &MF).isValid()) {
611 // Emit the type info and forward function declaration to the first MBB
612 // to ensure VReg definition dependencies are valid across all MBBs.
613 MachineIRBuilder FirstBlockBuilder;
614 FirstBlockBuilder.setMF(MF);
615 FirstBlockBuilder.setMBB(*MF.getBlockNumbered(N: 0));
616
617 SmallVector<ArrayRef<Register>, 8> VRegArgs;
618 SmallVector<SmallVector<Register, 1>, 8> ToInsert;
619 for (const Argument &Arg : CF->args()) {
620 if (MIRBuilder.getDataLayout().getTypeStoreSize(Ty: Arg.getType()).isZero())
621 continue; // Don't handle zero sized types.
622 Register Reg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
623 MRI->setRegClass(Reg, RC: &SPIRV::iIDRegClass);
624 ToInsert.push_back(Elt: {Reg});
625 VRegArgs.push_back(Elt: ToInsert.back());
626 }
627 // TODO: Reuse FunctionLoweringInfo
628 FunctionLoweringInfo FuncInfo;
629 lowerFormalArguments(MIRBuilder&: FirstBlockBuilder, F: *CF, VRegs: VRegArgs, FLI&: FuncInfo);
630 }
631
632 // Ignore the call if it's called from the internal service function
633 if (MIRBuilder.getMF()
634 .getFunction()
635 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
636 .isValid()) {
637 // insert a no-op
638 MIRBuilder.buildTrap();
639 return true;
640 }
641
642 unsigned CallOp;
643 if (Info.CB && Info.CB->isIndirectCall()) {
644 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
645 report_fatal_error(reason: "An indirect call is encountered but SPIR-V without "
646 "extensions does not support it",
647 gen_crash_diag: false);
648 // Set instruction operation according to SPV_INTEL_function_pointers
649 CallOp = SPIRV::OpFunctionPointerCallINTEL;
650 // Collect information about the indirect call to create correct types.
651 Register CalleeReg = Info.Callee.getReg();
652 if (CalleeReg.isValid()) {
653 SPIRVCallLowering::SPIRVIndirectCall IndirectCall;
654 IndirectCall.Callee = CalleeReg;
655 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *Info.CB);
656 IndirectCall.RetTy = OrigRetTy = FTy->getReturnType();
657 assert(FTy->getNumParams() == Info.OrigArgs.size() &&
658 "Function types mismatch");
659 for (unsigned I = 0; I != Info.OrigArgs.size(); ++I) {
660 assert(Info.OrigArgs[I].Regs.size() == 1 &&
661 "Call arg has multiple VRegs");
662 IndirectCall.ArgTys.push_back(Elt: FTy->getParamType(i: I));
663 IndirectCall.ArgRegs.push_back(Elt: Info.OrigArgs[I].Regs[0]);
664 }
665 produceIndirectPtrType(MIRBuilder, IC: IndirectCall);
666 }
667 } else {
668 // Emit a regular OpFunctionCall
669 CallOp = SPIRV::OpFunctionCall;
670 }
671
672 // Make sure there's a valid return reg, even for functions returning void.
673 if (!ResVReg.isValid())
674 ResVReg = MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
675 SPIRVTypeInst RetType = GR->assignTypeToVReg(
676 Type: OrigRetTy, VReg: ResVReg, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
677
678 // Emit the call instruction and its args.
679 auto MIB = MIRBuilder.buildInstr(Opcode: CallOp)
680 .addDef(RegNo: ResVReg)
681 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
682 .add(MO: Info.Callee);
683
684 for (const auto &Arg : Info.OrigArgs) {
685 // Currently call args should have single vregs.
686 if (Arg.Regs.size() > 1)
687 return false;
688 MIB.addUse(RegNo: Arg.Regs[0]);
689 }
690
691 if (Info.CB)
692 MIB.getInstr()->copyIRFlags(I: *Info.CB);
693
694 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
695 // Process aliasing metadata.
696 const CallBase *CI = Info.CB;
697 if (CI && CI->hasMetadata()) {
698 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alias_scope))
699 GR->buildMemAliasingOpDecorate(Reg: ResVReg, MIRBuilder,
700 Dec: SPIRV::Decoration::AliasScopeINTEL, GVarMD: MD);
701 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_noalias))
702 GR->buildMemAliasingOpDecorate(Reg: ResVReg, MIRBuilder,
703 Dec: SPIRV::Decoration::NoAliasINTEL, GVarMD: MD);
704 }
705 }
706
707 MIB.constrainAllUses(TII: MIRBuilder.getTII(), TRI: *ST->getRegisterInfo(),
708 RBI: *ST->getRegBankInfo());
709 return true;
710}
711