1//===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the lowering of LLVM calls to machine code calls for
10// GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVCallLowering.h"
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "SPIRV.h"
17#include "SPIRVBuiltins.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVISelLowering.h"
20#include "SPIRVMetadata.h"
21#include "SPIRVRegisterInfo.h"
22#include "SPIRVSubtarget.h"
23#include "SPIRVUtils.h"
24#include "llvm/CodeGen/FunctionLoweringInfo.h"
25#include "llvm/IR/IntrinsicInst.h"
26#include "llvm/IR/IntrinsicsSPIRV.h"
27#include "llvm/Support/ModRef.h"
28
29using namespace llvm;
30
31SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI,
32 SPIRVGlobalRegistry *GR)
33 : CallLowering(&TLI), GR(GR) {}
34
35bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
36 const Value *Val, ArrayRef<Register> VRegs,
37 FunctionLoweringInfo &FLI,
38 Register SwiftErrorVReg) const {
39 // Ignore if called from the internal service function
40 if (MIRBuilder.getMF()
41 .getFunction()
42 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
43 .isValid())
44 return true;
45
46 // Currently all return types should use a single register.
47 // TODO: handle the case of multiple registers.
48 if (VRegs.size() > 1)
49 return false;
50
51 if (Val) {
52 const auto &STI = MIRBuilder.getMF().getSubtarget();
53 MIRBuilder.buildInstr(Opcode: SPIRV::OpReturnValue)
54 .addUse(RegNo: VRegs[0])
55 .constrainAllUses(TII: MIRBuilder.getTII(), TRI: *STI.getRegisterInfo(),
56 RBI: *STI.getRegBankInfo());
57 return true;
58 }
59 MIRBuilder.buildInstr(Opcode: SPIRV::OpReturn);
60 return true;
61}
62
63// Based on the LLVM function attributes, get a SPIR-V FunctionControl.
64static uint32_t getFunctionControl(const Function &F,
65 const SPIRVSubtarget *ST) {
66 MemoryEffects MemEffects = F.getMemoryEffects();
67
68 uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None);
69
70 if (F.hasFnAttribute(Kind: Attribute::AttrKind::NoInline))
71 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline);
72 else if (F.hasFnAttribute(Kind: Attribute::AttrKind::AlwaysInline))
73 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline);
74
75 if (MemEffects.doesNotAccessMemory())
76 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure);
77 else if (MemEffects.onlyReadsMemory())
78 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const);
79
80 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone) ||
81 ST->canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone))
82 if (F.hasFnAttribute(Kind: Attribute::OptimizeNone))
83 FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::OptNoneEXT);
84
85 return FuncControl;
86}
87
88static ConstantInt *getConstInt(MDNode *MD, unsigned NumOp) {
89 if (MD->getNumOperands() > NumOp) {
90 auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MD->getOperand(I: NumOp));
91 if (CMeta)
92 return dyn_cast<ConstantInt>(Val: CMeta->getValue());
93 }
94 return nullptr;
95}
96
97// If the function has pointer arguments, we are forced to re-create this
98// function type from the very beginning, changing PointerType by
99// TypedPointerType for each pointer argument. Otherwise, the same `Type*`
100// potentially corresponds to different SPIR-V function type, effectively
101// invalidating logic behind global registry and duplicates tracker.
102static FunctionType *
103fixFunctionTypeIfPtrArgs(SPIRVGlobalRegistry *GR, const Function &F,
104 FunctionType *FTy, SPIRVTypeInst SRetTy,
105 const SmallVector<SPIRVTypeInst, 4> &SArgTys) {
106 bool hasArgPtrs = false;
107 for (auto &Arg : F.args()) {
108 // check if it's an instance of a non-typed PointerType
109 if (Arg.getType()->isPointerTy()) {
110 hasArgPtrs = true;
111 break;
112 }
113 }
114 if (!hasArgPtrs) {
115 Type *RetTy = FTy->getReturnType();
116 // check if it's an instance of a non-typed PointerType
117 if (!RetTy->isPointerTy())
118 return FTy;
119 }
120
121 // re-create function type, using TypedPointerType instead of PointerType to
122 // properly trace argument types
123 const Type *RetTy = GR->getTypeForSPIRVType(Ty: SRetTy);
124 SmallVector<Type *, 4> ArgTys;
125 for (auto SArgTy : SArgTys)
126 ArgTys.push_back(Elt: const_cast<Type *>(GR->getTypeForSPIRVType(Ty: SArgTy)));
127 return FunctionType::get(Result: const_cast<Type *>(RetTy), Params: ArgTys, isVarArg: false);
128}
129
130static SPIRV::AccessQualifier::AccessQualifier
131getArgAccessQual(const Function &F, unsigned ArgIdx) {
132 if (F.getCallingConv() != CallingConv::SPIR_KERNEL)
133 return SPIRV::AccessQualifier::ReadWrite;
134
135 MDString *ArgAttribute = getOCLKernelArgAccessQual(F, ArgIdx);
136 if (!ArgAttribute)
137 return SPIRV::AccessQualifier::ReadWrite;
138
139 if (ArgAttribute->getString() == "read_only")
140 return SPIRV::AccessQualifier::ReadOnly;
141 if (ArgAttribute->getString() == "write_only")
142 return SPIRV::AccessQualifier::WriteOnly;
143 return SPIRV::AccessQualifier::ReadWrite;
144}
145
146static std::vector<SPIRV::Decoration::Decoration>
147getKernelArgTypeQual(const Function &F, unsigned ArgIdx) {
148 MDString *ArgAttribute = getOCLKernelArgTypeQual(F, ArgIdx);
149 if (ArgAttribute && ArgAttribute->getString() == "volatile")
150 return {SPIRV::Decoration::Volatile};
151 return {};
152}
153
154static SPIRVTypeInst getArgSPIRVType(const Function &F, unsigned ArgIdx,
155 SPIRVGlobalRegistry *GR,
156 MachineIRBuilder &MIRBuilder,
157 const SPIRVSubtarget &ST) {
158 // Read argument's access qualifier from metadata or default.
159 SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
160 getArgAccessQual(F, ArgIdx);
161
162 Type *OriginalArgType =
163 SPIRV::getOriginalFunctionType(F)->getParamType(i: ArgIdx);
164
165 // If OriginalArgType is non-pointer, use the OriginalArgType (the type cannot
166 // be legally reassigned later).
167 if (!isPointerTy(T: OriginalArgType))
168 return GR->getOrCreateSPIRVType(Type: OriginalArgType, MIRBuilder, AQ: ArgAccessQual,
169 EmitIR: true);
170
171 Argument *Arg = F.getArg(i: ArgIdx);
172 Type *ArgType = Arg->getType();
173 if (isTypedPointerTy(T: ArgType)) {
174 return GR->getOrCreateSPIRVPointerType(
175 BaseType: cast<TypedPointerType>(Val: ArgType)->getElementType(), MIRBuilder,
176 SC: addressSpaceToStorageClass(AddrSpace: getPointerAddressSpace(T: ArgType), STI: ST));
177 }
178
179 // In case OriginalArgType is of untyped pointer type, there are three
180 // possibilities:
181 // 1) This is a pointer of an LLVM IR element type, passed byval/byref.
182 // 2) This is an OpenCL/SPIR-V builtin type if there is spv_assign_type
183 // intrinsic assigning a TargetExtType.
184 // 3) This is a pointer, try to retrieve pointer element type from a
185 // spv_assign_ptr_type intrinsic or otherwise use default pointer element
186 // type.
187 if (hasPointeeTypeAttr(Arg)) {
188 return GR->getOrCreateSPIRVPointerType(
189 BaseType: getPointeeTypeByAttr(Arg), MIRBuilder,
190 SC: addressSpaceToStorageClass(AddrSpace: getPointerAddressSpace(T: ArgType), STI: ST));
191 }
192
193 for (auto User : Arg->users()) {
194 auto *II = dyn_cast<IntrinsicInst>(Val: User);
195 // Check if this is spv_assign_type assigning OpenCL/SPIR-V builtin type.
196 if (II && II->getIntrinsicID() == Intrinsic::spv_assign_type) {
197 MetadataAsValue *VMD = cast<MetadataAsValue>(Val: II->getOperand(i_nocapture: 1));
198 Type *BuiltinType =
199 cast<ConstantAsMetadata>(Val: VMD->getMetadata())->getType();
200 assert(BuiltinType->isTargetExtTy() && "Expected TargetExtType");
201 return GR->getOrCreateSPIRVType(Type: BuiltinType, MIRBuilder, AQ: ArgAccessQual,
202 EmitIR: true);
203 }
204
205 // Check if this is spv_assign_ptr_type assigning pointer element type.
206 if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type)
207 continue;
208
209 MetadataAsValue *VMD = cast<MetadataAsValue>(Val: II->getOperand(i_nocapture: 1));
210 Type *ElementTy =
211 toTypedPointer(Ty: cast<ConstantAsMetadata>(Val: VMD->getMetadata())->getType());
212 return GR->getOrCreateSPIRVPointerType(
213 BaseType: ElementTy, MIRBuilder,
214 SC: addressSpaceToStorageClass(
215 AddrSpace: cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getZExtValue(), STI: ST));
216 }
217
218 // Replace PointerType with TypedPointerType to be able to map SPIR-V types to
219 // LLVM types in a consistent manner
220 return GR->getOrCreateSPIRVType(Type: toTypedPointer(Ty: OriginalArgType), MIRBuilder,
221 AQ: ArgAccessQual, EmitIR: true);
222}
223
224static SPIRV::ExecutionModel::ExecutionModel
225getExecutionModel(const SPIRVSubtarget &STI, const Function &F) {
226 assert(STI.getEnv() != SPIRVSubtarget::Unknown &&
227 "Environment must be resolved before lowering entry points.");
228
229 if (STI.isKernel())
230 return SPIRV::ExecutionModel::Kernel;
231
232 auto attribute = F.getFnAttribute(Kind: "hlsl.shader");
233 if (!attribute.isValid()) {
234 report_fatal_error(
235 reason: "This entry point lacks mandatory hlsl.shader attribute.");
236 }
237
238 const auto value = attribute.getValueAsString();
239 if (value == "compute")
240 return SPIRV::ExecutionModel::GLCompute;
241 if (value == "vertex")
242 return SPIRV::ExecutionModel::Vertex;
243 if (value == "pixel")
244 return SPIRV::ExecutionModel::Fragment;
245
246 report_fatal_error(reason: "This HLSL entry point is not supported by this backend.");
247}
248
249bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
250 const Function &F,
251 ArrayRef<ArrayRef<Register>> VRegs,
252 FunctionLoweringInfo &FLI) const {
253 // Discard the internal service function
254 if (F.getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME).isValid())
255 return true;
256
257 assert(GR && "Must initialize the SPIRV type registry before lowering args.");
258 GR->setCurrentFunc(MIRBuilder.getMF());
259
260 // Get access to information about available extensions
261 const SPIRVSubtarget *ST =
262 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
263
264 // Assign types and names to all args, and store their types for later.
265 SmallVector<SPIRVTypeInst, 4> ArgTypeVRegs;
266 if (VRegs.size() > 0) {
267 unsigned i = 0;
268 for (const auto &Arg : F.args()) {
269 // Currently formal args should use single registers.
270 // TODO: handle the case of multiple registers.
271 if (VRegs[i].size() > 1)
272 return false;
273 SPIRVTypeInst SpirvTy = getArgSPIRVType(F, ArgIdx: i, GR, MIRBuilder, ST: *ST);
274 GR->assignSPIRVTypeToVReg(Type: SpirvTy, VReg: VRegs[i][0], MF: MIRBuilder.getMF());
275 ArgTypeVRegs.push_back(Elt: SpirvTy);
276
277 if (Arg.hasName())
278 buildOpName(Target: VRegs[i][0], Name: Arg.getName(), MIRBuilder);
279 if (isPointerTyOrWrapper(Ty: Arg.getType())) {
280 auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes());
281 if (DerefBytes != 0)
282 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
283 Dec: SPIRV::Decoration::MaxByteOffset, DecArgs: {DerefBytes});
284 }
285 if (Arg.hasAttribute(Kind: Attribute::Alignment) && !ST->isShader()) {
286 auto Alignment = static_cast<unsigned>(
287 Arg.getAttribute(Kind: Attribute::Alignment).getValueAsInt());
288 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder, Dec: SPIRV::Decoration::Alignment,
289 DecArgs: {Alignment});
290 }
291 if (Arg.hasAttribute(Kind: Attribute::ReadOnly)) {
292 auto Attr =
293 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite);
294 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
295 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
296 }
297 if (Arg.hasAttribute(Kind: Attribute::ZExt)) {
298 auto Attr =
299 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext);
300 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
301 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
302 }
303 if (Arg.hasAttribute(Kind: Attribute::NoAlias)) {
304 auto Attr =
305 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias);
306 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
307 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
308 }
309 // TODO: the AMDGPU BE only supports ByRef argument passing, thus for
310 // AMDGCN flavoured SPIRV we CodeGen for ByRef, but lower it to
311 // ByVal, handling the impedance mismatch during reverse
312 // translation from SPIRV to LLVM IR; the vendor check should be
313 // removed once / if SPIRV adds ByRef support.
314 if (Arg.hasAttribute(Kind: Attribute::ByVal) ||
315 (Arg.hasAttribute(Kind: Attribute::ByRef) &&
316 F.getParent()->getTargetTriple().getVendor() ==
317 Triple::VendorType::AMD)) {
318 auto Attr =
319 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::ByVal);
320 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
321 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
322 }
323 if (Arg.hasAttribute(Kind: Attribute::StructRet)) {
324 auto Attr =
325 static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sret);
326 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder,
327 Dec: SPIRV::Decoration::FuncParamAttr, DecArgs: {Attr});
328 }
329
330 if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
331 std::vector<SPIRV::Decoration::Decoration> ArgTypeQualDecs =
332 getKernelArgTypeQual(F, ArgIdx: i);
333 for (SPIRV::Decoration::Decoration Decoration : ArgTypeQualDecs)
334 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder, Dec: Decoration, DecArgs: {});
335 }
336
337 MDNode *Node = F.getMetadata(Kind: "spirv.ParameterDecorations");
338 if (Node && i < Node->getNumOperands() &&
339 isa<MDNode>(Val: Node->getOperand(I: i))) {
340 MDNode *MD = cast<MDNode>(Val: Node->getOperand(I: i));
341 for (const MDOperand &MDOp : MD->operands()) {
342 MDNode *MD2 = dyn_cast<MDNode>(Val: MDOp);
343 assert(MD2 && "Metadata operand is expected");
344 ConstantInt *Const = getConstInt(MD: MD2, NumOp: 0);
345 assert(Const && "MDOperand should be ConstantInt");
346 auto Dec =
347 static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue());
348 std::vector<uint32_t> DecVec;
349 for (unsigned j = 1; j < MD2->getNumOperands(); j++) {
350 ConstantInt *Const = getConstInt(MD: MD2, NumOp: j);
351 assert(Const && "MDOperand should be ConstantInt");
352 DecVec.push_back(x: static_cast<uint32_t>(Const->getZExtValue()));
353 }
354 buildOpDecorate(Reg: VRegs[i][0], MIRBuilder, Dec, DecArgs: DecVec);
355 }
356 }
357 ++i;
358 }
359 }
360
361 auto MRI = MIRBuilder.getMRI();
362 Register FuncVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
363 MRI->setRegClass(Reg: FuncVReg, RC: &SPIRV::iIDRegClass);
364 FunctionType *FTy = SPIRV::getOriginalFunctionType(F);
365 Type *FRetTy = FTy->getReturnType();
366 if (isUntypedPointerTy(T: FRetTy)) {
367 if (Type *FRetElemTy = GR->findDeducedElementType(Val: &F)) {
368 TypedPointerType *DerivedTy = TypedPointerType::get(
369 ElementType: toTypedPointer(Ty: FRetElemTy), AddressSpace: getPointerAddressSpace(T: FRetTy));
370 GR->addReturnType(ArgF: &F, DerivedTy);
371 FRetTy = DerivedTy;
372 }
373 }
374 SPIRVTypeInst RetTy = GR->getOrCreateSPIRVType(
375 Type: FRetTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
376 FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, SRetTy: RetTy, SArgTys: ArgTypeVRegs);
377 SPIRVTypeInst FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
378 Ty: FTy, RetType: RetTy, ArgTypes: ArgTypeVRegs, MIRBuilder);
379 uint32_t FuncControl = getFunctionControl(F, ST);
380
381 // Add OpFunction instruction
382 MachineInstrBuilder MB = MIRBuilder.buildInstr(Opcode: SPIRV::OpFunction)
383 .addDef(RegNo: FuncVReg)
384 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetTy))
385 .addImm(Val: FuncControl)
386 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncTy));
387 GR->recordFunctionDefinition(F: &F, MO: &MB.getInstr()->getOperand(i: 0));
388 GR->addGlobalObject(V: &F, MF: &MIRBuilder.getMF(), R: FuncVReg);
389 if (F.isDeclaration())
390 GR->add(V: &F, MI: MB);
391
392 // Add OpFunctionParameter instructions
393 int i = 0;
394 for (const auto &Arg : F.args()) {
395 assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
396 Register ArgReg = VRegs[i][0];
397 MRI->setRegClass(Reg: ArgReg, RC: GR->getRegClass(SpvType: ArgTypeVRegs[i]));
398 MRI->setType(VReg: ArgReg, Ty: GR->getRegType(SpvType: ArgTypeVRegs[i]));
399 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpFunctionParameter)
400 .addDef(RegNo: ArgReg)
401 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ArgTypeVRegs[i]));
402 if (F.isDeclaration())
403 GR->add(V: &Arg, MI: MIB);
404 GR->addGlobalObject(V: &Arg, MF: &MIRBuilder.getMF(), R: ArgReg);
405 i++;
406 }
407 // Name the function.
408 if (F.hasName())
409 buildOpName(Target: FuncVReg, Name: F.getName(), MIRBuilder);
410
411 // Handle entry points and function linkage.
412 if (isEntryPoint(F)) {
413 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpEntryPoint)
414 .addImm(Val: static_cast<uint32_t>(getExecutionModel(STI: *ST, F)))
415 .addUse(RegNo: FuncVReg);
416 addStringImm(Str: F.getName(), MIB);
417 } else if (const auto LnkTy = getSpirvLinkageTypeFor(ST: *ST, GV: F)) {
418 buildOpDecorate(Reg: FuncVReg, MIRBuilder, Dec: SPIRV::Decoration::LinkageAttributes,
419 DecArgs: {static_cast<uint32_t>(*LnkTy)}, StrImm: F.getName());
420 }
421
422 // Handle function pointers decoration
423 bool hasFunctionPointers =
424 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
425 if (hasFunctionPointers) {
426 if (F.hasFnAttribute(Kind: "referenced-indirectly")) {
427 assert((F.getCallingConv() != CallingConv::SPIR_KERNEL) &&
428 "Unexpected 'referenced-indirectly' attribute of the kernel "
429 "function");
430 buildOpDecorate(Reg: FuncVReg, MIRBuilder,
431 Dec: SPIRV::Decoration::ReferencedIndirectlyINTEL, DecArgs: {});
432 }
433 }
434
435 return true;
436}
437
438// TODO:
439// - add a topological sort of IndirectCalls to ensure the best types knowledge
440// - we may need to fix function formal parameter types if they are opaque
441// pointers used as function pointers in these indirect calls
442// - defaulting to StorageClass::Function in the absence of the
443// SPV_INTEL_function_pointers extension seems wrong, as that might not be
444// able to hold a full width pointer to function, and it also does not model
445// the semantics of a pointer to function in a generic fashion.
446void SPIRVCallLowering::produceIndirectPtrType(
447 MachineIRBuilder &MIRBuilder,
448 const SPIRVCallLowering::SPIRVIndirectCall &IC) const {
449 // Create indirect call data type if any
450 MachineFunction &MF = MIRBuilder.getMF();
451 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
452 SPIRVTypeInst SpirvRetTy = GR->getOrCreateSPIRVType(
453 Type: IC.RetTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
454 SmallVector<SPIRVTypeInst, 4> SpirvArgTypes;
455 for (size_t i = 0; i < IC.ArgTys.size(); ++i) {
456 SPIRVTypeInst SPIRVTy = GR->getOrCreateSPIRVType(
457 Type: IC.ArgTys[i], MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
458 SpirvArgTypes.push_back(Elt: SPIRVTy);
459 if (!GR->getSPIRVTypeForVReg(VReg: IC.ArgRegs[i]))
460 GR->assignSPIRVTypeToVReg(Type: SPIRVTy, VReg: IC.ArgRegs[i], MF);
461 }
462 // SPIR-V function type:
463 FunctionType *FTy =
464 FunctionType::get(Result: const_cast<Type *>(IC.RetTy), Params: IC.ArgTys, isVarArg: false);
465 SPIRVTypeInst SpirvFuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
466 Ty: FTy, RetType: SpirvRetTy, ArgTypes: SpirvArgTypes, MIRBuilder);
467 // SPIR-V pointer to function type:
468 auto SC = ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)
469 ? SPIRV::StorageClass::CodeSectionINTEL
470 : SPIRV::StorageClass::Function;
471 SPIRVTypeInst IndirectFuncPtrTy =
472 GR->getOrCreateSPIRVPointerType(BaseType: SpirvFuncTy, MIRBuilder, SC);
473 // Correct the Callee type
474 GR->assignSPIRVTypeToVReg(Type: IndirectFuncPtrTy, VReg: IC.Callee, MF);
475}
476
477bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
478 CallLoweringInfo &Info) const {
479 // Currently call returns should have single vregs.
480 // TODO: handle the case of multiple registers.
481 if (Info.OrigRet.Regs.size() > 1)
482 return false;
483 MachineFunction &MF = MIRBuilder.getMF();
484 GR->setCurrentFunc(MF);
485 const Function *CF = nullptr;
486 std::string DemangledName;
487 const Type *OrigRetTy = Info.OrigRet.Ty;
488
489 // Emit a regular OpFunctionCall. If it's an externally declared function,
490 // be sure to emit its type and function declaration here. It will be hoisted
491 // globally later.
492 if (Info.Callee.isGlobal()) {
493 std::string FuncName = Info.Callee.getGlobal()->getName().str();
494 DemangledName = getOclOrSpirvBuiltinDemangledName(Name: FuncName);
495 CF = dyn_cast_or_null<const Function>(Val: Info.Callee.getGlobal());
496 // TODO: support constexpr casts and indirect calls.
497 if (CF == nullptr)
498 return false;
499
500 FunctionType *FTy = SPIRV::getOriginalFunctionType(F: *CF);
501 OrigRetTy = FTy->getReturnType();
502 if (isUntypedPointerTy(T: OrigRetTy)) {
503 if (auto *DerivedRetTy = GR->findReturnType(ArgF: CF))
504 OrigRetTy = DerivedRetTy;
505 }
506 }
507
508 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
509 Register ResVReg =
510 Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0];
511 const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
512
513 bool isFunctionDecl = CF && CF->isDeclaration();
514 if (isFunctionDecl && !DemangledName.empty()) {
515 if (ResVReg.isValid()) {
516 if (!GR->getSPIRVTypeForVReg(VReg: ResVReg)) {
517 const Type *RetTy = OrigRetTy;
518 if (auto *PtrRetTy = dyn_cast<PointerType>(Val: OrigRetTy)) {
519 const Value *OrigValue = Info.OrigRet.OrigValue;
520 if (!OrigValue)
521 OrigValue = Info.CB;
522 if (OrigValue)
523 if (Type *ElemTy = GR->findDeducedElementType(Val: OrigValue))
524 RetTy =
525 TypedPointerType::get(ElementType: ElemTy, AddressSpace: PtrRetTy->getAddressSpace());
526 }
527 setRegClassType(Reg: ResVReg, Ty: RetTy, GR, MIRBuilder,
528 AccessQual: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
529 }
530 } else {
531 ResVReg = createVirtualRegister(Ty: OrigRetTy, GR, MIRBuilder,
532 AccessQual: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
533 }
534 SmallVector<Register, 8> ArgVRegs;
535 for (auto Arg : Info.OrigArgs) {
536 assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
537 Register ArgReg = Arg.Regs[0];
538 ArgVRegs.push_back(Elt: ArgReg);
539 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: ArgReg);
540 if (!SpvType) {
541 Type *ArgTy = nullptr;
542 if (auto *PtrArgTy = dyn_cast<PointerType>(Val: Arg.Ty)) {
543 // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
544 // don't have access to original value in LLVM IR or info about
545 // deduced pointee type, then we should wait with setting the type for
546 // the virtual register until pre-legalizer step when we access
547 // @llvm.spv.assign.ptr.type.p...(...)'s info.
548 if (Arg.OrigValue)
549 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg.OrigValue))
550 ArgTy =
551 TypedPointerType::get(ElementType: ElemTy, AddressSpace: PtrArgTy->getAddressSpace());
552 } else {
553 ArgTy = Arg.Ty;
554 }
555 if (ArgTy) {
556 SpvType = GR->getOrCreateSPIRVType(
557 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
558 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ArgReg, MF);
559 }
560 }
561 if (!MRI->getRegClassOrNull(Reg: ArgReg)) {
562 // Either we have SpvType created, or Arg.Ty is an untyped pointer and
563 // we know its virtual register's class and type even if we don't know
564 // pointee type.
565 MRI->setRegClass(Reg: ArgReg, RC: SpvType ? GR->getRegClass(SpvType)
566 : &SPIRV::pIDRegClass);
567 MRI->setType(
568 VReg: ArgReg,
569 Ty: SpvType ? GR->getRegType(SpvType)
570 : LLT::pointer(AddressSpace: cast<PointerType>(Val: Arg.Ty)->getAddressSpace(),
571 SizeInBits: GR->getPointerSize()));
572 }
573 }
574 if (auto Res = SPIRV::lowerBuiltin(
575 DemangledCall: DemangledName, Set: ST->getPreferredInstructionSet(), MIRBuilder,
576 OrigRet: ResVReg, OrigRetTy, Args: ArgVRegs, GR, CB: *Info.CB))
577 return *Res;
578 }
579
580 if (isFunctionDecl && !GR->find(V: CF, MF: &MF).isValid()) {
581 // Emit the type info and forward function declaration to the first MBB
582 // to ensure VReg definition dependencies are valid across all MBBs.
583 MachineIRBuilder FirstBlockBuilder;
584 FirstBlockBuilder.setMF(MF);
585 FirstBlockBuilder.setMBB(*MF.getBlockNumbered(N: 0));
586
587 SmallVector<ArrayRef<Register>, 8> VRegArgs;
588 SmallVector<SmallVector<Register, 1>, 8> ToInsert;
589 for (const Argument &Arg : CF->args()) {
590 if (MIRBuilder.getDataLayout().getTypeStoreSize(Ty: Arg.getType()).isZero())
591 continue; // Don't handle zero sized types.
592 Register Reg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
593 MRI->setRegClass(Reg, RC: &SPIRV::iIDRegClass);
594 ToInsert.push_back(Elt: {Reg});
595 VRegArgs.push_back(Elt: ToInsert.back());
596 }
597 // TODO: Reuse FunctionLoweringInfo
598 FunctionLoweringInfo FuncInfo;
599 lowerFormalArguments(MIRBuilder&: FirstBlockBuilder, F: *CF, VRegs: VRegArgs, FLI&: FuncInfo);
600 }
601
602 // Ignore the call if it's called from the internal service function
603 if (MIRBuilder.getMF()
604 .getFunction()
605 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
606 .isValid()) {
607 // insert a no-op
608 MIRBuilder.buildTrap();
609 return true;
610 }
611
612 unsigned CallOp;
613 if (Info.CB->isIndirectCall()) {
614 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
615 report_fatal_error(reason: "An indirect call is encountered but SPIR-V without "
616 "extensions does not support it",
617 gen_crash_diag: false);
618 // Set instruction operation according to SPV_INTEL_function_pointers
619 CallOp = SPIRV::OpFunctionPointerCallINTEL;
620 // Collect information about the indirect call to create correct types.
621 Register CalleeReg = Info.Callee.getReg();
622 if (CalleeReg.isValid()) {
623 SPIRVCallLowering::SPIRVIndirectCall IndirectCall;
624 IndirectCall.Callee = CalleeReg;
625 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *Info.CB);
626 IndirectCall.RetTy = OrigRetTy = FTy->getReturnType();
627 assert(FTy->getNumParams() == Info.OrigArgs.size() &&
628 "Function types mismatch");
629 for (unsigned I = 0; I != Info.OrigArgs.size(); ++I) {
630 assert(Info.OrigArgs[I].Regs.size() == 1 &&
631 "Call arg has multiple VRegs");
632 IndirectCall.ArgTys.push_back(Elt: FTy->getParamType(i: I));
633 IndirectCall.ArgRegs.push_back(Elt: Info.OrigArgs[I].Regs[0]);
634 }
635 produceIndirectPtrType(MIRBuilder, IC: IndirectCall);
636 }
637 } else {
638 // Emit a regular OpFunctionCall
639 CallOp = SPIRV::OpFunctionCall;
640 }
641
642 // Make sure there's a valid return reg, even for functions returning void.
643 if (!ResVReg.isValid())
644 ResVReg = MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
645 SPIRVTypeInst RetType = GR->assignTypeToVReg(
646 Type: OrigRetTy, VReg: ResVReg, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
647
648 // Emit the call instruction and its args.
649 auto MIB = MIRBuilder.buildInstr(Opcode: CallOp)
650 .addDef(RegNo: ResVReg)
651 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
652 .add(MO: Info.Callee);
653
654 for (const auto &Arg : Info.OrigArgs) {
655 // Currently call args should have single vregs.
656 if (Arg.Regs.size() > 1)
657 return false;
658 MIB.addUse(RegNo: Arg.Regs[0]);
659 }
660
661 if (ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
662 // Process aliasing metadata.
663 const CallBase *CI = Info.CB;
664 if (CI && CI->hasMetadata()) {
665 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alias_scope))
666 GR->buildMemAliasingOpDecorate(Reg: ResVReg, MIRBuilder,
667 Dec: SPIRV::Decoration::AliasScopeINTEL, GVarMD: MD);
668 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_noalias))
669 GR->buildMemAliasingOpDecorate(Reg: ResVReg, MIRBuilder,
670 Dec: SPIRV::Decoration::NoAliasINTEL, GVarMD: MD);
671 }
672 }
673
674 MIB.constrainAllUses(TII: MIRBuilder.getTII(), TRI: *ST->getRegisterInfo(),
675 RBI: *ST->getRegBankInfo());
676 return true;
677}
678