1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass prepares IR for legalization: it assigns SPIR-V types to registers
10// and removes intrinsics which holded these types during IR translation.
11// Also it processes constants and registers them in GR to avoid duplication.
12//
13//===----------------------------------------------------------------------===//
14
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
21#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
22#include "llvm/CodeGen/MachinePassManager.h"
23#include "llvm/IR/Analysis.h"
24#include "llvm/IR/Attributes.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/InstrTypes.h"
27#include "llvm/IR/IntrinsicsSPIRV.h"
28#include "llvm/Support/MathExtras.h"
29
30#define DEBUG_TYPE "spirv-prelegalizer"
31
32using namespace llvm;
33
34namespace {
35class SPIRVPreLegalizerLegacy : public MachineFunctionPass {
36public:
37 static char ID;
38 SPIRVPreLegalizerLegacy() : MachineFunctionPass(ID) {}
39 bool runOnMachineFunction(MachineFunction &MF) override;
40 void getAnalysisUsage(AnalysisUsage &AU) const override;
41};
42} // namespace
43
44void SPIRVPreLegalizerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
45 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
46 MachineFunctionPass::getAnalysisUsage(AU);
47}
48
49static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
50 MachineInstr *MI) {
51 GR->invalidateMachineInstr(MI);
52 MI->eraseFromParent();
53}
54
55static void
56addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
57 const SPIRVSubtarget &STI,
58 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
59 MachineRegisterInfo &MRI = MF.getRegInfo();
60 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
61 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
62 for (MachineBasicBlock &MBB : MF) {
63 for (MachineInstr &MI : MBB) {
64 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant))
65 continue;
66 ToErase.push_back(Elt: &MI);
67 Register SrcReg = MI.getOperand(i: 2).getReg();
68 auto *Const =
69 cast<Constant>(Val: cast<ConstantAsMetadata>(
70 Val: MI.getOperand(i: 3).getMetadata()->getOperand(I: 0))
71 ->getValue());
72 if (auto *GV = dyn_cast<GlobalValue>(Val: Const)) {
73 Register Reg = GR->find(V: GV, MF: &MF);
74 if (!Reg.isValid()) {
75 GR->add(V: GV, MI: MRI.getVRegDef(Reg: SrcReg));
76 GR->addGlobalObject(V: GV, MF: &MF, R: SrcReg);
77 } else
78 RegsAlreadyAddedToDT[&MI] = Reg;
79 } else {
80 Register Reg = GR->find(V: Const, MF: &MF);
81 if (!Reg.isValid()) {
82 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Val: Const)) {
83 auto *BuildVec = MRI.getVRegDef(Reg: SrcReg);
84 assert(BuildVec &&
85 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
86 GR->add(V: Const, MI: BuildVec);
87 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
88 // Ensure that OpConstantComposite reuses a constant when it's
89 // already created and available in the same machine function.
90 Constant *ElemConst = ConstVec->getElementAsConstant(i);
91 Register ElemReg = GR->find(V: ElemConst, MF: &MF);
92 if (!ElemReg.isValid())
93 GR->add(V: ElemConst,
94 MI: MRI.getVRegDef(Reg: BuildVec->getOperand(i: 1 + i).getReg()));
95 else
96 BuildVec->getOperand(i: 1 + i).setReg(ElemReg);
97 }
98 }
99 if (Const->getType()->isTargetExtTy()) {
100 // remember association so that we can restore it when assign types
101 MachineInstr *SrcMI = MRI.getVRegDef(Reg: SrcReg);
102 if (SrcMI)
103 GR->add(V: Const, MI: SrcMI);
104 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
105 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
106 TargetExtConstTypes[SrcMI] = Const->getType();
107 if (Const->isNullValue()) {
108 MachineBasicBlock &DepMBB = MF.front();
109 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());
110 SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType(
111 Type: Const->getType(), MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
112 EmitIR: true);
113 assert(SrcMI && "Expected source instruction to be valid");
114 SrcMI->setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
115 SrcMI->addOperand(Op: MachineOperand::CreateReg(
116 Reg: GR->getSPIRVTypeID(SpirvType: ExtType), isDef: false));
117 }
118 }
119 } else {
120 RegsAlreadyAddedToDT[&MI] = Reg;
121 // This MI is unused and will be removed. If the MI uses
122 // const_composite, it will be unused and should be removed too.
123 assert(MI.getOperand(2).isReg() && "Reg operand is expected");
124 MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
125 if (SrcMI && isSpvIntrinsic(MI: *SrcMI, IntrinsicID: Intrinsic::spv_const_composite))
126 ToEraseComposites.push_back(Elt: SrcMI);
127 }
128 }
129 }
130 }
131 for (MachineInstr *MI : ToErase) {
132 Register Reg = MI->getOperand(i: 2).getReg();
133 auto It = RegsAlreadyAddedToDT.find(Val: MI);
134 if (It != RegsAlreadyAddedToDT.end())
135 Reg = It->second;
136 auto *RC = MRI.getRegClassOrNull(Reg: MI->getOperand(i: 0).getReg());
137 if (!MRI.getRegClassOrNull(Reg) && RC)
138 MRI.setRegClass(Reg, RC);
139 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: Reg);
140 invalidateAndEraseMI(GR, MI);
141 }
142 for (MachineInstr *MI : ToEraseComposites)
143 invalidateAndEraseMI(GR, MI);
144}
145
146static void foldConstantsIntoIntrinsics(MachineFunction &MF,
147 SPIRVGlobalRegistry *GR,
148 MachineIRBuilder MIB) {
149 SmallVector<MachineInstr *, 64> ToErase;
150 for (MachineBasicBlock &MBB : MF) {
151 for (MachineInstr &MI : MBB) {
152 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_name))
153 continue;
154 const MDNode *MD = MI.getOperand(i: 2).getMetadata();
155 StringRef ValueName = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
156 if (ValueName.size() > 0) {
157 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
158 buildOpName(Target: MI.getOperand(i: 1).getReg(), Name: ValueName, MIRBuilder&: MIB);
159 }
160 ToErase.push_back(Elt: &MI);
161 }
162 for (MachineInstr *MI : ToErase)
163 invalidateAndEraseMI(GR, MI);
164 ToErase.clear();
165 }
166}
167
168static MachineInstr *findAssignTypeInstr(Register Reg,
169 MachineRegisterInfo *MRI) {
170 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(RegNo: Reg),
171 IE = MRI->use_instr_end();
172 I != IE; ++I) {
173 MachineInstr *UseMI = &*I;
174 if ((isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_ptr_type) ||
175 isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_type)) &&
176 UseMI->getOperand(i: 1).getReg() == Reg)
177 return UseMI;
178 }
179 return nullptr;
180}
181
182static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
183 Register ResVReg, Register OpReg) {
184 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResVReg);
185 SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg);
186 assert(ResType && OpType && "Operand types are expected");
187 if (!GR->isBitcastCompatible(Type1: ResType, Type2: OpType))
188 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
189 MachineRegisterInfo *MRI = MIB.getMRI();
190 if (!MRI->getRegClassOrNull(Reg: ResVReg))
191 MRI->setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: ResType));
192 if (ResType == OpType)
193 MIB.buildInstr(Opcode: TargetOpcode::COPY).addDef(RegNo: ResVReg).addUse(RegNo: OpReg);
194 else
195 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
196 .addDef(RegNo: ResVReg)
197 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResType))
198 .addUse(RegNo: OpReg);
199}
200
201// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.
202// The verifier checks if the source and destination LLTs of a G_BITCAST are
203// different, but this check is too strict for SPIR-V's typed pointers, which
204// may have the same LLT but different SPIRV type (e.g. pointers to different
205// pointee types). By lowering to OpBitcast here, we bypass the verifier's
206// check. See discussion in https://github.com/llvm/llvm-project/pull/110270
207// for more context.
208//
209// We also handle the llvm.spv.bitcast intrinsic here. If the source and
210// destination SPIR-V types are the same, we lower it to a COPY to enable
211// further optimizations like copy propagation.
212static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
213 MachineIRBuilder MIB) {
214 SmallVector<MachineInstr *, 16> ToErase;
215 for (MachineBasicBlock &MBB : MF) {
216 for (MachineInstr &MI : MBB) {
217 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_bitcast)) {
218 Register DstReg = MI.getOperand(i: 0).getReg();
219 Register SrcReg = MI.getOperand(i: 2).getReg();
220 SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(VReg: DstReg);
221 assert(
222 DstType &&
223 "Expected destination SPIR-V type to have been assigned already.");
224 SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(VReg: SrcReg);
225 assert(SrcType &&
226 "Expected source SPIR-V type to have been assigned already.");
227 if (DstType == SrcType) {
228 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
229 MIB.buildCopy(Res: DstReg, Op: SrcReg);
230 ToErase.push_back(Elt: &MI);
231 continue;
232 }
233 }
234
235 if (MI.getOpcode() != TargetOpcode::G_BITCAST)
236 continue;
237
238 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
239 buildOpBitcast(GR, MIB, ResVReg: MI.getOperand(i: 0).getReg(),
240 OpReg: MI.getOperand(i: 1).getReg());
241 ToErase.push_back(Elt: &MI);
242 }
243 }
244 for (MachineInstr *MI : ToErase)
245 invalidateAndEraseMI(GR, MI);
246}
247
248static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
249 MachineIRBuilder MIB) {
250 // Get access to information about available extensions
251 const SPIRVSubtarget *ST =
252 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
253 SmallVector<MachineInstr *, 10> ToErase;
254 for (MachineBasicBlock &MBB : MF) {
255 for (MachineInstr &MI : MBB) {
256 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_ptrcast))
257 continue;
258 assert(MI.getOperand(2).isReg());
259 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
260 ToErase.push_back(Elt: &MI);
261 Register Def = MI.getOperand(i: 0).getReg();
262 Register Source = MI.getOperand(i: 2).getReg();
263 Type *ElemTy = getMDOperandAsType(N: MI.getOperand(i: 3).getMetadata(), I: 0);
264 auto SC =
265 isa<FunctionType>(Val: ElemTy) &&
266 ST->canUseExtension(
267 E: SPIRV::Extension::SPV_INTEL_function_pointers)
268 ? SPIRV::StorageClass::CodeSectionINTEL
269 : addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 4).getImm(), STI: *ST);
270 SPIRVTypeInst AssignedPtrType =
271 GR->getOrCreateSPIRVPointerType(BaseType: ElemTy, I&: MI, SC);
272
273 // If the ptrcast would be redundant, replace all uses with the source
274 // register.
275 MachineRegisterInfo *MRI = MIB.getMRI();
276 // For untyped pointers the SPIR-V pointer type does not encode the
277 // pointee, so two pointers with different element types share the same
278 // pointer type. The element type still matters because it selects the
279 // Base Type operand of OpUntyped*AccessChainKHR. Treat the cast as
280 // redundant only when the source already carries the same element type.
281 // Otherwise keep a distinct register so the element type is preserved.
282 bool Redundant =
283 AssignedPtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR
284 ? GR->getUntypedPtrElementType(Reg: Source) ==
285 GR->getOrCreateSPIRVType(Type: ElemTy, MIRBuilder&: MIB,
286 AQ: SPIRV::AccessQualifier::ReadWrite,
287 /*EmitIR=*/true)
288 : GR->getSPIRVTypeForVReg(VReg: Source) == AssignedPtrType;
289 if (Redundant) {
290 // Erase Def's assign type instruction if we are going to replace Def.
291 if (MachineInstr *AssignMI = findAssignTypeInstr(Reg: Def, MRI))
292 ToErase.push_back(Elt: AssignMI);
293 MRI->replaceRegWith(FromReg: Def, ToReg: Source);
294 } else {
295 if (!GR->getSPIRVTypeForVReg(VReg: Def, MF: &MF))
296 GR->assignSPIRVTypeToVReg(Type: AssignedPtrType, VReg: Def, MF);
297 MIB.buildBitcast(Dst: Def, Src: Source);
298 }
299 }
300 }
301 for (MachineInstr *MI : ToErase)
302 invalidateAndEraseMI(GR, MI);
303}
304
305// Translating GV, IRTranslator sometimes generates following IR:
306// %1 = G_GLOBAL_VALUE
307// %2 = COPY %1
308// %3 = G_ADDRSPACE_CAST %2
309//
310// or
311//
312// %1 = G_ZEXT %2
313// G_MEMCPY ... %2 ...
314//
315// New registers have no SPIRV type and no register class info.
316//
317// Set SPIRV type for GV, propagate it from GV to other instructions,
318// also set register classes.
319static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
320 SPIRVGlobalRegistry *GR,
321 MachineRegisterInfo &MRI,
322 MachineIRBuilder &MIB) {
323 SPIRVTypeInst SpvType = nullptr;
324 assert(MI && "Machine instr is expected");
325 if (MI->getOperand(i: 0).isReg()) {
326 Register Reg = MI->getOperand(i: 0).getReg();
327 SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
328 if (!SpvType) {
329 switch (MI->getOpcode()) {
330 case TargetOpcode::G_FCONSTANT:
331 case TargetOpcode::G_CONSTANT: {
332 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
333 Type *Ty = MI->getOperand(i: 1).getCImm()->getType();
334 SpvType = GR->getOrCreateSPIRVType(
335 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
336 break;
337 }
338 case TargetOpcode::G_GLOBAL_VALUE: {
339 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
340 const GlobalValue *Global = MI->getOperand(i: 1).getGlobal();
341 Type *ElementTy = toTypedPointer(Ty: GR->getDeducedGlobalValueType(Global));
342 unsigned AddrSpace = Global->getType()->getAddressSpace();
343 // Function pointers use CodeSectionINTEL storage class in SPIR-V when
344 // the SPV_INTEL_function_pointers extension is enabled.
345 const SPIRVSubtarget &ST = MIB.getMF().getSubtarget<SPIRVSubtarget>();
346 if (isa<Function>(Val: Global) &&
347 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
348 AddrSpace =
349 storageClassToAddressSpace(SC: SPIRV::StorageClass::CodeSectionINTEL);
350 auto *Ty = TypedPointerType::get(ElementType: ElementTy, AddressSpace: AddrSpace);
351 SpvType = GR->getOrCreateSPIRVType(
352 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
353 break;
354 }
355 case TargetOpcode::G_ANYEXT:
356 case TargetOpcode::G_SEXT:
357 case TargetOpcode::G_ZEXT: {
358 if (MI->getOperand(i: 1).isReg()) {
359 if (MachineInstr *DefInstr =
360 MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
361 if (SPIRVTypeInst Def =
362 propagateSPIRVType(MI: DefInstr, GR, MRI, MIB)) {
363 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Type: Def);
364 unsigned ExpectedBW =
365 std::max(a: MRI.getType(Reg).getScalarSizeInBits(), b: CurrentBW);
366 unsigned NumElements = GR->getScalarOrVectorComponentCount(Type: Def);
367 SpvType = GR->getOrCreateSPIRVIntegerType(BitWidth: ExpectedBW, MIRBuilder&: MIB);
368 if (NumElements > 1)
369 SpvType = GR->getOrCreateSPIRVVectorType(BaseType: SpvType, NumElements,
370 MIRBuilder&: MIB, EmitIR: true);
371 }
372 }
373 }
374 break;
375 }
376 case TargetOpcode::G_PTRTOINT:
377 SpvType = GR->getOrCreateSPIRVIntegerType(
378 BitWidth: MRI.getType(Reg).getScalarSizeInBits(), MIRBuilder&: MIB);
379 break;
380 case TargetOpcode::G_TRUNC:
381 case TargetOpcode::G_ADDRSPACE_CAST:
382 case TargetOpcode::G_PTR_ADD:
383 case TargetOpcode::COPY: {
384 MachineOperand &Op = MI->getOperand(i: 1);
385 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Reg: Op.getReg()) : nullptr;
386 if (Def)
387 SpvType = propagateSPIRVType(MI: Def, GR, MRI, MIB);
388 break;
389 }
390 default:
391 break;
392 }
393 if (SpvType) {
394 // check if the address space needs correction
395 LLT RegType = MRI.getType(Reg);
396 if (SpvType.isPointer() && RegType.isPointer() &&
397 storageClassToAddressSpace(SC: GR->getPointerStorageClass(Type: SpvType)) !=
398 RegType.getAddressSpace()) {
399 // Don't correct CodeSectionINTEL back to Function for function
400 // pointer G_GLOBAL_VALUE - the LLVM register has address space 0
401 // but the SPIR-V type was intentionally set to CodeSectionINTEL.
402 bool SkipCorrection =
403 MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
404 GR->getPointerStorageClass(Type: SpvType) ==
405 SPIRV::StorageClass::CodeSectionINTEL;
406 if (!SkipCorrection) {
407 const SPIRVSubtarget &ST =
408 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
409 auto TSC =
410 addressSpaceToStorageClass(AddrSpace: RegType.getAddressSpace(), STI: ST);
411 SpvType = GR->changePointerStorageClass(PtrType: SpvType, SC: TSC, I&: *MI);
412 }
413 }
414 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
415 }
416 if (!MRI.getRegClassOrNull(Reg))
417 MRI.setRegClass(Reg, RC: SpvType ? GR->getRegClass(SpvType)
418 : &SPIRV::iIDRegClass);
419 }
420 }
421 return SpvType;
422}
423
424// To support current approach and limitations wrt. bit width here we widen a
425// scalar register with a bit width greater than 1 to valid sizes and cap it to
426// 128 width.
427static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {
428 if (BitWidth == 1)
429 return 1; // No need to widen 1-bit values
430 return std::min(a: std::max<unsigned>(a: PowerOf2Ceil(A: BitWidth), b: 8u), b: 128u);
431}
432
433static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {
434 LLT RegType = MRI.getType(Reg);
435 if (!RegType.isScalar())
436 return;
437 unsigned CurrentWidth = RegType.getScalarSizeInBits();
438 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
439 if (NewWidth != CurrentWidth)
440 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: NewWidth));
441}
442
443static void widenCImmType(MachineOperand &MOP) {
444 const ConstantInt *CImmVal = MOP.getCImm();
445 unsigned CurrentWidth = CImmVal->getBitWidth();
446 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
447 if (NewWidth != CurrentWidth) {
448 // Replace the immediate value with the widened version
449 MOP.setCImm(ConstantInt::get(Context&: CImmVal->getType()->getContext(),
450 V: CImmVal->getValue().zextOrTrunc(width: NewWidth)));
451 }
452}
453
454static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
455 MachineBasicBlock &MBB = *Def->getParent();
456 MachineBasicBlock::iterator DefIt =
457 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();
458 // Skip all the PHI and debug instructions.
459 while (DefIt != MBB.end() &&
460 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))
461 DefIt = std::next(x: DefIt);
462 MIB.setInsertPt(MBB, II: DefIt);
463}
464
465namespace llvm {
466void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType,
467 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
468 MachineRegisterInfo &MRI) {
469 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
470 MachineInstr *Def = MRI.getVRegDef(Reg);
471 setInsertPtAfterDef(MIB, Def);
472 if (!SpvType)
473 SpvType = GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder&: MIB,
474 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
475 if (!MRI.getRegClassOrNull(Reg))
476 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType));
477 if (!MRI.getType(Reg).isValid())
478 MRI.setType(VReg: Reg, Ty: GR->getRegType(SpvType));
479 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
480}
481
482void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
483 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
484 SPIRVTypeInst KnownResType) {
485 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
486 for (auto &Op : MI.operands()) {
487 if (!Op.isReg() || Op.isDef())
488 continue;
489 Register OpReg = Op.getReg();
490 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OpReg);
491 if (!SpvType && KnownResType) {
492 SpvType = KnownResType;
493 GR->assignSPIRVTypeToVReg(Type: KnownResType, VReg: OpReg, MF: *MI.getMF());
494 }
495 assert(SpvType);
496 if (!MRI.getRegClassOrNull(Reg: OpReg))
497 MRI.setRegClass(Reg: OpReg, RC: GR->getRegClass(SpvType));
498 if (!MRI.getType(Reg: OpReg).isValid())
499 MRI.setType(VReg: OpReg, Ty: GR->getRegType(SpvType));
500 }
501}
502} // namespace llvm
503
504// Sign-sensitive integer ops: their result depends on the value of the input
505// sign bit at position (width-1). On sub-pow2 widths the general widening
506// loop is a pure LLT relabel, which leaves the sign bit at the *original*
507// position instead of the widened MSB. These ops therefore need an explicit
508// G_SEXT_INREG on each value operand to move the sign bit up.
509//
510// Signed-vs-unsigned G_ICMP is distinguished by its predicate operand.
511//
512// TODO: follow-up PRs will add the remaining sign-sensitive opcodes
513// (e.g. G_SMIN/G_SMAX, G_SADDSAT/G_SSUBSAT, signed overflow ops).
514static bool isSignSensitiveOp(const MachineInstr &MI) {
515 switch (MI.getOpcode()) {
516 case TargetOpcode::G_ASHR:
517 case TargetOpcode::G_SDIV:
518 case TargetOpcode::G_SREM:
519 return true;
520 case TargetOpcode::G_ICMP:
521 return CmpInst::isSigned(
522 Pred: static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate()));
523 default:
524 return false;
525 }
526}
527
528struct SignSensitiveWideningInfo {
529 // Width before widening of each value-operand vreg (one entry per vreg).
530 DenseMap<Register, unsigned> OrigWidth;
531 // Ops whose value operand(s) need replacing, ordered for reproducible vreg
532 // numbering.
533 SmallVector<MachineInstr *> Worklist;
534};
535
536// Collect sign-sensitive ops with narrow scalar value operands and their
537// pre-widening widths, before later passes retype those vregs to pow2 LLTs
538// and the original width is no longer recoverable.
539static SignSensitiveWideningInfo
540recordSignSensitiveOperandWidths(MachineFunction &MF,
541 MachineRegisterInfo &MRI) {
542 SignSensitiveWideningInfo Info;
543 auto RecordIfNarrow = [&](Register Reg) {
544 LLT Ty = MRI.getType(Reg);
545 if (!Ty.isScalar())
546 return false;
547 unsigned W = Ty.getScalarSizeInBits();
548 if (widenBitWidthToNextPow2(BitWidth: W) == W)
549 return false;
550 Info.OrigWidth.try_emplace(Key: Reg, Args&: W);
551 return true;
552 };
553 for (MachineBasicBlock &MBB : MF) {
554 for (MachineInstr &MI : MBB) {
555 if (!isSignSensitiveOp(MI))
556 continue;
557 // Value operands are the trailing two, past any def or predicate.
558 unsigned N = MI.getNumOperands();
559 const MachineOperand &LHS = MI.getOperand(i: N - 2);
560 const MachineOperand &RHS = MI.getOperand(i: N - 1);
561 // Sign-sensitive opcodes carry register operands only.
562 assert(LHS.isReg() && RHS.isReg());
563 bool NeedsRewrite = RecordIfNarrow(LHS.getReg());
564 NeedsRewrite = RecordIfNarrow(RHS.getReg()) || NeedsRewrite;
565 if (NeedsRewrite)
566 Info.Worklist.push_back(Elt: &MI);
567 }
568 }
569 return Info;
570}
571
572// For every recorded sign-sensitive op, insert G_SEXT_INREG on each value
573// operand whose original width was narrower than the widened pow2 width and
574// retype the operand's vreg LLT in place to the widened width.
575//
576// Info must have been populated by recordSignSensitiveOperandWidths before
577// other passes retyped the vregs; otherwise the narrow widths needed here
578// are lost.
579//
580// TODO: handle vector operands.
581static void widenSignSensitiveOps(MachineFunction &MF, SPIRVGlobalRegistry *GR,
582 MachineIRBuilder &MIB,
583 MachineRegisterInfo &MRI,
584 const SignSensitiveWideningInfo &Info) {
585 // Emit G_SEXT_INREG from Reg's recorded narrow width; retypes Reg to the
586 // widened width and returns the sign-extended vreg.
587 auto SignExtendReg = [&](Register Reg, unsigned OldW,
588 MachineInstr &MI) -> Register {
589 unsigned NewW = widenBitWidthToNextPow2(BitWidth: OldW);
590 LLT NewLLT = LLT::scalar(SizeInBits: NewW);
591 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
592 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewW, MIRBuilder&: MIB);
593 Register SExted = MRI.createGenericVirtualRegister(Ty: NewLLT);
594 GR->assignSPIRVTypeToVReg(Type: SpvTy, VReg: SExted, MF);
595 MRI.setRegClass(Reg: SExted, RC: GR->getRegClass(SpvType: SpvTy));
596 MRI.setType(VReg: Reg, Ty: NewLLT);
597 MIB.buildSExtInReg(Res: SExted, Op: Reg, ImmOp: OldW);
598 return SExted;
599 };
600
601 // TODO: when the same narrow vreg feeds multiple sign-sensitive ops (e.g.
602 // sdiv %x, %y and srem %x, %y), emit one shared G_SEXT_INREG instead of one
603 // per use.
604 for (MachineInstr *MI : Info.Worklist) {
605 unsigned N = MI->getNumOperands();
606 MachineOperand &LHS = MI->getOperand(i: N - 2);
607 MachineOperand &RHS = MI->getOperand(i: N - 1);
608 Register LHSReg = LHS.getReg();
609 Register RHSReg = RHS.getReg();
610 if (auto It = Info.OrigWidth.find(Val: LHSReg); It != Info.OrigWidth.end())
611 LHS.setReg(SignExtendReg(LHSReg, It->second, *MI));
612 // Same vreg on both sides (e.g. G_ICMP slt %x, %x): reuse the sext just
613 // emitted for LHS instead of emitting a second one.
614 if (RHSReg == LHSReg) {
615 RHS.setReg(LHS.getReg());
616 continue;
617 }
618 if (auto It = Info.OrigWidth.find(Val: RHSReg); It != Info.OrigWidth.end())
619 RHS.setReg(SignExtendReg(RHSReg, It->second, *MI));
620 }
621}
622
623static void
624generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
625 MachineIRBuilder MIB,
626 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
627 // Get access to information about available extensions
628 const SPIRVSubtarget *ST =
629 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
630
631 MachineRegisterInfo &MRI = MF.getRegInfo();
632 SmallVector<MachineInstr *, 10> ToErase;
633 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
634
635 bool IsExtendedInts =
636 ST->canUseExtension(
637 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
638 ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
639 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
640
641 if (!IsExtendedInts) {
642 // Without arbitrary precision integer extensions, SPIR-V only supports
643 // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40)
644 // must be widened to the next power of two.
645 //
646 // Record the original widths of sign-sensitive operands before either
647 // the G_TRUNC handling or the general widening loop retypes vregs, then
648 // rewrite those ops after G_TRUNC processing using the recorded widths.
649 SignSensitiveWideningInfo SignSensitiveInfo =
650 recordSignSensitiveOperandWidths(MF, MRI);
651
652 // G_TRUNC requires special handling because its semantics depend on the
653 // original destination width. For example:
654 // %dst:s24 = G_TRUNC %src:s64
655 // After widening s24 to s32, we cannot simply do:
656 // %dst:s32 = G_TRUNC %src:s64
657 // because this would keep 32 bits instead of 24. Instead, we insert a
658 // G_AND to mask the value to the original width:
659 // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask
660 // %masked:s64 = G_AND %src:s64, %mask
661 // %dst:s32 = G_TRUNC %masked:s64
662 // If src and dst widen to the same size, G_TRUNC is replaced entirely:
663 // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask
664 // %dst:s64 = G_AND %src:s64, %mask
665 SmallVector<MachineInstr *, 8> TruncToRemove;
666 for (MachineBasicBlock &MBB : MF) {
667 for (MachineInstr &MI : MBB) {
668 unsigned MIOp = MI.getOpcode();
669 if (MIOp != TargetOpcode::G_TRUNC)
670 continue;
671 assert(MI.getNumOperands() == 2);
672 assert(MI.getOperand(0).isReg());
673 assert(MI.getOperand(1).isReg());
674
675 Register DstReg = MI.getOperand(i: 0).getReg();
676 Register SrcReg = MI.getOperand(i: 1).getReg();
677
678 LLT DstTy = MRI.getType(Reg: DstReg);
679 LLT SrcTy = MRI.getType(Reg: SrcReg);
680 assert((DstTy.isScalar() || DstTy.isVector()) &&
681 (SrcTy.isScalar() || SrcTy.isVector()) &&
682 "Expected scalar or vector G_TRUNC types");
683 assert(DstTy.isVector() == SrcTy.isVector() &&
684 "Expected matching scalar/vector G_TRUNC types");
685 assert((!DstTy.isVector() ||
686 DstTy.getElementCount() == SrcTy.getElementCount()) &&
687 "Expected equal vector element counts");
688
689 unsigned OriginalDstWidth = DstTy.getScalarSizeInBits();
690 unsigned OriginalSrcWidth = SrcTy.getScalarSizeInBits();
691
692 unsigned NewDstWidth = widenBitWidthToNextPow2(BitWidth: OriginalDstWidth);
693 unsigned NewSrcWidth = widenBitWidthToNextPow2(BitWidth: OriginalSrcWidth);
694 LLT NewDstTy = DstTy.changeElementSize(NewEltSize: NewDstWidth);
695 LLT NewSrcTy = SrcTy.changeElementSize(NewEltSize: NewSrcWidth);
696
697 // No Dst width change means no truncation semantics change, but the
698 // source still needs a legal type.
699 if (OriginalDstWidth == NewDstWidth) {
700 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
701 continue;
702 }
703
704 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
705 MRI.setType(VReg: DstReg, Ty: NewDstTy);
706
707 MIB.setInsertPt(MBB, II: MI.getIterator());
708 APInt Mask = APInt::getLowBitsSet(numBits: NewSrcWidth, loBitsSet: OriginalDstWidth);
709 MachineInstrBuilder MaskReg =
710 DstTy.isVector()
711 ? MIB.buildBuildVectorConstant(
712 Res: NewSrcTy,
713 Ops: SmallVector<APInt, 4>(DstTy.getNumElements(), Mask))
714 : MIB.buildConstant(Res: NewSrcTy, Val: Mask);
715 Register MaskedReg = MRI.createGenericVirtualRegister(Ty: NewSrcTy);
716 MIB.buildAnd(Dst: MaskedReg, Src0: SrcReg, Src1: MaskReg);
717
718 if (NewSrcWidth == NewDstWidth) {
719 // Rekey OrigWidth from DstReg to MaskedReg so widenSignSensitiveOps
720 // still sees the narrow original width after replaceRegWith.
721 if (auto It = SignSensitiveInfo.OrigWidth.find(Val: DstReg);
722 It != SignSensitiveInfo.OrigWidth.end()) {
723 unsigned W = It->second;
724 SignSensitiveInfo.OrigWidth.erase(I: It);
725 SignSensitiveInfo.OrigWidth.try_emplace(Key: MaskedReg, Args&: W);
726 }
727 MRI.replaceRegWith(FromReg: DstReg, ToReg: MaskedReg);
728 TruncToRemove.push_back(Elt: &MI);
729 } else {
730 MI.getOperand(i: 1).setReg(MaskedReg);
731 }
732 }
733 }
734 for (MachineInstr *MI : TruncToRemove)
735 MI->eraseFromParent();
736
737 widenSignSensitiveOps(MF, GR, MIB, MRI, Info: SignSensitiveInfo);
738 }
739
740 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
741 if (MBB->empty())
742 continue;
743
744 bool ReachedBegin = false;
745 for (auto MII = std::prev(x: MBB->end()), Begin = MBB->begin();
746 !ReachedBegin;) {
747 MachineInstr &MI = *MII;
748 unsigned MIOp = MI.getOpcode();
749
750 if (!IsExtendedInts) {
751 // validate bit width of scalar registers and constant immediates
752 for (auto &MOP : MI.operands()) {
753 if (MOP.isReg())
754 widenScalarType(Reg: MOP.getReg(), MRI);
755 else if (MOP.isCImm())
756 widenCImmType(MOP);
757 }
758 }
759
760 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) {
761 Register Reg = MI.getOperand(i: 1).getReg();
762 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
763 Type *ElementTy = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
764 auto SC = addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 3).getImm(), STI: *ST);
765 if (SC == SPIRV::StorageClass::Function &&
766 isa<FunctionType>(Val: ElementTy) &&
767 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
768 SC = SPIRV::StorageClass::CodeSectionINTEL;
769 SPIRVTypeInst AssignedPtrType =
770 GR->getOrCreateSPIRVPointerType(BaseType: ElementTy, I&: MI, SC);
771
772 // For untyped pointers, store the element type for later use.
773 if (ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers) &&
774 !ST->isShader()) {
775 SPIRVTypeInst ElemSpvType = GR->getOrCreateSPIRVType(
776 Type: ElementTy, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
777 /*EmitIR=*/true);
778 GR->setUntypedPtrElementType(Reg, ElemType: ElemSpvType);
779 }
780
781 // The intrinsic also carries vector-of-pointer values produced by
782 // scalarized vector GEPs; wrap the pointer in OpTypeVector to match
783 // the vreg's LLT.
784 LLT RegTy = MRI.getType(Reg);
785 if (RegTy.isValid() && RegTy.isVector())
786 AssignedPtrType = GR->getOrCreateSPIRVVectorType(
787 BaseType: AssignedPtrType, NumElements: RegTy.getNumElements(), MIRBuilder&: MIB,
788 /*EmitIR=*/true);
789 MachineInstr *Def = MRI.getVRegDef(Reg);
790 assert(Def && "Expecting an instruction that defines the register");
791 // G_GLOBAL_VALUE already has type info.
792 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
793 updateRegType(Reg, Ty: nullptr, SpvType: AssignedPtrType, GR, MIB,
794 MRI&: MF.getRegInfo());
795 ToErase.push_back(Elt: &MI);
796 } else if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_type)) {
797 Register Reg = MI.getOperand(i: 1).getReg();
798 Type *Ty = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
799 MachineInstr *Def = MRI.getVRegDef(Reg);
800 assert(Def && "Expecting an instruction that defines the register");
801 // G_GLOBAL_VALUE already has type info.
802 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
803 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
804 if (Def->getOpcode() == TargetOpcode::COPY && isVector1(Ty))
805 updateRegType(Reg: passCopy(Def, MRI: &MF.getRegInfo())->getOperand(i: 0).getReg(),
806 Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
807 ToErase.push_back(Elt: &MI);
808 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
809 MachineInstr *MdMI = MI.getPrevNode();
810 if (MdMI && isSpvIntrinsic(MI: *MdMI, IntrinsicID: Intrinsic::spv_value_md)) {
811 // It's an internal service info from before IRTranslator passes.
812 MachineInstr *Def = getVRegDef(MRI, Reg: MI.getOperand(i: 0).getReg());
813 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
814 if (getVRegDef(MRI, Reg: MI.getOperand(i: I).getReg()) != Def)
815 Def = nullptr;
816 if (Def) {
817 const MDNode *MD = MdMI->getOperand(i: 1).getMetadata();
818 StringRef ValueName =
819 cast<MDString>(Val: MD->getOperand(I: 1))->getString();
820 const MDNode *TypeMD = cast<MDNode>(Val: MD->getOperand(I: 0));
821 Type *ValueTy = getMDOperandAsType(N: TypeMD, I: 0);
822 GR->addValueAttrs(Key: Def, Val: std::make_pair(x&: ValueTy, y: ValueName.str()));
823 }
824 ToErase.push_back(Elt: MdMI);
825 }
826 ToErase.push_back(Elt: &MI);
827 } else if (MIOp == TargetOpcode::G_CONSTANT ||
828 MIOp == TargetOpcode::G_FCONSTANT ||
829 MIOp == TargetOpcode::G_BUILD_VECTOR) {
830 // %rc = G_CONSTANT ty Val
831 // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
832 Register Reg = MI.getOperand(i: 0).getReg();
833 bool NeedAssignType = !GR->getSPIRVTypeForVReg(VReg: Reg);
834 Type *Ty = nullptr;
835 if (MIOp == TargetOpcode::G_CONSTANT) {
836 auto TargetExtIt = TargetExtConstTypes.find(Val: &MI);
837 Ty = TargetExtIt == TargetExtConstTypes.end()
838 ? MI.getOperand(i: 1).getCImm()->getType()
839 : TargetExtIt->second;
840 const ConstantInt *OpCI = MI.getOperand(i: 1).getCImm();
841 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =
842 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe
843 // at this point not i64/i32 constant but null pointer in the
844 // corresponding address space of RegType.getAddressSpace(). This may
845 // help to successfully validate the case when a OpConstantComposite's
846 // constituent has type that does not match Result Type of
847 // OpConstantComposite (see, for example,
848 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).
849 Register PrimaryReg = GR->find(V: OpCI, MF: &MF);
850 if (!PrimaryReg.isValid()) {
851 GR->add(V: OpCI, MI: &MI);
852 } else if (PrimaryReg != Reg &&
853 MRI.getType(Reg) == MRI.getType(Reg: PrimaryReg)) {
854 auto *RCReg = MRI.getRegClassOrNull(Reg);
855 auto *RCPrimary = MRI.getRegClassOrNull(Reg: PrimaryReg);
856 if (!RCReg || RCPrimary == RCReg) {
857 RegsAlreadyAddedToDT[&MI] = PrimaryReg;
858 ToErase.push_back(Elt: &MI);
859 NeedAssignType = false;
860 }
861 }
862 } else if (MIOp == TargetOpcode::G_FCONSTANT) {
863 Ty = MI.getOperand(i: 1).getFPImm()->getType();
864 } else {
865 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
866 Type *ElemTy = nullptr;
867 MachineInstr *ElemMI = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
868 assert(ElemMI);
869
870 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
871 ElemTy = ElemMI->getOperand(i: 1).getCImm()->getType();
872 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
873 ElemTy = ElemMI->getOperand(i: 1).getFPImm()->getType();
874 } else {
875 if (SPIRVTypeInst ElemSpvType =
876 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg(), MF: &MF))
877 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(Ty: ElemSpvType));
878 }
879 if (ElemTy)
880 Ty = VectorType::get(
881 ElementType: ElemTy, NumElements: MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
882 Scalable: false);
883 else
884 NeedAssignType = false;
885 }
886 if (NeedAssignType)
887 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI);
888 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
889 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
890 }
891
892 if (MII == Begin)
893 ReachedBegin = true;
894 else
895 --MII;
896 }
897 }
898 for (MachineInstr *MI : ToErase) {
899 auto It = RegsAlreadyAddedToDT.find(Val: MI);
900 if (It != RegsAlreadyAddedToDT.end())
901 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: It->second);
902 invalidateAndEraseMI(GR, MI);
903 }
904
905 // Address the case when IRTranslator introduces instructions with new
906 // registers without associated SPIRV type.
907 for (MachineBasicBlock &MBB : MF) {
908 for (MachineInstr &MI : MBB) {
909 switch (MI.getOpcode()) {
910 case TargetOpcode::G_TRUNC:
911 case TargetOpcode::G_ANYEXT:
912 case TargetOpcode::G_SEXT:
913 case TargetOpcode::G_ZEXT:
914 case TargetOpcode::G_PTRTOINT:
915 case TargetOpcode::COPY:
916 case TargetOpcode::G_ADDRSPACE_CAST:
917 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
918 break;
919 }
920 }
921 }
922}
923
924static void processInstrsWithTypeFolding(MachineFunction &MF,
925 SPIRVGlobalRegistry *GR,
926 MachineIRBuilder MIB) {
927 MachineRegisterInfo &MRI = MF.getRegInfo();
928 for (MachineBasicBlock &MBB : MF)
929 for (MachineInstr &MI : MBB)
930 if (isTypeFoldingSupported(Opcode: MI.getOpcode()))
931 processInstr(MI, MIB, MRI, GR, KnownResType: nullptr);
932}
933
934static Register
935collectInlineAsmInstrOperands(MachineInstr *MI,
936 SmallVector<unsigned, 4> *Ops = nullptr) {
937 Register DefReg;
938 unsigned StartOp = InlineAsm::MIOp_FirstOperand,
939 AsmDescOp = InlineAsm::MIOp_FirstOperand;
940 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
941 ++Idx) {
942 const MachineOperand &MO = MI->getOperand(i: Idx);
943 if (MO.isMetadata())
944 continue;
945 if (Idx == AsmDescOp && MO.isImm()) {
946 // compute the index of the next operand descriptor
947 const InlineAsm::Flag F(MO.getImm());
948 AsmDescOp += 1 + F.getNumOperandRegisters();
949 continue;
950 }
951 if (MO.isReg() && MO.isDef()) {
952 if (!Ops)
953 return MO.getReg();
954 DefReg = MO.getReg();
955 } else if (Ops) {
956 Ops->push_back(Elt: Idx);
957 }
958 }
959 return DefReg;
960}
961
962static void
963insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
964 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
965 const SmallVector<MachineInstr *> &ToProcess) {
966 MachineRegisterInfo &MRI = MF.getRegInfo();
967 Register AsmTargetReg;
968 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
969 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
970 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
971 MIRBuilder.setInsertPt(MBB&: *I2->getParent(), II: *I2);
972
973 if (!AsmTargetReg.isValid()) {
974 // define vendor specific assembly target or dialect
975 AsmTargetReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
976 MRI.setRegClass(Reg: AsmTargetReg, RC: &SPIRV::iIDRegClass);
977 auto AsmTargetMIB =
978 MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmTargetINTEL).addDef(RegNo: AsmTargetReg);
979 addStringImm(Str: ST.getTargetTripleAsStr(), MIB&: AsmTargetMIB);
980 GR->add(Obj: AsmTargetMIB.getInstr(), MI: AsmTargetMIB);
981 }
982
983 // create types
984 const MDNode *IAMD = I1->getOperand(i: 1).getMetadata();
985 FunctionType *FTy = cast<FunctionType>(Val: getMDOperandAsType(N: IAMD, I: 0));
986 SmallVector<SPIRVTypeInst, 4> ArgTypes;
987 for (const auto &ArgTy : FTy->params())
988 ArgTypes.push_back(Elt: GR->getOrCreateSPIRVType(
989 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true));
990 SPIRVTypeInst RetType =
991 GR->getOrCreateSPIRVType(Type: FTy->getReturnType(), MIRBuilder,
992 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
993 SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
994 Ty: FTy, RetType, ArgTypes, MIRBuilder);
995
996 // define vendor specific assembly instructions string
997 Register AsmReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
998 MRI.setRegClass(Reg: AsmReg, RC: &SPIRV::iIDRegClass);
999 auto AsmMIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmINTEL)
1000 .addDef(RegNo: AsmReg)
1001 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1002 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncType))
1003 .addUse(RegNo: AsmTargetReg);
1004 // inline asm string:
1005 addStringImm(Str: I2->getOperand(i: InlineAsm::MIOp_AsmString).getSymbolName(),
1006 MIB&: AsmMIB);
1007 // inline asm constraint string:
1008 addStringImm(Str: cast<MDString>(Val: I1->getOperand(i: 2).getMetadata()->getOperand(I: 0))
1009 ->getString(),
1010 MIB&: AsmMIB);
1011 GR->add(Obj: AsmMIB.getInstr(), MI: AsmMIB);
1012
1013 // calls the inline assembly instruction
1014 unsigned ExtraInfo = I2->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
1015 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
1016 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
1017 .addUse(RegNo: AsmReg)
1018 .addImm(Val: static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
1019
1020 Register DefReg = collectInlineAsmInstrOperands(MI: I2);
1021 if (!DefReg.isValid()) {
1022 DefReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1023 MRI.setRegClass(Reg: DefReg, RC: &SPIRV::iIDRegClass);
1024 SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType(
1025 Type: Type::getVoidTy(C&: MF.getFunction().getContext()), MIRBuilder,
1026 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1027 GR->assignSPIRVTypeToVReg(Type: VoidType, VReg: DefReg, MF);
1028 }
1029
1030 auto AsmCall = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmCallINTEL)
1031 .addDef(RegNo: DefReg)
1032 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1033 .addUse(RegNo: AsmReg);
1034 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
1035 AsmCall.addUse(RegNo: I1->getOperand(i: IntrIdx).getReg());
1036
1037 // IRTranslator gets a bit confused when lowering inline ASM with outputs
1038 // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
1039 // we have to clean that up here to prevent erroneous trunc casts either on
1040 // a struct (for multiple outputs) or same width integers to get lowered
1041 // into SPIR-V
1042 if (MRI.hasOneUse(RegNo: DefReg)) {
1043 MachineInstr &CopyMI = *MRI.use_instr_begin(RegNo: DefReg);
1044 if (CopyMI.getOpcode() == TargetOpcode::COPY) {
1045 Register CopyDst = CopyMI.getOperand(i: 0).getReg();
1046 if (MRI.hasOneUse(RegNo: CopyDst)) {
1047 MachineInstr &TruncMI = *MRI.use_instr_begin(RegNo: CopyDst);
1048 if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
1049 MRI.setType(VReg: DefReg, Ty: GR->getRegType(SpvType: RetType));
1050 Register TruncReg = TruncMI.defs().begin()->getReg();
1051 MRI.replaceRegWith(FromReg: TruncReg, ToReg: DefReg);
1052 invalidateAndEraseMI(GR, MI: &TruncMI);
1053 invalidateAndEraseMI(GR, MI: &CopyMI);
1054 }
1055 }
1056 }
1057 }
1058 }
1059 for (MachineInstr *MI : ToProcess)
1060 invalidateAndEraseMI(GR, MI);
1061}
1062
1063static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1064 const SPIRVSubtarget &ST,
1065 MachineIRBuilder MIRBuilder) {
1066 SmallVector<MachineInstr *> ToProcess;
1067 for (MachineBasicBlock &MBB : MF) {
1068 for (MachineInstr &MI : MBB) {
1069 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_inline_asm) ||
1070 MI.getOpcode() == TargetOpcode::INLINEASM)
1071 ToProcess.push_back(Elt: &MI);
1072 }
1073 }
1074 if (ToProcess.size() == 0)
1075 return;
1076
1077 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly))
1078 report_fatal_error(reason: "Inline assembly instructions require the "
1079 "following SPIR-V extension: SPV_INTEL_inline_assembly",
1080 gen_crash_diag: false);
1081
1082 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
1083}
1084
1085static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1086 MachineIRBuilder MIB) {
1087 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MIB.getMF().getSubtarget());
1088 SmallVector<MachineInstr *, 10> ToErase;
1089 for (MachineBasicBlock &MBB : MF) {
1090 for (MachineInstr &MI : MBB) {
1091 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration) &&
1092 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_aliasing_decoration) &&
1093 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration))
1094 continue;
1095 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getNextNode());
1096 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration)) {
1097 buildOpSpirvDecorations(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1098 GVarMD: MI.getOperand(i: 2).getMetadata(), ST);
1099 } else if (isSpvIntrinsic(MI,
1100 IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration)) {
1101 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(
1102 MD: MI.getOperand(i: 2).getMetadata()->getOperand(I: 0));
1103 uint32_t OpValue = OpV->getValueAPF().bitcastToAPInt().getZExtValue();
1104
1105 buildOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1106 Dec: SPIRV::Decoration::FPMaxErrorDecorationINTEL,
1107 DecArgs: {OpValue});
1108 } else {
1109 GR->buildMemAliasingOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1110 Dec: MI.getOperand(i: 2).getImm(),
1111 GVarMD: MI.getOperand(i: 3).getMetadata());
1112 }
1113
1114 ToErase.push_back(Elt: &MI);
1115 }
1116 }
1117 for (MachineInstr *MI : ToErase)
1118 invalidateAndEraseMI(GR, MI);
1119}
1120
1121// LLVM allows the switches to use registers as cases, while SPIR-V required
1122// those to be immediate values. This function replaces such operands with the
1123// equivalent immediate constant.
1124static void processSwitchesConstants(MachineFunction &MF,
1125 SPIRVGlobalRegistry *GR,
1126 MachineIRBuilder MIB) {
1127 MachineRegisterInfo &MRI = MF.getRegInfo();
1128 for (MachineBasicBlock &MBB : MF) {
1129 for (MachineInstr &MI : MBB) {
1130 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch))
1131 continue;
1132
1133 SmallVector<MachineOperand, 8> NewOperands;
1134 NewOperands.push_back(Elt: MI.getOperand(i: 0)); // Opcode
1135 NewOperands.push_back(Elt: MI.getOperand(i: 1)); // Condition
1136 NewOperands.push_back(Elt: MI.getOperand(i: 2)); // Default
1137 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
1138 Register Reg = MI.getOperand(i).getReg();
1139 MachineInstr *ConstInstr = getDefInstrMaybeConstant(ConstReg&: Reg, MRI: &MRI);
1140 NewOperands.push_back(
1141 Elt: MachineOperand::CreateCImm(CI: ConstInstr->getOperand(i: 1).getCImm()));
1142
1143 NewOperands.push_back(Elt: MI.getOperand(i: i + 1));
1144 }
1145
1146 assert(MI.getNumOperands() == NewOperands.size());
1147 while (MI.getNumOperands() > 0)
1148 MI.removeOperand(OpNo: 0);
1149 for (auto &MO : NewOperands)
1150 MI.addOperand(Op: MO);
1151 }
1152 }
1153}
1154
1155// Some instructions are used during CodeGen but should never be emitted.
1156// Cleaning up those.
1157static void cleanupHelperInstructions(MachineFunction &MF,
1158 SPIRVGlobalRegistry *GR) {
1159 SmallVector<MachineInstr *, 8> ToEraseMI;
1160 for (MachineBasicBlock &MBB : MF) {
1161 for (MachineInstr &MI : MBB) {
1162 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant) ||
1163 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
1164 ToEraseMI.push_back(Elt: &MI);
1165 }
1166 }
1167
1168 for (MachineInstr *MI : ToEraseMI)
1169 invalidateAndEraseMI(GR, MI);
1170}
1171
1172// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
1173// operands/registers by the actual MBB it references.
1174static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1175 MachineIRBuilder MIB) {
1176 // Gather the reverse-mapping BB -> MBB.
1177 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
1178 for (MachineBasicBlock &MBB : MF)
1179 BB2MBB[MBB.getBasicBlock()] = &MBB;
1180
1181 // Gather instructions requiring patching. For now, only those can use
1182 // G_BLOCK_ADDR.
1183 SmallVector<MachineInstr *, 8> InstructionsToPatch;
1184 for (MachineBasicBlock &MBB : MF) {
1185 for (MachineInstr &MI : MBB) {
1186 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch) ||
1187 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_loop_merge) ||
1188 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_selection_merge))
1189 InstructionsToPatch.push_back(Elt: &MI);
1190 }
1191 }
1192
1193 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
1194 // the actual MBB it references. Once those references have been updated, we
1195 // can cleanup remaining G_BLOCK_ADDR references.
1196 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
1197 SmallPtrSet<MachineInstr *, 8> ToEraseMI;
1198 MachineRegisterInfo &MRI = MF.getRegInfo();
1199 for (MachineInstr *MI : InstructionsToPatch) {
1200 SmallVector<MachineOperand, 8> NewOps;
1201 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1202 // The operand is not a register, keep as-is.
1203 if (!MI->getOperand(i).isReg()) {
1204 NewOps.push_back(Elt: MI->getOperand(i));
1205 continue;
1206 }
1207
1208 Register Reg = MI->getOperand(i).getReg();
1209 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
1210 // The register is not the result of G_BLOCK_ADDR, keep as-is.
1211 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
1212 NewOps.push_back(Elt: MI->getOperand(i));
1213 continue;
1214 }
1215
1216 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
1217 BuildMBB->getOperand(1).isBlockAddress() &&
1218 BuildMBB->getOperand(1).getBlockAddress());
1219 BasicBlock *BB =
1220 BuildMBB->getOperand(i: 1).getBlockAddress()->getBasicBlock();
1221 auto It = BB2MBB.find(Val: BB);
1222 if (It == BB2MBB.end())
1223 report_fatal_error(reason: "cannot find a machine basic block by a basic block "
1224 "in a switch statement");
1225 MachineBasicBlock *ReferencedBlock = It->second;
1226 NewOps.push_back(Elt: MachineOperand::CreateMBB(MBB: ReferencedBlock));
1227
1228 ClearAddressTaken.insert(Ptr: ReferencedBlock);
1229 ToEraseMI.insert(Ptr: BuildMBB);
1230 }
1231
1232 // Replace the operands.
1233 assert(MI->getNumOperands() == NewOps.size());
1234 while (MI->getNumOperands() > 0)
1235 MI->removeOperand(OpNo: 0);
1236 for (auto &MO : NewOps)
1237 MI->addOperand(Op: MO);
1238
1239 if (MachineInstr *Next = MI->getNextNode()) {
1240 if (isSpvIntrinsic(MI: *Next, IntrinsicID: Intrinsic::spv_track_constant)) {
1241 ToEraseMI.insert(Ptr: Next);
1242 Next = MI->getNextNode();
1243 }
1244 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
1245 ToEraseMI.insert(Ptr: Next);
1246 }
1247 }
1248
1249 // BlockAddress operands were used to keep information between passes,
1250 // let's undo the "address taken" status to reflect that Succ doesn't
1251 // actually correspond to an IR-level basic block.
1252 for (MachineBasicBlock *Succ : ClearAddressTaken)
1253 Succ->setAddressTakenIRBlock(nullptr);
1254
1255 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
1256 // this leaves their BasicBlock counterparts in a "address taken" status. This
1257 // would make AsmPrinter to generate a series of unneeded labels of a "Address
1258 // of block that was removed by CodeGen" kind. Let's first ensure that we
1259 // don't have a dangling BlockAddress constants by zapping the BlockAddress
1260 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
1261 Constant *Replacement =
1262 ConstantInt::get(Ty: Type::getInt32Ty(C&: MF.getFunction().getContext()), V: 1);
1263 for (MachineInstr *BlockAddrI : ToEraseMI) {
1264 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
1265 BlockAddress *BA = const_cast<BlockAddress *>(
1266 BlockAddrI->getOperand(i: 1).getBlockAddress());
1267 BA->replaceAllUsesWith(
1268 V: ConstantExpr::getIntToPtr(C: Replacement, Ty: BA->getType()));
1269 BA->destroyConstant();
1270 }
1271 invalidateAndEraseMI(GR, MI: BlockAddrI);
1272 }
1273}
1274
1275static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
1276 if (MBB.empty())
1277 return MBB.getNextNode() != nullptr;
1278
1279 // Branching SPIR-V intrinsics are not detected by this generic method.
1280 // Thus, we can only trust negative result.
1281 if (!MBB.canFallThrough())
1282 return false;
1283
1284 // Otherwise, we must manually check if we have a SPIR-V intrinsic which
1285 // prevent an implicit fallthrough.
1286 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
1287 It != E; ++It) {
1288 if (isSpvIntrinsic(MI: *It, IntrinsicID: Intrinsic::spv_switch))
1289 return false;
1290 }
1291 return true;
1292}
1293
1294static void removeImplicitFallthroughs(MachineFunction &MF,
1295 MachineIRBuilder MIB) {
1296 // It is valid for MachineBasicBlocks to not finish with a branch instruction.
1297 // In such cases, they will simply fallthrough their immediate successor.
1298 for (MachineBasicBlock &MBB : MF) {
1299 if (!isImplicitFallthrough(MBB))
1300 continue;
1301
1302 assert(MBB.succ_size() == 1);
1303 MIB.setInsertPt(MBB, II: MBB.end());
1304 MIB.buildBr(Dest&: **MBB.successors().begin());
1305 }
1306}
1307
1308static bool runPreLegalizer(MachineFunction &MF) {
1309 // Initialize the type registry.
1310 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
1311 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1312 GR->setCurrentFunc(MF);
1313 MachineIRBuilder MIB(MF);
1314 // a registry of target extension constants
1315 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
1316 // to keep record of tracked constants
1317 addConstantsToTrack(MF, GR, STI: ST, TargetExtConstTypes);
1318 foldConstantsIntoIntrinsics(MF, GR, MIB);
1319 insertBitcasts(MF, GR, MIB);
1320 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
1321
1322 processSwitchesConstants(MF, GR, MIB);
1323 processBlockAddr(MF, GR, MIB);
1324 cleanupHelperInstructions(MF, GR);
1325
1326 processInstrsWithTypeFolding(MF, GR, MIB);
1327 removeImplicitFallthroughs(MF, MIB);
1328 insertSpirvDecorations(MF, GR, MIB);
1329 insertInlineAsm(MF, GR, ST, MIRBuilder: MIB);
1330 lowerBitcasts(MF, GR, MIB);
1331
1332 return true;
1333}
1334
1335INITIALIZE_PASS(SPIRVPreLegalizerLegacy, DEBUG_TYPE, "SPIRV pre legalizer",
1336 false, false)
1337
1338char SPIRVPreLegalizerLegacy::ID = 0;
1339
1340FunctionPass *llvm::createSPIRVPreLegalizerLegacyPass() {
1341 return new SPIRVPreLegalizerLegacy();
1342}
1343
1344bool SPIRVPreLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
1345 return runPreLegalizer(MF);
1346}
1347
1348PreservedAnalyses
1349SPIRVPreLegalizerPass::run(MachineFunction &MF,
1350 MachineFunctionAnalysisManager &MFAM) {
1351 bool Changed = runPreLegalizer(MF);
1352 if (!Changed)
1353 return PreservedAnalyses::all();
1354
1355 return getMachineFunctionPassPreservedAnalyses()
1356 .preserve<GISelValueTrackingAnalysis>();
1357}
1358