1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass prepares IR for legalization: it assigns SPIR-V types to registers
10// and removes intrinsics which holded these types during IR translation.
11// Also it processes constants and registers them in GR to avoid duplication.
12//
13//===----------------------------------------------------------------------===//
14
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
23#include "llvm/CodeGen/MachinePassManager.h"
24#include "llvm/IR/Analysis.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/InstrTypes.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30
31#define DEBUG_TYPE "spirv-prelegalizer"
32
33using namespace llvm;
34using namespace llvm::MIPatternMatch;
35
36namespace {
37class SPIRVPreLegalizerLegacy : public MachineFunctionPass {
38public:
39 static char ID;
40 SPIRVPreLegalizerLegacy() : MachineFunctionPass(ID) {}
41 bool runOnMachineFunction(MachineFunction &MF) override;
42 void getAnalysisUsage(AnalysisUsage &AU) const override;
43};
44} // namespace
45
46void SPIRVPreLegalizerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
47 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
48 MachineFunctionPass::getAnalysisUsage(AU);
49}
50
51static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
52 MachineInstr *MI) {
53 GR->invalidateMachineInstr(MI);
54 MI->eraseFromParent();
55}
56
57static void
58addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
59 const SPIRVSubtarget &STI,
60 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
61 MachineRegisterInfo &MRI = MF.getRegInfo();
62 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
63 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
64 for (MachineBasicBlock &MBB : MF) {
65 for (MachineInstr &MI : MBB) {
66 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant))
67 continue;
68 ToErase.push_back(Elt: &MI);
69 Register SrcReg = MI.getOperand(i: 2).getReg();
70 auto *Const =
71 cast<Constant>(Val: cast<ConstantAsMetadata>(
72 Val: MI.getOperand(i: 3).getMetadata()->getOperand(I: 0))
73 ->getValue());
74 if (auto *GV = dyn_cast<GlobalValue>(Val: Const)) {
75 Register Reg = GR->find(V: GV, MF: &MF);
76 if (!Reg.isValid()) {
77 GR->add(V: GV, MI: MRI.getVRegDef(Reg: SrcReg));
78 GR->addGlobalObject(V: GV, MF: &MF, R: SrcReg);
79 } else
80 RegsAlreadyAddedToDT[&MI] = Reg;
81 } else {
82 Register Reg = GR->find(V: Const, MF: &MF);
83 if (!Reg.isValid()) {
84 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Val: Const)) {
85 auto *BuildVec = MRI.getVRegDef(Reg: SrcReg);
86 assert(BuildVec &&
87 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
88 GR->add(V: Const, MI: BuildVec);
89 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
90 // Ensure that OpConstantComposite reuses a constant when it's
91 // already created and available in the same machine function.
92 Constant *ElemConst = ConstVec->getElementAsConstant(i);
93 Register ElemReg = GR->find(V: ElemConst, MF: &MF);
94 if (!ElemReg.isValid())
95 GR->add(V: ElemConst,
96 MI: MRI.getVRegDef(Reg: BuildVec->getOperand(i: 1 + i).getReg()));
97 else
98 BuildVec->getOperand(i: 1 + i).setReg(ElemReg);
99 }
100 }
101 if (Const->getType()->isTargetExtTy()) {
102 // remember association so that we can restore it when assign types
103 MachineInstr *SrcMI = MRI.getVRegDef(Reg: SrcReg);
104 if (SrcMI)
105 GR->add(V: Const, MI: SrcMI);
106 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
107 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
108 TargetExtConstTypes[SrcMI] = Const->getType();
109 if (Const->isNullValue()) {
110 MachineBasicBlock &DepMBB = MF.front();
111 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());
112 SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType(
113 Type: Const->getType(), MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
114 EmitIR: true);
115 assert(SrcMI && "Expected source instruction to be valid");
116 SrcMI->setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
117 SrcMI->addOperand(Op: MachineOperand::CreateReg(
118 Reg: GR->getSPIRVTypeID(SpirvType: ExtType), isDef: false));
119 }
120 }
121 } else {
122 RegsAlreadyAddedToDT[&MI] = Reg;
123 // This MI is unused and will be removed. If the MI uses
124 // const_composite, it will be unused and should be removed too.
125 assert(MI.getOperand(2).isReg() && "Reg operand is expected");
126 MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
127 if (SrcMI && isSpvIntrinsic(MI: *SrcMI, IntrinsicID: Intrinsic::spv_const_composite))
128 ToEraseComposites.push_back(Elt: SrcMI);
129 }
130 }
131 }
132 }
133 for (MachineInstr *MI : ToErase) {
134 Register Reg = MI->getOperand(i: 2).getReg();
135 auto It = RegsAlreadyAddedToDT.find(Val: MI);
136 if (It != RegsAlreadyAddedToDT.end())
137 Reg = It->second;
138 auto *RC = MRI.getRegClassOrNull(Reg: MI->getOperand(i: 0).getReg());
139 if (!MRI.getRegClassOrNull(Reg) && RC)
140 MRI.setRegClass(Reg, RC);
141 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: Reg);
142 invalidateAndEraseMI(GR, MI);
143 }
144 for (MachineInstr *MI : ToEraseComposites)
145 invalidateAndEraseMI(GR, MI);
146}
147
148static void foldConstantsIntoIntrinsics(MachineFunction &MF,
149 SPIRVGlobalRegistry *GR,
150 MachineIRBuilder MIB) {
151 SmallVector<MachineInstr *, 64> ToErase;
152 for (MachineBasicBlock &MBB : MF) {
153 for (MachineInstr &MI : MBB) {
154 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_name))
155 continue;
156 const MDNode *MD = MI.getOperand(i: 2).getMetadata();
157 StringRef ValueName = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
158 if (ValueName.size() > 0) {
159 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
160 buildOpName(Target: MI.getOperand(i: 1).getReg(), Name: ValueName, MIRBuilder&: MIB);
161 }
162 ToErase.push_back(Elt: &MI);
163 }
164 for (MachineInstr *MI : ToErase)
165 invalidateAndEraseMI(GR, MI);
166 ToErase.clear();
167 }
168}
169
170static MachineInstr *findAssignTypeInstr(Register Reg,
171 MachineRegisterInfo *MRI) {
172 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(RegNo: Reg),
173 IE = MRI->use_instr_end();
174 I != IE; ++I) {
175 MachineInstr *UseMI = &*I;
176 if ((isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_ptr_type) ||
177 isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_type)) &&
178 UseMI->getOperand(i: 1).getReg() == Reg)
179 return UseMI;
180 }
181 return nullptr;
182}
183
184static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
185 Register ResVReg, Register OpReg) {
186 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResVReg);
187 SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg);
188 assert(ResType && OpType && "Operand types are expected");
189 if (!GR->isBitcastCompatible(Type1: ResType, Type2: OpType))
190 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
191 MachineRegisterInfo *MRI = MIB.getMRI();
192 if (!MRI->getRegClassOrNull(Reg: ResVReg))
193 MRI->setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: ResType));
194 if (ResType == OpType)
195 MIB.buildInstr(Opcode: TargetOpcode::COPY).addDef(RegNo: ResVReg).addUse(RegNo: OpReg);
196 else
197 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
198 .addDef(RegNo: ResVReg)
199 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResType))
200 .addUse(RegNo: OpReg);
201}
202
203// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.
204// The verifier checks if the source and destination LLTs of a G_BITCAST are
205// different, but this check is too strict for SPIR-V's typed pointers, which
206// may have the same LLT but different SPIRV type (e.g. pointers to different
207// pointee types). By lowering to OpBitcast here, we bypass the verifier's
208// check. See discussion in https://github.com/llvm/llvm-project/pull/110270
209// for more context.
210//
211// We also handle the llvm.spv.bitcast intrinsic here. If the source and
212// destination SPIR-V types are the same, we lower it to a COPY to enable
213// further optimizations like copy propagation.
214static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
215 MachineIRBuilder MIB) {
216 SmallVector<MachineInstr *, 16> ToErase;
217 for (MachineBasicBlock &MBB : MF) {
218 for (MachineInstr &MI : MBB) {
219 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_bitcast)) {
220 Register DstReg = MI.getOperand(i: 0).getReg();
221 Register SrcReg = MI.getOperand(i: 2).getReg();
222 SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(VReg: DstReg);
223 assert(
224 DstType &&
225 "Expected destination SPIR-V type to have been assigned already.");
226 SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(VReg: SrcReg);
227 assert(SrcType &&
228 "Expected source SPIR-V type to have been assigned already.");
229 if (DstType == SrcType) {
230 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
231 MIB.buildCopy(Res: DstReg, Op: SrcReg);
232 ToErase.push_back(Elt: &MI);
233 continue;
234 }
235 }
236
237 if (MI.getOpcode() != TargetOpcode::G_BITCAST)
238 continue;
239
240 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
241 buildOpBitcast(GR, MIB, ResVReg: MI.getOperand(i: 0).getReg(),
242 OpReg: MI.getOperand(i: 1).getReg());
243 ToErase.push_back(Elt: &MI);
244 }
245 }
246 for (MachineInstr *MI : ToErase)
247 invalidateAndEraseMI(GR, MI);
248}
249
250static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
251 MachineIRBuilder MIB) {
252 // Get access to information about available extensions
253 const SPIRVSubtarget *ST =
254 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
255 SmallVector<MachineInstr *, 10> ToErase;
256 for (MachineBasicBlock &MBB : MF) {
257 for (MachineInstr &MI : MBB) {
258 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_ptrcast))
259 continue;
260 assert(MI.getOperand(2).isReg());
261 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
262 ToErase.push_back(Elt: &MI);
263 Register Def = MI.getOperand(i: 0).getReg();
264 Register Source = MI.getOperand(i: 2).getReg();
265 Type *ElemTy = getMDOperandAsType(N: MI.getOperand(i: 3).getMetadata(), I: 0);
266 auto SC =
267 isa<FunctionType>(Val: ElemTy) &&
268 ST->canUseExtension(
269 E: SPIRV::Extension::SPV_INTEL_function_pointers)
270 ? SPIRV::StorageClass::CodeSectionINTEL
271 : addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 4).getImm(), STI: *ST);
272 SPIRVTypeInst AssignedPtrType =
273 GR->getOrCreateSPIRVPointerType(BaseType: ElemTy, I&: MI, SC);
274
275 // If the ptrcast would be redundant, replace all uses with the source
276 // register.
277 MachineRegisterInfo *MRI = MIB.getMRI();
278 // For untyped pointers the SPIR-V pointer type does not encode the
279 // pointee, so two pointers with different element types share the same
280 // pointer type. The element type still matters because it selects the
281 // Base Type operand of OpUntyped*AccessChainKHR. Treat the cast as
282 // redundant only when the source already carries the same element type.
283 // Otherwise keep a distinct register so the element type is preserved.
284 bool Redundant =
285 AssignedPtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR
286 ? GR->getUntypedPtrElementType(Reg: Source) ==
287 GR->getOrCreateSPIRVType(Type: ElemTy, MIRBuilder&: MIB,
288 AQ: SPIRV::AccessQualifier::ReadWrite,
289 /*EmitIR=*/true)
290 : GR->getSPIRVTypeForVReg(VReg: Source) == AssignedPtrType;
291 if (Redundant) {
292 // Erase Def's assign type instruction if we are going to replace Def.
293 if (MachineInstr *AssignMI = findAssignTypeInstr(Reg: Def, MRI))
294 ToErase.push_back(Elt: AssignMI);
295 MRI->replaceRegWith(FromReg: Def, ToReg: Source);
296 } else {
297 if (!GR->getSPIRVTypeForVReg(VReg: Def, MF: &MF))
298 GR->assignSPIRVTypeToVReg(Type: AssignedPtrType, VReg: Def, MF);
299 MIB.buildBitcast(Dst: Def, Src: Source);
300 }
301 }
302 }
303 for (MachineInstr *MI : ToErase)
304 invalidateAndEraseMI(GR, MI);
305}
306
307// Translating GV, IRTranslator sometimes generates following IR:
308// %1 = G_GLOBAL_VALUE
309// %2 = COPY %1
310// %3 = G_ADDRSPACE_CAST %2
311//
312// or
313//
314// %1 = G_ZEXT %2
315// G_MEMCPY ... %2 ...
316//
317// New registers have no SPIRV type and no register class info.
318//
319// Set SPIRV type for GV, propagate it from GV to other instructions,
320// also set register classes.
321static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
322 SPIRVGlobalRegistry *GR,
323 MachineRegisterInfo &MRI,
324 MachineIRBuilder &MIB) {
325 SPIRVTypeInst SpvType = nullptr;
326 assert(MI && "Machine instr is expected");
327 if (MI->getOperand(i: 0).isReg()) {
328 Register Reg = MI->getOperand(i: 0).getReg();
329 SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
330 if (!SpvType) {
331 switch (MI->getOpcode()) {
332 case TargetOpcode::G_FCONSTANT:
333 case TargetOpcode::G_CONSTANT: {
334 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
335 Type *Ty = MI->getOperand(i: 1).getCImm()->getType();
336 SpvType = GR->getOrCreateSPIRVType(
337 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
338 break;
339 }
340 case TargetOpcode::G_GLOBAL_VALUE: {
341 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
342 const GlobalValue *Global = MI->getOperand(i: 1).getGlobal();
343 Type *ElementTy = toTypedPointer(Ty: GR->getDeducedGlobalValueType(Global));
344 unsigned AddrSpace = Global->getType()->getAddressSpace();
345 // Function pointers use CodeSectionINTEL storage class in SPIR-V when
346 // the SPV_INTEL_function_pointers extension is enabled.
347 const SPIRVSubtarget &ST = MIB.getMF().getSubtarget<SPIRVSubtarget>();
348 if (isa<Function>(Val: Global) &&
349 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
350 AddrSpace =
351 storageClassToAddressSpace(SC: SPIRV::StorageClass::CodeSectionINTEL);
352 auto *Ty = TypedPointerType::get(ElementType: ElementTy, AddressSpace: AddrSpace);
353 SpvType = GR->getOrCreateSPIRVType(
354 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
355 break;
356 }
357 case TargetOpcode::G_ANYEXT:
358 case TargetOpcode::G_SEXT:
359 case TargetOpcode::G_ZEXT: {
360 if (MI->getOperand(i: 1).isReg()) {
361 if (MachineInstr *DefInstr =
362 MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
363 if (SPIRVTypeInst Def =
364 propagateSPIRVType(MI: DefInstr, GR, MRI, MIB)) {
365 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Type: Def);
366 unsigned ExpectedBW =
367 std::max(a: MRI.getType(Reg).getScalarSizeInBits(), b: CurrentBW);
368 unsigned NumElements = GR->getScalarOrVectorComponentCount(Type: Def);
369 SpvType = GR->getOrCreateSPIRVIntegerType(BitWidth: ExpectedBW, MIRBuilder&: MIB);
370 if (NumElements > 1)
371 SpvType = GR->getOrCreateSPIRVVectorType(BaseType: SpvType, NumElements,
372 MIRBuilder&: MIB, EmitIR: true);
373 }
374 }
375 }
376 break;
377 }
378 case TargetOpcode::G_PTRTOINT:
379 SpvType = GR->getOrCreateSPIRVIntegerType(
380 BitWidth: MRI.getType(Reg).getScalarSizeInBits(), MIRBuilder&: MIB);
381 break;
382 case TargetOpcode::G_TRUNC:
383 case TargetOpcode::G_ADDRSPACE_CAST:
384 case TargetOpcode::G_PTR_ADD:
385 case TargetOpcode::COPY: {
386 MachineOperand &Op = MI->getOperand(i: 1);
387 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Reg: Op.getReg()) : nullptr;
388 if (Def)
389 SpvType = propagateSPIRVType(MI: Def, GR, MRI, MIB);
390 break;
391 }
392 default:
393 break;
394 }
395 if (SpvType) {
396 // check if the address space needs correction
397 LLT RegType = MRI.getType(Reg);
398 if (SpvType.isPointer() && RegType.isPointer() &&
399 storageClassToAddressSpace(SC: GR->getPointerStorageClass(Type: SpvType)) !=
400 RegType.getAddressSpace()) {
401 // Don't correct CodeSectionINTEL back to Function for function
402 // pointer G_GLOBAL_VALUE - the LLVM register has address space 0
403 // but the SPIR-V type was intentionally set to CodeSectionINTEL.
404 bool SkipCorrection =
405 MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
406 GR->getPointerStorageClass(Type: SpvType) ==
407 SPIRV::StorageClass::CodeSectionINTEL;
408 if (!SkipCorrection) {
409 const SPIRVSubtarget &ST =
410 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
411 auto TSC =
412 addressSpaceToStorageClass(AddrSpace: RegType.getAddressSpace(), STI: ST);
413 SpvType = GR->changePointerStorageClass(PtrType: SpvType, SC: TSC, I&: *MI);
414 }
415 }
416 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
417 }
418 if (!MRI.getRegClassOrNull(Reg))
419 MRI.setRegClass(Reg, RC: SpvType ? GR->getRegClass(SpvType)
420 : &SPIRV::iIDRegClass);
421 }
422 }
423 return SpvType;
424}
425
426// To support current approach and limitations wrt. bit width here we widen a
427// scalar register with a bit width greater than 1 to valid sizes and cap it to
428// 128 width.
429static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {
430 if (BitWidth == 1)
431 return 1; // No need to widen 1-bit values
432 return std::min(a: std::max<unsigned>(a: PowerOf2Ceil(A: BitWidth), b: 8u), b: 128u);
433}
434
435static std::optional<unsigned>
436getNarrowScalarWidth(Register Reg, const MachineRegisterInfo &MRI) {
437 LLT Ty = MRI.getType(Reg);
438 if (!Ty.isScalar())
439 return std::nullopt;
440 unsigned W = Ty.getScalarSizeInBits();
441 // <= and not == because widenBitWidthToNextPow2 caps at 128.
442 if (widenBitWidthToNextPow2(BitWidth: W) <= W)
443 return std::nullopt;
444 return W;
445}
446
447static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {
448 LLT RegType = MRI.getType(Reg);
449 if (!RegType.isScalar())
450 return;
451 unsigned CurrentWidth = RegType.getScalarSizeInBits();
452 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
453 if (NewWidth != CurrentWidth)
454 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: NewWidth));
455}
456
457static void widenCImmType(MachineOperand &MOP) {
458 const ConstantInt *CImmVal = MOP.getCImm();
459 unsigned CurrentWidth = CImmVal->getBitWidth();
460 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
461 if (NewWidth != CurrentWidth) {
462 // Replace the immediate value with the widened version
463 MOP.setCImm(ConstantInt::get(Context&: CImmVal->getType()->getContext(),
464 V: CImmVal->getValue().zextOrTrunc(width: NewWidth)));
465 }
466}
467
468static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
469 MachineBasicBlock &MBB = *Def->getParent();
470 MachineBasicBlock::iterator DefIt =
471 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();
472 // Skip all the PHI and debug instructions.
473 while (DefIt != MBB.end() &&
474 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))
475 DefIt = std::next(x: DefIt);
476 MIB.setInsertPt(MBB, II: DefIt);
477}
478
479namespace llvm {
480void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType,
481 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
482 MachineRegisterInfo &MRI) {
483 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
484 MachineInstr *Def = MRI.getVRegDef(Reg);
485 setInsertPtAfterDef(MIB, Def);
486 if (!SpvType)
487 SpvType = GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder&: MIB,
488 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
489 if (!MRI.getRegClassOrNull(Reg))
490 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType));
491 if (!MRI.getType(Reg).isValid())
492 MRI.setType(VReg: Reg, Ty: GR->getRegType(SpvType));
493 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
494}
495
496void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
497 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
498 SPIRVTypeInst KnownResType) {
499 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
500 for (auto &Op : MI.operands()) {
501 if (!Op.isReg() || Op.isDef())
502 continue;
503 Register OpReg = Op.getReg();
504 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OpReg);
505 if (!SpvType && KnownResType) {
506 SpvType = KnownResType;
507 GR->assignSPIRVTypeToVReg(Type: KnownResType, VReg: OpReg, MF: *MI.getMF());
508 }
509 assert(SpvType);
510 if (!MRI.getRegClassOrNull(Reg: OpReg))
511 MRI.setRegClass(Reg: OpReg, RC: GR->getRegClass(SpvType));
512 if (!MRI.getType(Reg: OpReg).isValid())
513 MRI.setType(VReg: OpReg, Ty: GR->getRegType(SpvType));
514 }
515}
516} // namespace llvm
517
518// Sign-sensitive integer ops: their result depends on the value of the input
519// sign bit at position (width-1). On sub-pow2 widths the general widening
520// loop is a pure LLT relabel, which leaves the sign bit at the *original*
521// position instead of the widened MSB. These ops therefore need an explicit
522// G_SEXT_INREG on each value operand to move the sign bit up.
523//
524// Signed-vs-unsigned G_ICMP is distinguished by its predicate operand.
525//
526// TODO: follow-up PRs will add the remaining sign-sensitive opcodes
527// (e.g. G_SMIN/G_SMAX, G_SADDSAT/G_SSUBSAT, signed overflow ops).
528static bool isSignSensitiveOp(const MachineInstr &MI) {
529 switch (MI.getOpcode()) {
530 case TargetOpcode::G_ASHR:
531 case TargetOpcode::G_SDIV:
532 case TargetOpcode::G_SREM:
533 return true;
534 case TargetOpcode::G_ICMP:
535 return CmpInst::isSigned(
536 Pred: static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate()));
537 default:
538 return false;
539 }
540}
541
542struct NarrowWideningInfo {
543 // Width before widening of each sign-sensitive value-operand vreg (one entry
544 // per vreg).
545 DenseMap<Register, unsigned> OrigWidth;
546 // Sign-sensitive ops whose value operand(s) need replacing, ordered for
547 // reproducible vreg numbering.
548 SmallVector<MachineInstr *> SignSensitiveWorklist;
549 // Keyed by instruction, not vreg: G_TRUNC handling can replace the source.
550 SmallVector<std::pair<MachineInstr *, unsigned>> BitCountWorklist;
551};
552
553// G_CTTZ_ZERO_POISON is absent because its low bits are known non-zero, G_CTLS
554// because the backend does not select it.
555static bool isWidthSensitiveBitCountOp(unsigned Opcode) {
556 switch (Opcode) {
557 case TargetOpcode::G_CTLZ:
558 case TargetOpcode::G_CTLZ_ZERO_POISON:
559 case TargetOpcode::G_CTTZ:
560 case TargetOpcode::G_CTPOP:
561 return true;
562 default:
563 return false;
564 }
565}
566
567// Collect ops whose semantics depend on the operand width along with their
568// pre-widening widths, before later passes retype those vregs to pow2 LLTs
569// and the original width is no longer recoverable.
570static NarrowWideningInfo
571recordNarrowOperandWidths(MachineFunction &MF, const MachineRegisterInfo &MRI) {
572 NarrowWideningInfo Info;
573 auto RecordIfNarrow = [&](Register Reg) {
574 std::optional<unsigned> W = getNarrowScalarWidth(Reg, MRI);
575 if (!W)
576 return false;
577 Info.OrigWidth.try_emplace(Key: Reg, Args&: *W);
578 return true;
579 };
580 for (MachineBasicBlock &MBB : MF) {
581 for (MachineInstr &MI : MBB) {
582 if (isWidthSensitiveBitCountOp(Opcode: MI.getOpcode())) {
583 if (std::optional<unsigned> W =
584 getNarrowScalarWidth(Reg: MI.getOperand(i: 1).getReg(), MRI))
585 Info.BitCountWorklist.emplace_back(Args: &MI, Args&: *W);
586 continue;
587 }
588 if (!isSignSensitiveOp(MI))
589 continue;
590 // Value operands are the trailing two, past any def or predicate.
591 unsigned N = MI.getNumOperands();
592 const MachineOperand &LHS = MI.getOperand(i: N - 2);
593 const MachineOperand &RHS = MI.getOperand(i: N - 1);
594 // Sign-sensitive opcodes carry register operands only.
595 assert(LHS.isReg() && RHS.isReg());
596 bool NeedsRewrite = RecordIfNarrow(LHS.getReg());
597 NeedsRewrite = RecordIfNarrow(RHS.getReg()) || NeedsRewrite;
598 if (NeedsRewrite)
599 Info.SignSensitiveWorklist.push_back(Elt: &MI);
600 }
601 }
602 return Info;
603}
604
605// For every recorded sign-sensitive op, insert G_SEXT_INREG on each value
606// operand whose original width was narrower than the widened pow2 width and
607// retype the operand's vreg LLT in place to the widened width.
608//
609// Info must have been populated by recordNarrowOperandWidths before
610// other passes retyped the vregs; otherwise the narrow widths needed here
611// are lost.
612//
613// TODO: handle vector operands.
614static void widenSignSensitiveOps(MachineFunction &MF, SPIRVGlobalRegistry *GR,
615 MachineIRBuilder &MIB,
616 MachineRegisterInfo &MRI,
617 const NarrowWideningInfo &Info) {
618 // Emit G_SEXT_INREG from Reg's recorded narrow width; retypes Reg to the
619 // widened width and returns the sign-extended vreg.
620 auto SignExtendReg = [&](Register Reg, unsigned OldW,
621 MachineInstr &MI) -> Register {
622 unsigned NewW = widenBitWidthToNextPow2(BitWidth: OldW);
623 LLT NewLLT = LLT::scalar(SizeInBits: NewW);
624 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
625 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewW, MIRBuilder&: MIB);
626 Register SExted = MRI.createGenericVirtualRegister(Ty: NewLLT);
627 GR->assignSPIRVTypeToVReg(Type: SpvTy, VReg: SExted, MF);
628 MRI.setRegClass(Reg: SExted, RC: GR->getRegClass(SpvType: SpvTy));
629 MRI.setType(VReg: Reg, Ty: NewLLT);
630 MIB.buildSExtInReg(Res: SExted, Op: Reg, ImmOp: OldW);
631 return SExted;
632 };
633
634 // TODO: when the same narrow vreg feeds multiple sign-sensitive ops (e.g.
635 // sdiv %x, %y and srem %x, %y), emit one shared G_SEXT_INREG instead of one
636 // per use.
637 for (MachineInstr *MI : Info.SignSensitiveWorklist) {
638 unsigned N = MI->getNumOperands();
639 MachineOperand &LHS = MI->getOperand(i: N - 2);
640 MachineOperand &RHS = MI->getOperand(i: N - 1);
641 Register LHSReg = LHS.getReg();
642 Register RHSReg = RHS.getReg();
643 if (auto It = Info.OrigWidth.find(Val: LHSReg); It != Info.OrigWidth.end())
644 LHS.setReg(SignExtendReg(LHSReg, It->second, *MI));
645 // Same vreg on both sides (e.g. G_ICMP slt %x, %x): reuse the sext just
646 // emitted for LHS instead of emitting a second one.
647 if (RHSReg == LHSReg) {
648 RHS.setReg(LHS.getReg());
649 continue;
650 }
651 if (auto It = Info.OrigWidth.find(Val: RHSReg); It != Info.OrigWidth.end())
652 RHS.setReg(SignExtendReg(RHSReg, It->second, *MI));
653 }
654}
655
656// LegalizerHelper::widenScalar has the same cases but cannot be reached: the
657// relabel retypes every narrow scalar to a pow2 LLT, so no illegal narrow type
658// ever reaches the legalizer.
659//
660// TODO: handle vector operands.
661static void widenBitCountOps(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
662 MachineRegisterInfo &MRI,
663 const NarrowWideningInfo &Info) {
664 for (auto [MI, OldWidth] : Info.BitCountWorklist) {
665 Register SrcReg = MI->getOperand(i: 1).getReg();
666 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: OldWidth);
667 LLT NewTy = LLT::scalar(SizeInBits: NewWidth);
668 widenScalarType(Reg: SrcReg, MRI);
669 MIB.setInstrAndDebugLoc(*MI);
670 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewWidth, MIRBuilder&: MIB);
671
672 // The G_TRUNC lowering masks its result to the narrow width, so a source
673 // coming from it needs no second mask.
674 APInt Cst;
675 bool HighBitsAlreadyZero =
676 mi_match(R: SrcReg, MRI, P: m_GAnd(L: m_Reg(), R: m_ICst(Cst))) &&
677 Cst.isSubsetOf(RHS: APInt::getLowBitsSet(numBits: Cst.getBitWidth(), loBitsSet: OldWidth));
678 auto ClearHighBits = [&](unsigned Width) -> Register {
679 if (HighBitsAlreadyZero)
680 return SrcReg;
681 Register Masked = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
682 MIB.buildZExtInReg(Res: Masked, Op: SrcReg, ImmOp: Width);
683 return Masked;
684 };
685
686 Register Input;
687 switch (MI->getOpcode()) {
688 case TargetOpcode::G_CTLZ_ZERO_POISON: {
689 // Shifting up to the widened MSB moves the poison out too, so no
690 // adjustment.
691 Input = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
692 auto Diff = MIB.buildConstant(Res: NewTy, Val: NewWidth - OldWidth);
693 MIB.buildShl(Dst: Input, Src0: SrcReg, Src1: Diff);
694 break;
695 }
696 case TargetOpcode::G_CTTZ: {
697 // Keeps an all-zero narrow value counting exactly OldWidth zeros.
698 Input = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
699 auto TopBit =
700 MIB.buildConstant(Res: NewTy, Val: APInt::getOneBitSet(numBits: NewWidth, BitNo: OldWidth));
701 MIB.buildOr(Dst: Input, Src0: SrcReg, Src1: TopBit);
702 break;
703 }
704 case TargetOpcode::G_CTPOP:
705 Input = ClearHighBits(OldWidth);
706 break;
707 case TargetOpcode::G_CTLZ: {
708 // Clearing the extra bits adds leading zeros the count has to drop.
709 Input = ClearHighBits(OldWidth);
710 Register DstReg = MI->getOperand(i: 0).getReg();
711 widenScalarType(Reg: DstReg, MRI);
712 Register Count = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
713 MI->getOperand(i: 0).setReg(Count);
714 setInsertPtAfterDef(MIB, Def: MI);
715 auto Diff = MIB.buildConstant(Res: NewTy, Val: NewWidth - OldWidth);
716 MIB.buildSub(Dst: DstReg, Src0: Count, Src1: Diff);
717 break;
718 }
719 default:
720 llvm_unreachable("unexpected width-sensitive bit-count opcode");
721 }
722 MI->getOperand(i: 1).setReg(Input);
723 }
724}
725
726static void
727generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
728 MachineIRBuilder MIB,
729 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
730 // Get access to information about available extensions
731 const SPIRVSubtarget *ST =
732 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
733
734 MachineRegisterInfo &MRI = MF.getRegInfo();
735 SmallVector<MachineInstr *, 10> ToErase;
736 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
737
738 bool IsExtendedInts =
739 ST->canUseExtension(
740 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
741 ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
742 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
743
744 if (!IsExtendedInts) {
745 // Without arbitrary precision integer extensions, SPIR-V only supports
746 // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40)
747 // must be widened to the next power of two.
748 //
749 // Record the original widths of width-sensitive operands before either
750 // the G_TRUNC handling or the general widening loop retypes vregs, then
751 // rewrite those ops after G_TRUNC processing using the recorded widths.
752 NarrowWideningInfo WideningInfo = recordNarrowOperandWidths(MF, MRI);
753
754 // G_TRUNC requires special handling because its semantics depend on the
755 // original destination width. For example:
756 // %dst:s24 = G_TRUNC %src:s64
757 // After widening s24 to s32, we cannot simply do:
758 // %dst:s32 = G_TRUNC %src:s64
759 // because this would keep 32 bits instead of 24. Instead, we insert a
760 // G_AND to mask the value to the original width:
761 // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask
762 // %masked:s64 = G_AND %src:s64, %mask
763 // %dst:s32 = G_TRUNC %masked:s64
764 // If src and dst widen to the same size, G_TRUNC is replaced entirely:
765 // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask
766 // %dst:s64 = G_AND %src:s64, %mask
767 SmallVector<MachineInstr *, 8> TruncToRemove;
768 for (MachineBasicBlock &MBB : MF) {
769 for (MachineInstr &MI : MBB) {
770 unsigned MIOp = MI.getOpcode();
771 if (MIOp != TargetOpcode::G_TRUNC)
772 continue;
773 assert(MI.getNumOperands() == 2);
774 assert(MI.getOperand(0).isReg());
775 assert(MI.getOperand(1).isReg());
776
777 Register DstReg = MI.getOperand(i: 0).getReg();
778 Register SrcReg = MI.getOperand(i: 1).getReg();
779
780 LLT DstTy = MRI.getType(Reg: DstReg);
781 LLT SrcTy = MRI.getType(Reg: SrcReg);
782 assert((DstTy.isScalar() || DstTy.isVector()) &&
783 (SrcTy.isScalar() || SrcTy.isVector()) &&
784 "Expected scalar or vector G_TRUNC types");
785 assert(DstTy.isVector() == SrcTy.isVector() &&
786 "Expected matching scalar/vector G_TRUNC types");
787 assert((!DstTy.isVector() ||
788 DstTy.getElementCount() == SrcTy.getElementCount()) &&
789 "Expected equal vector element counts");
790
791 unsigned OriginalDstWidth = DstTy.getScalarSizeInBits();
792 unsigned OriginalSrcWidth = SrcTy.getScalarSizeInBits();
793
794 unsigned NewDstWidth = widenBitWidthToNextPow2(BitWidth: OriginalDstWidth);
795 unsigned NewSrcWidth = widenBitWidthToNextPow2(BitWidth: OriginalSrcWidth);
796 LLT NewDstTy = DstTy.changeElementSize(NewEltSize: NewDstWidth);
797 LLT NewSrcTy = SrcTy.changeElementSize(NewEltSize: NewSrcWidth);
798
799 // No Dst width change means no truncation semantics change, but the
800 // source still needs a legal type.
801 if (OriginalDstWidth == NewDstWidth) {
802 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
803 continue;
804 }
805
806 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
807 MRI.setType(VReg: DstReg, Ty: NewDstTy);
808
809 MIB.setInsertPt(MBB, II: MI.getIterator());
810 APInt Mask = APInt::getLowBitsSet(numBits: NewSrcWidth, loBitsSet: OriginalDstWidth);
811 MachineInstrBuilder MaskReg =
812 DstTy.isVector()
813 ? MIB.buildBuildVectorConstant(
814 Res: NewSrcTy,
815 Ops: SmallVector<APInt, 4>(DstTy.getNumElements(), Mask))
816 : MIB.buildConstant(Res: NewSrcTy, Val: Mask);
817 Register MaskedReg = MRI.createGenericVirtualRegister(Ty: NewSrcTy);
818 MIB.buildAnd(Dst: MaskedReg, Src0: SrcReg, Src1: MaskReg);
819
820 if (NewSrcWidth == NewDstWidth) {
821 // Rekey OrigWidth from DstReg to MaskedReg so widenSignSensitiveOps
822 // still sees the narrow original width after replaceRegWith.
823 if (auto It = WideningInfo.OrigWidth.find(Val: DstReg);
824 It != WideningInfo.OrigWidth.end()) {
825 unsigned W = It->second;
826 WideningInfo.OrigWidth.erase(I: It);
827 WideningInfo.OrigWidth.try_emplace(Key: MaskedReg, Args&: W);
828 }
829 MRI.replaceRegWith(FromReg: DstReg, ToReg: MaskedReg);
830 TruncToRemove.push_back(Elt: &MI);
831 } else {
832 MI.getOperand(i: 1).setReg(MaskedReg);
833 }
834 }
835 }
836 for (MachineInstr *MI : TruncToRemove)
837 MI->eraseFromParent();
838
839 widenSignSensitiveOps(MF, GR, MIB, MRI, Info: WideningInfo);
840 widenBitCountOps(GR, MIB, MRI, Info: WideningInfo);
841 }
842
843 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
844 if (MBB->empty())
845 continue;
846
847 bool ReachedBegin = false;
848 for (auto MII = std::prev(x: MBB->end()), Begin = MBB->begin();
849 !ReachedBegin;) {
850 MachineInstr &MI = *MII;
851 unsigned MIOp = MI.getOpcode();
852
853 if (!IsExtendedInts) {
854 // validate bit width of scalar registers and constant immediates
855 for (auto &MOP : MI.operands()) {
856 if (MOP.isReg())
857 widenScalarType(Reg: MOP.getReg(), MRI);
858 else if (MOP.isCImm())
859 widenCImmType(MOP);
860 }
861 }
862
863 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) {
864 Register Reg = MI.getOperand(i: 1).getReg();
865 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
866 Type *ElementTy = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
867 auto SC = addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 3).getImm(), STI: *ST);
868 if (SC == SPIRV::StorageClass::Function &&
869 isa<FunctionType>(Val: ElementTy) &&
870 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
871 SC = SPIRV::StorageClass::CodeSectionINTEL;
872 SPIRVTypeInst AssignedPtrType =
873 GR->getOrCreateSPIRVPointerType(BaseType: ElementTy, I&: MI, SC);
874
875 // For untyped pointers, store the element type for later use.
876 if (ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers) &&
877 !ST->isShader()) {
878 SPIRVTypeInst ElemSpvType = GR->getOrCreateSPIRVType(
879 Type: ElementTy, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
880 /*EmitIR=*/true);
881 GR->setUntypedPtrElementType(Reg, ElemType: ElemSpvType);
882 }
883
884 // The intrinsic also carries vector-of-pointer values produced by
885 // scalarized vector GEPs; wrap the pointer in OpTypeVector to match
886 // the vreg's LLT.
887 LLT RegTy = MRI.getType(Reg);
888 if (RegTy.isValid() && RegTy.isVector())
889 AssignedPtrType = GR->getOrCreateSPIRVVectorType(
890 BaseType: AssignedPtrType, NumElements: RegTy.getNumElements(), MIRBuilder&: MIB,
891 /*EmitIR=*/true);
892 MachineInstr *Def = MRI.getVRegDef(Reg);
893 assert(Def && "Expecting an instruction that defines the register");
894 // G_GLOBAL_VALUE already has type info.
895 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
896 updateRegType(Reg, Ty: nullptr, SpvType: AssignedPtrType, GR, MIB,
897 MRI&: MF.getRegInfo());
898 ToErase.push_back(Elt: &MI);
899 } else if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_type)) {
900 Register Reg = MI.getOperand(i: 1).getReg();
901 Type *Ty = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
902 MachineInstr *Def = MRI.getVRegDef(Reg);
903 assert(Def && "Expecting an instruction that defines the register");
904 // G_GLOBAL_VALUE already has type info.
905 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
906 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
907 if (Def->getOpcode() == TargetOpcode::COPY && isVector1(Ty))
908 updateRegType(Reg: passCopy(Def, MRI: &MF.getRegInfo())->getOperand(i: 0).getReg(),
909 Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
910 ToErase.push_back(Elt: &MI);
911 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
912 MachineInstr *MdMI = MI.getPrevNode();
913 if (MdMI && isSpvIntrinsic(MI: *MdMI, IntrinsicID: Intrinsic::spv_value_md)) {
914 // It's an internal service info from before IRTranslator passes.
915 MachineInstr *Def = getVRegDef(MRI, Reg: MI.getOperand(i: 0).getReg());
916 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
917 if (getVRegDef(MRI, Reg: MI.getOperand(i: I).getReg()) != Def)
918 Def = nullptr;
919 if (Def) {
920 const MDNode *MD = MdMI->getOperand(i: 1).getMetadata();
921 StringRef ValueName =
922 cast<MDString>(Val: MD->getOperand(I: 1))->getString();
923 const MDNode *TypeMD = cast<MDNode>(Val: MD->getOperand(I: 0));
924 Type *ValueTy = getMDOperandAsType(N: TypeMD, I: 0);
925 GR->addValueAttrs(Key: Def, Val: std::make_pair(x&: ValueTy, y: ValueName.str()));
926 }
927 ToErase.push_back(Elt: MdMI);
928 }
929 ToErase.push_back(Elt: &MI);
930 } else if (MIOp == TargetOpcode::G_CONSTANT ||
931 MIOp == TargetOpcode::G_FCONSTANT ||
932 MIOp == TargetOpcode::G_BUILD_VECTOR) {
933 // %rc = G_CONSTANT ty Val
934 // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
935 Register Reg = MI.getOperand(i: 0).getReg();
936 bool NeedAssignType = !GR->getSPIRVTypeForVReg(VReg: Reg);
937 Type *Ty = nullptr;
938 if (MIOp == TargetOpcode::G_CONSTANT) {
939 auto TargetExtIt = TargetExtConstTypes.find(Val: &MI);
940 Ty = TargetExtIt == TargetExtConstTypes.end()
941 ? MI.getOperand(i: 1).getCImm()->getType()
942 : TargetExtIt->second;
943 const ConstantInt *OpCI = MI.getOperand(i: 1).getCImm();
944 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =
945 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe
946 // at this point not i64/i32 constant but null pointer in the
947 // corresponding address space of RegType.getAddressSpace(). This may
948 // help to successfully validate the case when a OpConstantComposite's
949 // constituent has type that does not match Result Type of
950 // OpConstantComposite (see, for example,
951 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).
952 Register PrimaryReg = GR->find(V: OpCI, MF: &MF);
953 if (!PrimaryReg.isValid()) {
954 GR->add(V: OpCI, MI: &MI);
955 } else if (PrimaryReg != Reg &&
956 MRI.getType(Reg) == MRI.getType(Reg: PrimaryReg)) {
957 auto *RCReg = MRI.getRegClassOrNull(Reg);
958 auto *RCPrimary = MRI.getRegClassOrNull(Reg: PrimaryReg);
959 if (!RCReg || RCPrimary == RCReg) {
960 RegsAlreadyAddedToDT[&MI] = PrimaryReg;
961 ToErase.push_back(Elt: &MI);
962 NeedAssignType = false;
963 }
964 }
965 } else if (MIOp == TargetOpcode::G_FCONSTANT) {
966 Ty = MI.getOperand(i: 1).getFPImm()->getType();
967 } else {
968 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
969 Type *ElemTy = nullptr;
970 MachineInstr *ElemMI = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
971 assert(ElemMI);
972
973 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
974 ElemTy = ElemMI->getOperand(i: 1).getCImm()->getType();
975 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
976 ElemTy = ElemMI->getOperand(i: 1).getFPImm()->getType();
977 } else {
978 if (SPIRVTypeInst ElemSpvType =
979 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg(), MF: &MF))
980 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(Ty: ElemSpvType));
981 }
982 if (ElemTy)
983 Ty = VectorType::get(
984 ElementType: ElemTy, NumElements: MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
985 Scalable: false);
986 else
987 NeedAssignType = false;
988 }
989 if (NeedAssignType)
990 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI);
991 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
992 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
993 }
994
995 if (MII == Begin)
996 ReachedBegin = true;
997 else
998 --MII;
999 }
1000 }
1001 for (MachineInstr *MI : ToErase) {
1002 auto It = RegsAlreadyAddedToDT.find(Val: MI);
1003 if (It != RegsAlreadyAddedToDT.end())
1004 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: It->second);
1005 invalidateAndEraseMI(GR, MI);
1006 }
1007
1008 // Address the case when IRTranslator introduces instructions with new
1009 // registers without associated SPIRV type.
1010 for (MachineBasicBlock &MBB : MF) {
1011 for (MachineInstr &MI : MBB) {
1012 switch (MI.getOpcode()) {
1013 case TargetOpcode::G_TRUNC:
1014 case TargetOpcode::G_ANYEXT:
1015 case TargetOpcode::G_SEXT:
1016 case TargetOpcode::G_ZEXT:
1017 case TargetOpcode::G_PTRTOINT:
1018 case TargetOpcode::COPY:
1019 case TargetOpcode::G_ADDRSPACE_CAST:
1020 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
1021 break;
1022 }
1023 }
1024 }
1025}
1026
1027static void processInstrsWithTypeFolding(MachineFunction &MF,
1028 SPIRVGlobalRegistry *GR,
1029 MachineIRBuilder MIB) {
1030 MachineRegisterInfo &MRI = MF.getRegInfo();
1031 for (MachineBasicBlock &MBB : MF)
1032 for (MachineInstr &MI : MBB)
1033 if (isTypeFoldingSupported(Opcode: MI.getOpcode()))
1034 processInstr(MI, MIB, MRI, GR, KnownResType: nullptr);
1035}
1036
1037static Register
1038collectInlineAsmInstrOperands(MachineInstr *MI,
1039 SmallVector<unsigned, 4> *Ops = nullptr) {
1040 Register DefReg;
1041 unsigned StartOp = InlineAsm::MIOp_FirstOperand,
1042 AsmDescOp = InlineAsm::MIOp_FirstOperand;
1043 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
1044 ++Idx) {
1045 const MachineOperand &MO = MI->getOperand(i: Idx);
1046 if (MO.isMetadata())
1047 continue;
1048 if (Idx == AsmDescOp && MO.isImm()) {
1049 // compute the index of the next operand descriptor
1050 const InlineAsm::Flag F(MO.getImm());
1051 AsmDescOp += 1 + F.getNumOperandRegisters();
1052 continue;
1053 }
1054 if (MO.isReg() && MO.isDef()) {
1055 if (!Ops)
1056 return MO.getReg();
1057 DefReg = MO.getReg();
1058 } else if (Ops) {
1059 Ops->push_back(Elt: Idx);
1060 }
1061 }
1062 return DefReg;
1063}
1064
1065static void
1066insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1067 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
1068 const SmallVector<MachineInstr *> &ToProcess) {
1069 MachineRegisterInfo &MRI = MF.getRegInfo();
1070 Register AsmTargetReg;
1071 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
1072 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
1073 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
1074 MIRBuilder.setInsertPt(MBB&: *I2->getParent(), II: *I2);
1075
1076 if (!AsmTargetReg.isValid()) {
1077 // define vendor specific assembly target or dialect
1078 AsmTargetReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1079 MRI.setRegClass(Reg: AsmTargetReg, RC: &SPIRV::iIDRegClass);
1080 auto AsmTargetMIB =
1081 MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmTargetINTEL).addDef(RegNo: AsmTargetReg);
1082 addStringImm(Str: ST.getTargetTripleAsStr(), MIB&: AsmTargetMIB);
1083 GR->add(Obj: AsmTargetMIB.getInstr(), MI: AsmTargetMIB);
1084 }
1085
1086 // create types
1087 const MDNode *IAMD = I1->getOperand(i: 1).getMetadata();
1088 FunctionType *FTy = cast<FunctionType>(Val: getMDOperandAsType(N: IAMD, I: 0));
1089 SmallVector<SPIRVTypeInst, 4> ArgTypes;
1090 for (const auto &ArgTy : FTy->params())
1091 ArgTypes.push_back(Elt: GR->getOrCreateSPIRVType(
1092 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true));
1093 SPIRVTypeInst RetType =
1094 GR->getOrCreateSPIRVType(Type: FTy->getReturnType(), MIRBuilder,
1095 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1096 SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
1097 Ty: FTy, RetType, ArgTypes, MIRBuilder);
1098
1099 // define vendor specific assembly instructions string
1100 Register AsmReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1101 MRI.setRegClass(Reg: AsmReg, RC: &SPIRV::iIDRegClass);
1102 auto AsmMIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmINTEL)
1103 .addDef(RegNo: AsmReg)
1104 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1105 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncType))
1106 .addUse(RegNo: AsmTargetReg);
1107 // inline asm string:
1108 addStringImm(Str: I2->getOperand(i: InlineAsm::MIOp_AsmString).getSymbolName(),
1109 MIB&: AsmMIB);
1110 // inline asm constraint string:
1111 addStringImm(Str: cast<MDString>(Val: I1->getOperand(i: 2).getMetadata()->getOperand(I: 0))
1112 ->getString(),
1113 MIB&: AsmMIB);
1114 GR->add(Obj: AsmMIB.getInstr(), MI: AsmMIB);
1115
1116 // calls the inline assembly instruction
1117 unsigned ExtraInfo = I2->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
1118 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
1119 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
1120 .addUse(RegNo: AsmReg)
1121 .addImm(Val: static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
1122
1123 Register DefReg = collectInlineAsmInstrOperands(MI: I2);
1124 if (!DefReg.isValid()) {
1125 DefReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1126 MRI.setRegClass(Reg: DefReg, RC: &SPIRV::iIDRegClass);
1127 SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType(
1128 Type: Type::getVoidTy(C&: MF.getFunction().getContext()), MIRBuilder,
1129 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1130 GR->assignSPIRVTypeToVReg(Type: VoidType, VReg: DefReg, MF);
1131 }
1132
1133 auto AsmCall = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmCallINTEL)
1134 .addDef(RegNo: DefReg)
1135 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1136 .addUse(RegNo: AsmReg);
1137 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
1138 AsmCall.addUse(RegNo: I1->getOperand(i: IntrIdx).getReg());
1139
1140 // IRTranslator gets a bit confused when lowering inline ASM with outputs
1141 // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
1142 // we have to clean that up here to prevent erroneous trunc casts either on
1143 // a struct (for multiple outputs) or same width integers to get lowered
1144 // into SPIR-V
1145 if (MRI.hasOneUse(RegNo: DefReg)) {
1146 MachineInstr &CopyMI = *MRI.use_instr_begin(RegNo: DefReg);
1147 if (CopyMI.getOpcode() == TargetOpcode::COPY) {
1148 Register CopyDst = CopyMI.getOperand(i: 0).getReg();
1149 if (MRI.hasOneUse(RegNo: CopyDst)) {
1150 MachineInstr &TruncMI = *MRI.use_instr_begin(RegNo: CopyDst);
1151 if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
1152 MRI.setType(VReg: DefReg, Ty: GR->getRegType(SpvType: RetType));
1153 Register TruncReg = TruncMI.defs().begin()->getReg();
1154 MRI.replaceRegWith(FromReg: TruncReg, ToReg: DefReg);
1155 invalidateAndEraseMI(GR, MI: &TruncMI);
1156 invalidateAndEraseMI(GR, MI: &CopyMI);
1157 }
1158 }
1159 }
1160 }
1161 }
1162 for (MachineInstr *MI : ToProcess)
1163 invalidateAndEraseMI(GR, MI);
1164}
1165
1166static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1167 const SPIRVSubtarget &ST,
1168 MachineIRBuilder MIRBuilder) {
1169 SmallVector<MachineInstr *> ToProcess;
1170 for (MachineBasicBlock &MBB : MF) {
1171 for (MachineInstr &MI : MBB) {
1172 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_inline_asm) ||
1173 MI.getOpcode() == TargetOpcode::INLINEASM)
1174 ToProcess.push_back(Elt: &MI);
1175 }
1176 }
1177 if (ToProcess.size() == 0)
1178 return;
1179
1180 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly))
1181 report_fatal_error(reason: "Inline assembly instructions require the "
1182 "following SPIR-V extension: SPV_INTEL_inline_assembly",
1183 gen_crash_diag: false);
1184
1185 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
1186}
1187
1188static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1189 MachineIRBuilder MIB) {
1190 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MIB.getMF().getSubtarget());
1191 SmallVector<MachineInstr *, 10> ToErase;
1192 for (MachineBasicBlock &MBB : MF) {
1193 for (MachineInstr &MI : MBB) {
1194 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration) &&
1195 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_aliasing_decoration) &&
1196 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration))
1197 continue;
1198 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getNextNode());
1199 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration)) {
1200 buildOpSpirvDecorations(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1201 GVarMD: MI.getOperand(i: 2).getMetadata(), ST);
1202 } else if (isSpvIntrinsic(MI,
1203 IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration)) {
1204 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(
1205 MD: MI.getOperand(i: 2).getMetadata()->getOperand(I: 0));
1206 uint32_t OpValue = OpV->getValueAPF().bitcastToAPInt().getZExtValue();
1207
1208 buildOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1209 Dec: SPIRV::Decoration::FPMaxErrorDecorationINTEL,
1210 DecArgs: {OpValue});
1211 } else {
1212 GR->buildMemAliasingOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1213 Dec: MI.getOperand(i: 2).getImm(),
1214 GVarMD: MI.getOperand(i: 3).getMetadata());
1215 }
1216
1217 ToErase.push_back(Elt: &MI);
1218 }
1219 }
1220 for (MachineInstr *MI : ToErase)
1221 invalidateAndEraseMI(GR, MI);
1222}
1223
1224// Returns the value of the switch case operand in Reg. The case value stays a
1225// G_CONSTANT until the module emits a SPIR-V constant for the same value, at
1226// which point the case register is replaced with the one defining that
1227// constant, which keeps its value in literal operands rather than in a CImm.
1228static const ConstantInt *getSwitchCaseValue(Register Reg,
1229 const MachineRegisterInfo &MRI,
1230 LLVMContext &Ctx) {
1231 APInt Val;
1232 if (mi_match(R: Reg, MRI, P: m_ICst(Cst&: Val)))
1233 return ConstantInt::get(Context&: Ctx, V: Val);
1234
1235 const MachineInstr *Def = nullptr;
1236 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
1237 llvm_unreachable("Switch case operand has no definition");
1238
1239 LLT Ty = MRI.getType(Reg);
1240 assert(Ty.isValid() && "Expected a typed switch case value");
1241 Val = APInt(Ty.getScalarSizeInBits(), 0);
1242
1243 switch (Def->getOpcode()) {
1244 case SPIRV::OpConstantNull:
1245 case SPIRV::OpConstantI:
1246 // The operands after the type are 32-bit literal words, least significant
1247 // first, as written by addNumImm(). OpConstantNull carries none, so it
1248 // decodes to zero without a case of its own.
1249 for (unsigned I = 2, E = Def->getNumExplicitOperands(); I != E; ++I) {
1250 uint32_t Word = static_cast<uint32_t>(Def->getOperand(i: I).getImm());
1251 Val |= APInt(Val.getBitWidth(), Word).shl(shiftAmt: (I - 2) * 32);
1252 }
1253 break;
1254 default:
1255 llvm_unreachable("Unexpected definition of a switch case value");
1256 }
1257 return ConstantInt::get(Context&: Ctx, V: Val);
1258}
1259
1260// LLVM allows the switches to use registers as cases, while SPIR-V required
1261// those to be immediate values. This function replaces such operands with the
1262// equivalent immediate constant.
1263static void processSwitchesConstants(MachineFunction &MF,
1264 SPIRVGlobalRegistry *GR,
1265 MachineIRBuilder MIB) {
1266 MachineRegisterInfo &MRI = MF.getRegInfo();
1267 LLVMContext &Ctx = MF.getFunction().getContext();
1268 for (MachineBasicBlock &MBB : MF) {
1269 for (MachineInstr &MI : MBB) {
1270 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch))
1271 continue;
1272
1273 SmallVector<MachineOperand, 8> NewOperands;
1274 NewOperands.push_back(Elt: MI.getOperand(i: 0)); // Opcode
1275 NewOperands.push_back(Elt: MI.getOperand(i: 1)); // Condition
1276 NewOperands.push_back(Elt: MI.getOperand(i: 2)); // Default
1277 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
1278 Register Reg = MI.getOperand(i).getReg();
1279 NewOperands.push_back(
1280 Elt: MachineOperand::CreateCImm(CI: getSwitchCaseValue(Reg, MRI, Ctx)));
1281
1282 NewOperands.push_back(Elt: MI.getOperand(i: i + 1));
1283 }
1284
1285 assert(MI.getNumOperands() == NewOperands.size());
1286 while (MI.getNumOperands() > 0)
1287 MI.removeOperand(OpNo: 0);
1288 for (auto &MO : NewOperands)
1289 MI.addOperand(Op: MO);
1290 }
1291 }
1292}
1293
1294// Some instructions are used during CodeGen but should never be emitted.
1295// Cleaning up those.
1296static void cleanupHelperInstructions(MachineFunction &MF,
1297 SPIRVGlobalRegistry *GR) {
1298 SmallVector<MachineInstr *, 8> ToEraseMI;
1299 for (MachineBasicBlock &MBB : MF) {
1300 for (MachineInstr &MI : MBB) {
1301 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant) ||
1302 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
1303 ToEraseMI.push_back(Elt: &MI);
1304 }
1305 }
1306
1307 for (MachineInstr *MI : ToEraseMI)
1308 invalidateAndEraseMI(GR, MI);
1309}
1310
1311// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
1312// operands/registers by the actual MBB it references.
1313static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1314 MachineIRBuilder MIB) {
1315 // Gather the reverse-mapping BB -> MBB.
1316 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
1317 for (MachineBasicBlock &MBB : MF)
1318 BB2MBB[MBB.getBasicBlock()] = &MBB;
1319
1320 // Gather instructions requiring patching. For now, only those can use
1321 // G_BLOCK_ADDR.
1322 SmallVector<MachineInstr *, 8> InstructionsToPatch;
1323 for (MachineBasicBlock &MBB : MF) {
1324 for (MachineInstr &MI : MBB) {
1325 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch) ||
1326 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_loop_merge) ||
1327 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_selection_merge))
1328 InstructionsToPatch.push_back(Elt: &MI);
1329 }
1330 }
1331
1332 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
1333 // the actual MBB it references. Once those references have been updated, we
1334 // can cleanup remaining G_BLOCK_ADDR references.
1335 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
1336 SmallPtrSet<MachineInstr *, 8> ToEraseMI;
1337 MachineRegisterInfo &MRI = MF.getRegInfo();
1338 for (MachineInstr *MI : InstructionsToPatch) {
1339 SmallVector<MachineOperand, 8> NewOps;
1340 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1341 // The operand is not a register, keep as-is.
1342 if (!MI->getOperand(i).isReg()) {
1343 NewOps.push_back(Elt: MI->getOperand(i));
1344 continue;
1345 }
1346
1347 Register Reg = MI->getOperand(i).getReg();
1348 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
1349 // The register is not the result of G_BLOCK_ADDR, keep as-is.
1350 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
1351 NewOps.push_back(Elt: MI->getOperand(i));
1352 continue;
1353 }
1354
1355 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
1356 BuildMBB->getOperand(1).isBlockAddress() &&
1357 BuildMBB->getOperand(1).getBlockAddress());
1358 BasicBlock *BB =
1359 BuildMBB->getOperand(i: 1).getBlockAddress()->getBasicBlock();
1360 auto It = BB2MBB.find(Val: BB);
1361 if (It == BB2MBB.end())
1362 report_fatal_error(reason: "cannot find a machine basic block by a basic block "
1363 "in a switch statement");
1364 MachineBasicBlock *ReferencedBlock = It->second;
1365 NewOps.push_back(Elt: MachineOperand::CreateMBB(MBB: ReferencedBlock));
1366
1367 ClearAddressTaken.insert(Ptr: ReferencedBlock);
1368 ToEraseMI.insert(Ptr: BuildMBB);
1369 }
1370
1371 // Replace the operands.
1372 assert(MI->getNumOperands() == NewOps.size());
1373 while (MI->getNumOperands() > 0)
1374 MI->removeOperand(OpNo: 0);
1375 for (auto &MO : NewOps)
1376 MI->addOperand(Op: MO);
1377
1378 if (MachineInstr *Next = MI->getNextNode()) {
1379 if (isSpvIntrinsic(MI: *Next, IntrinsicID: Intrinsic::spv_track_constant)) {
1380 ToEraseMI.insert(Ptr: Next);
1381 Next = MI->getNextNode();
1382 }
1383 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
1384 ToEraseMI.insert(Ptr: Next);
1385 }
1386 }
1387
1388 // BlockAddress operands were used to keep information between passes,
1389 // let's undo the "address taken" status to reflect that Succ doesn't
1390 // actually correspond to an IR-level basic block.
1391 for (MachineBasicBlock *Succ : ClearAddressTaken)
1392 Succ->setAddressTakenIRBlock(nullptr);
1393
1394 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
1395 // this leaves their BasicBlock counterparts in a "address taken" status. This
1396 // would make AsmPrinter to generate a series of unneeded labels of a "Address
1397 // of block that was removed by CodeGen" kind. Let's first ensure that we
1398 // don't have a dangling BlockAddress constants by zapping the BlockAddress
1399 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
1400 Constant *Replacement =
1401 ConstantInt::get(Ty: Type::getInt32Ty(C&: MF.getFunction().getContext()), V: 1);
1402 for (MachineInstr *BlockAddrI : ToEraseMI) {
1403 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
1404 BlockAddress *BA = const_cast<BlockAddress *>(
1405 BlockAddrI->getOperand(i: 1).getBlockAddress());
1406 BA->replaceAllUsesWith(
1407 V: ConstantExpr::getIntToPtr(C: Replacement, Ty: BA->getType()));
1408 BA->destroyConstant();
1409 }
1410 invalidateAndEraseMI(GR, MI: BlockAddrI);
1411 }
1412}
1413
1414static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
1415 if (MBB.empty())
1416 return MBB.getNextNode() != nullptr;
1417
1418 // Branching SPIR-V intrinsics are not detected by this generic method.
1419 // Thus, we can only trust negative result.
1420 if (!MBB.canFallThrough())
1421 return false;
1422
1423 // Otherwise, we must manually check if we have a SPIR-V intrinsic which
1424 // prevent an implicit fallthrough.
1425 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
1426 It != E; ++It) {
1427 if (isSpvIntrinsic(MI: *It, IntrinsicID: Intrinsic::spv_switch))
1428 return false;
1429 }
1430 return true;
1431}
1432
1433static void removeImplicitFallthroughs(MachineFunction &MF,
1434 MachineIRBuilder MIB) {
1435 // It is valid for MachineBasicBlocks to not finish with a branch instruction.
1436 // In such cases, they will simply fallthrough their immediate successor.
1437 for (MachineBasicBlock &MBB : MF) {
1438 if (!isImplicitFallthrough(MBB))
1439 continue;
1440
1441 assert(MBB.succ_size() == 1);
1442 MIB.setInsertPt(MBB, II: MBB.end());
1443 MIB.buildBr(Dest&: **MBB.successors().begin());
1444 }
1445}
1446
1447static bool runPreLegalizer(MachineFunction &MF) {
1448 // Initialize the type registry.
1449 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
1450 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1451 GR->setCurrentFunc(MF);
1452 MachineIRBuilder MIB(MF);
1453 // a registry of target extension constants
1454 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
1455 // to keep record of tracked constants
1456 addConstantsToTrack(MF, GR, STI: ST, TargetExtConstTypes);
1457 foldConstantsIntoIntrinsics(MF, GR, MIB);
1458 insertBitcasts(MF, GR, MIB);
1459 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
1460
1461 processSwitchesConstants(MF, GR, MIB);
1462 processBlockAddr(MF, GR, MIB);
1463 cleanupHelperInstructions(MF, GR);
1464
1465 processInstrsWithTypeFolding(MF, GR, MIB);
1466 removeImplicitFallthroughs(MF, MIB);
1467 insertSpirvDecorations(MF, GR, MIB);
1468 insertInlineAsm(MF, GR, ST, MIRBuilder: MIB);
1469 lowerBitcasts(MF, GR, MIB);
1470
1471 return true;
1472}
1473
1474INITIALIZE_PASS(SPIRVPreLegalizerLegacy, DEBUG_TYPE, "SPIRV pre legalizer",
1475 false, false)
1476
1477char SPIRVPreLegalizerLegacy::ID = 0;
1478
1479FunctionPass *llvm::createSPIRVPreLegalizerLegacyPass() {
1480 return new SPIRVPreLegalizerLegacy();
1481}
1482
1483bool SPIRVPreLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
1484 return runPreLegalizer(MF);
1485}
1486
1487PreservedAnalyses
1488SPIRVPreLegalizerPass::run(MachineFunction &MF,
1489 MachineFunctionAnalysisManager &MFAM) {
1490 bool Changed = runPreLegalizer(MF);
1491 if (!Changed)
1492 return PreservedAnalyses::all();
1493
1494 return getMachineFunctionPassPreservedAnalyses()
1495 .preserve<GISelValueTrackingAnalysis>();
1496}
1497