1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass prepares IR for legalization: it assigns SPIR-V types to registers
10// and removes intrinsics which holded these types during IR translation.
11// Also it processes constants and registers them in GR to avoid duplication.
12//
13//===----------------------------------------------------------------------===//
14
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
23#include "llvm/CodeGen/MachinePassManager.h"
24#include "llvm/IR/Analysis.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/InstrTypes.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/Support/MathExtras.h"
30
31#define DEBUG_TYPE "spirv-prelegalizer"
32
33using namespace llvm;
34using namespace llvm::MIPatternMatch;
35
36namespace {
37class SPIRVPreLegalizerLegacy : public MachineFunctionPass {
38public:
39 static char ID;
40 SPIRVPreLegalizerLegacy() : MachineFunctionPass(ID) {}
41 bool runOnMachineFunction(MachineFunction &MF) override;
42 void getAnalysisUsage(AnalysisUsage &AU) const override;
43};
44} // namespace
45
46void SPIRVPreLegalizerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
47 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
48 MachineFunctionPass::getAnalysisUsage(AU);
49}
50
51static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
52 MachineInstr *MI) {
53 GR->invalidateMachineInstr(MI);
54 MI->eraseFromParent();
55}
56
57static void
58addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
59 const SPIRVSubtarget &STI,
60 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
61 MachineRegisterInfo &MRI = MF.getRegInfo();
62 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
63 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
64 for (MachineBasicBlock &MBB : MF) {
65 for (MachineInstr &MI : MBB) {
66 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant))
67 continue;
68 ToErase.push_back(Elt: &MI);
69 Register SrcReg = MI.getOperand(i: 2).getReg();
70 auto *Const =
71 cast<Constant>(Val: cast<ConstantAsMetadata>(
72 Val: MI.getOperand(i: 3).getMetadata()->getOperand(I: 0))
73 ->getValue());
74 if (auto *GV = dyn_cast<GlobalValue>(Val: Const)) {
75 Register Reg = GR->find(V: GV, MF: &MF);
76 if (!Reg.isValid()) {
77 GR->add(V: GV, MI: MRI.getVRegDef(Reg: SrcReg));
78 GR->addGlobalObject(V: GV, MF: &MF, R: SrcReg);
79 } else
80 RegsAlreadyAddedToDT[&MI] = Reg;
81 } else {
82 Register Reg = GR->find(V: Const, MF: &MF);
83 if (!Reg.isValid()) {
84 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Val: Const)) {
85 auto *BuildVec = MRI.getVRegDef(Reg: SrcReg);
86 assert(BuildVec &&
87 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
88 GR->add(V: Const, MI: BuildVec);
89 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
90 // Ensure that OpConstantComposite reuses a constant when it's
91 // already created and available in the same machine function.
92 Constant *ElemConst = ConstVec->getElementAsConstant(i);
93 Register ElemReg = GR->find(V: ElemConst, MF: &MF);
94 if (!ElemReg.isValid())
95 GR->add(V: ElemConst,
96 MI: MRI.getVRegDef(Reg: BuildVec->getOperand(i: 1 + i).getReg()));
97 else
98 BuildVec->getOperand(i: 1 + i).setReg(ElemReg);
99 }
100 }
101 if (Const->getType()->isTargetExtTy()) {
102 // remember association so that we can restore it when assign types
103 MachineInstr *SrcMI = MRI.getVRegDef(Reg: SrcReg);
104 if (SrcMI)
105 GR->add(V: Const, MI: SrcMI);
106 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
107 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
108 TargetExtConstTypes[SrcMI] = Const->getType();
109 if (Const->isNullValue()) {
110 MachineBasicBlock &DepMBB = MF.front();
111 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());
112 SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType(
113 Type: Const->getType(), MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
114 EmitIR: true);
115 assert(SrcMI && "Expected source instruction to be valid");
116 SrcMI->setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
117 SrcMI->addOperand(Op: MachineOperand::CreateReg(
118 Reg: GR->getSPIRVTypeID(SpirvType: ExtType), isDef: false));
119 }
120 }
121 } else {
122 RegsAlreadyAddedToDT[&MI] = Reg;
123 // This MI is unused and will be removed. If the MI uses
124 // const_composite, it will be unused and should be removed too.
125 assert(MI.getOperand(2).isReg() && "Reg operand is expected");
126 MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
127 if (SrcMI && isSpvIntrinsic(MI: *SrcMI, IntrinsicID: Intrinsic::spv_const_composite))
128 ToEraseComposites.push_back(Elt: SrcMI);
129 }
130 }
131 }
132 }
133 for (MachineInstr *MI : ToErase) {
134 Register Reg = MI->getOperand(i: 2).getReg();
135 auto It = RegsAlreadyAddedToDT.find(Val: MI);
136 if (It != RegsAlreadyAddedToDT.end())
137 Reg = It->second;
138 auto *RC = MRI.getRegClassOrNull(Reg: MI->getOperand(i: 0).getReg());
139 if (!MRI.getRegClassOrNull(Reg) && RC)
140 MRI.setRegClass(Reg, RC);
141 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: Reg);
142 invalidateAndEraseMI(GR, MI);
143 }
144 for (MachineInstr *MI : ToEraseComposites)
145 invalidateAndEraseMI(GR, MI);
146}
147
148static void foldConstantsIntoIntrinsics(MachineFunction &MF,
149 SPIRVGlobalRegistry *GR,
150 MachineIRBuilder MIB) {
151 SmallVector<MachineInstr *, 64> ToErase;
152 for (MachineBasicBlock &MBB : MF) {
153 for (MachineInstr &MI : MBB) {
154 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_name))
155 continue;
156 const MDNode *MD = MI.getOperand(i: 2).getMetadata();
157 StringRef ValueName = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
158 if (ValueName.size() > 0) {
159 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
160 buildOpName(Target: MI.getOperand(i: 1).getReg(), Name: ValueName, MIRBuilder&: MIB);
161 }
162 ToErase.push_back(Elt: &MI);
163 }
164 for (MachineInstr *MI : ToErase)
165 invalidateAndEraseMI(GR, MI);
166 ToErase.clear();
167 }
168}
169
170static MachineInstr *findAssignTypeInstr(Register Reg,
171 MachineRegisterInfo *MRI) {
172 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(RegNo: Reg),
173 IE = MRI->use_instr_end();
174 I != IE; ++I) {
175 MachineInstr *UseMI = &*I;
176 if ((isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_ptr_type) ||
177 isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_type)) &&
178 UseMI->getOperand(i: 1).getReg() == Reg)
179 return UseMI;
180 }
181 return nullptr;
182}
183
184static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
185 Register ResVReg, Register OpReg) {
186 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResVReg);
187 SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg);
188 assert(ResType && OpType && "Operand types are expected");
189 if (!GR->isBitcastCompatible(Type1: ResType, Type2: OpType))
190 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
191 MachineRegisterInfo *MRI = MIB.getMRI();
192 if (!MRI->getRegClassOrNull(Reg: ResVReg))
193 MRI->setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: ResType));
194 if (ResType == OpType)
195 MIB.buildInstr(Opcode: TargetOpcode::COPY).addDef(RegNo: ResVReg).addUse(RegNo: OpReg);
196 else
197 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
198 .addDef(RegNo: ResVReg)
199 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResType))
200 .addUse(RegNo: OpReg);
201}
202
203// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.
204// The verifier checks if the source and destination LLTs of a G_BITCAST are
205// different, but this check is too strict for SPIR-V's typed pointers, which
206// may have the same LLT but different SPIRV type (e.g. pointers to different
207// pointee types). By lowering to OpBitcast here, we bypass the verifier's
208// check. See discussion in https://github.com/llvm/llvm-project/pull/110270
209// for more context.
210//
211// We also handle the llvm.spv.bitcast intrinsic here. If the source and
212// destination SPIR-V types are the same, we lower it to a COPY to enable
213// further optimizations like copy propagation.
214static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
215 MachineIRBuilder MIB) {
216 SmallVector<MachineInstr *, 16> ToErase;
217 for (MachineBasicBlock &MBB : MF) {
218 for (MachineInstr &MI : MBB) {
219 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_bitcast)) {
220 Register DstReg = MI.getOperand(i: 0).getReg();
221 Register SrcReg = MI.getOperand(i: 2).getReg();
222 SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(VReg: DstReg);
223 assert(
224 DstType &&
225 "Expected destination SPIR-V type to have been assigned already.");
226 SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(VReg: SrcReg);
227 assert(SrcType &&
228 "Expected source SPIR-V type to have been assigned already.");
229 if (DstType == SrcType) {
230 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
231 MIB.buildCopy(Res: DstReg, Op: SrcReg);
232 ToErase.push_back(Elt: &MI);
233 continue;
234 }
235 }
236
237 if (MI.getOpcode() != TargetOpcode::G_BITCAST)
238 continue;
239
240 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
241 buildOpBitcast(GR, MIB, ResVReg: MI.getOperand(i: 0).getReg(),
242 OpReg: MI.getOperand(i: 1).getReg());
243 ToErase.push_back(Elt: &MI);
244 }
245 }
246 for (MachineInstr *MI : ToErase)
247 invalidateAndEraseMI(GR, MI);
248}
249
250static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
251 MachineIRBuilder MIB) {
252 // Get access to information about available extensions
253 const SPIRVSubtarget *ST =
254 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
255 SmallVector<MachineInstr *, 10> ToErase;
256 for (MachineBasicBlock &MBB : MF) {
257 for (MachineInstr &MI : MBB) {
258 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_ptrcast))
259 continue;
260 assert(MI.getOperand(2).isReg());
261 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
262 ToErase.push_back(Elt: &MI);
263 Register Def = MI.getOperand(i: 0).getReg();
264 Register Source = MI.getOperand(i: 2).getReg();
265 Type *ElemTy = getMDOperandAsType(N: MI.getOperand(i: 3).getMetadata(), I: 0);
266 auto SC =
267 isa<FunctionType>(Val: ElemTy) &&
268 ST->canUseExtension(
269 E: SPIRV::Extension::SPV_INTEL_function_pointers)
270 ? SPIRV::StorageClass::CodeSectionINTEL
271 : addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 4).getImm(), STI: *ST);
272 SPIRVTypeInst AssignedPtrType =
273 GR->getOrCreateSPIRVPointerType(BaseType: ElemTy, I&: MI, SC);
274
275 // If the ptrcast would be redundant, replace all uses with the source
276 // register.
277 MachineRegisterInfo *MRI = MIB.getMRI();
278 // For untyped pointers the SPIR-V pointer type does not encode the
279 // pointee, so two pointers with different element types share the same
280 // pointer type. The element type still matters because it selects the
281 // Base Type operand of OpUntyped*AccessChainKHR. Treat the cast as
282 // redundant only when the source already carries the same element type.
283 // Otherwise keep a distinct register so the element type is preserved.
284 bool Redundant =
285 AssignedPtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR
286 ? GR->getUntypedPtrElementType(Reg: Source) ==
287 GR->getOrCreateSPIRVType(Type: ElemTy, MIRBuilder&: MIB,
288 AQ: SPIRV::AccessQualifier::ReadWrite,
289 /*EmitIR=*/true)
290 : GR->getSPIRVTypeForVReg(VReg: Source) == AssignedPtrType;
291 if (Redundant) {
292 // Erase Def's assign type instruction if we are going to replace Def.
293 if (MachineInstr *AssignMI = findAssignTypeInstr(Reg: Def, MRI))
294 ToErase.push_back(Elt: AssignMI);
295 MRI->replaceRegWith(FromReg: Def, ToReg: Source);
296 } else {
297 if (!GR->getSPIRVTypeForVReg(VReg: Def, MF: &MF))
298 GR->assignSPIRVTypeToVReg(Type: AssignedPtrType, VReg: Def, MF);
299 MIB.buildBitcast(Dst: Def, Src: Source);
300 }
301 }
302 }
303 for (MachineInstr *MI : ToErase)
304 invalidateAndEraseMI(GR, MI);
305}
306
307// Translating GV, IRTranslator sometimes generates following IR:
308// %1 = G_GLOBAL_VALUE
309// %2 = COPY %1
310// %3 = G_ADDRSPACE_CAST %2
311//
312// or
313//
314// %1 = G_ZEXT %2
315// G_MEMCPY ... %2 ...
316//
317// New registers have no SPIRV type and no register class info.
318//
319// Set SPIRV type for GV, propagate it from GV to other instructions,
320// also set register classes.
321static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
322 SPIRVGlobalRegistry *GR,
323 MachineRegisterInfo &MRI,
324 MachineIRBuilder &MIB) {
325 SPIRVTypeInst SpvType = nullptr;
326 assert(MI && "Machine instr is expected");
327 if (MI->getOperand(i: 0).isReg()) {
328 Register Reg = MI->getOperand(i: 0).getReg();
329 SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
330 if (!SpvType) {
331 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
332 switch (MI->getOpcode()) {
333 case TargetOpcode::G_FCONSTANT:
334 case TargetOpcode::G_CONSTANT: {
335 Type *Ty = MI->getOperand(i: 1).getCImm()->getType();
336 SpvType = GR->getOrCreateSPIRVType(
337 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
338 break;
339 }
340 case TargetOpcode::G_GLOBAL_VALUE: {
341 const GlobalValue *Global = MI->getOperand(i: 1).getGlobal();
342 Type *ElementTy = toTypedPointer(Ty: GR->getDeducedGlobalValueType(Global));
343 unsigned AddrSpace = Global->getType()->getAddressSpace();
344 // Function pointers use CodeSectionINTEL storage class in SPIR-V when
345 // the SPV_INTEL_function_pointers extension is enabled.
346 const SPIRVSubtarget &ST = MIB.getMF().getSubtarget<SPIRVSubtarget>();
347 if (isa<Function>(Val: Global) &&
348 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
349 AddrSpace =
350 storageClassToAddressSpace(SC: SPIRV::StorageClass::CodeSectionINTEL);
351 auto *Ty = TypedPointerType::get(ElementType: ElementTy, AddressSpace: AddrSpace);
352 SpvType = GR->getOrCreateSPIRVType(
353 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
354 break;
355 }
356 case TargetOpcode::G_ANYEXT:
357 case TargetOpcode::G_SEXT:
358 case TargetOpcode::G_ZEXT: {
359 if (MI->getOperand(i: 1).isReg()) {
360 if (MachineInstr *DefInstr =
361 MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
362 if (SPIRVTypeInst Def =
363 propagateSPIRVType(MI: DefInstr, GR, MRI, MIB)) {
364 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Type: Def);
365 unsigned ExpectedBW =
366 std::max(a: MRI.getType(Reg).getScalarSizeInBits(), b: CurrentBW);
367 unsigned NumElements = GR->getScalarOrVectorComponentCount(Type: Def);
368 SpvType = GR->getOrCreateSPIRVIntegerType(BitWidth: ExpectedBW, MIRBuilder&: MIB);
369 if (NumElements > 1)
370 SpvType = GR->getOrCreateSPIRVVectorType(BaseType: SpvType, NumElements,
371 MIRBuilder&: MIB, EmitIR: true);
372 }
373 }
374 }
375 break;
376 }
377 case TargetOpcode::G_PTRTOINT:
378 SpvType = GR->getOrCreateSPIRVIntegerType(
379 BitWidth: MRI.getType(Reg).getScalarSizeInBits(), MIRBuilder&: MIB);
380 break;
381 case TargetOpcode::G_TRUNC:
382 case TargetOpcode::G_ADDRSPACE_CAST:
383 case TargetOpcode::G_PTR_ADD:
384 case TargetOpcode::COPY: {
385 MachineOperand &Op = MI->getOperand(i: 1);
386 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Reg: Op.getReg()) : nullptr;
387 if (Def)
388 SpvType = propagateSPIRVType(MI: Def, GR, MRI, MIB);
389 break;
390 }
391 default:
392 break;
393 }
394 if (SpvType) {
395 // check if the address space needs correction
396 LLT RegType = MRI.getType(Reg);
397 if (SpvType.isPointer() && RegType.isPointer() &&
398 storageClassToAddressSpace(SC: GR->getPointerStorageClass(Type: SpvType)) !=
399 RegType.getAddressSpace()) {
400 // Don't correct CodeSectionINTEL back to Function for function
401 // pointer G_GLOBAL_VALUE - the LLVM register has address space 0
402 // but the SPIR-V type was intentionally set to CodeSectionINTEL.
403 bool SkipCorrection =
404 MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
405 GR->getPointerStorageClass(Type: SpvType) ==
406 SPIRV::StorageClass::CodeSectionINTEL;
407 if (!SkipCorrection) {
408 const SPIRVSubtarget &ST =
409 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
410 auto TSC =
411 addressSpaceToStorageClass(AddrSpace: RegType.getAddressSpace(), STI: ST);
412 SpvType = GR->changePointerStorageClass(PtrType: SpvType, SC: TSC, I&: *MI);
413 }
414 }
415 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
416 }
417 if (!MRI.getRegClassOrNull(Reg))
418 MRI.setRegClass(Reg, RC: SpvType ? GR->getRegClass(SpvType)
419 : &SPIRV::iIDRegClass);
420 }
421 }
422 return SpvType;
423}
424
425// To support current approach and limitations wrt. bit width here we widen a
426// scalar register with a bit width greater than 1 to valid sizes and cap it to
427// 128 width.
428static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {
429 if (BitWidth == 1)
430 return 1; // No need to widen 1-bit values
431 return std::min(a: std::max<unsigned>(a: PowerOf2Ceil(A: BitWidth), b: 8u), b: 128u);
432}
433
434static std::optional<unsigned>
435getNarrowScalarWidth(Register Reg, const MachineRegisterInfo &MRI) {
436 LLT Ty = MRI.getType(Reg);
437 if (!Ty.isScalar())
438 return std::nullopt;
439 unsigned W = Ty.getScalarSizeInBits();
440 // <= and not == because widenBitWidthToNextPow2 caps at 128.
441 if (widenBitWidthToNextPow2(BitWidth: W) <= W)
442 return std::nullopt;
443 return W;
444}
445
446static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {
447 LLT RegType = MRI.getType(Reg);
448 if (!RegType.isScalar())
449 return;
450 unsigned CurrentWidth = RegType.getScalarSizeInBits();
451 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
452 if (NewWidth != CurrentWidth)
453 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: NewWidth));
454}
455
456static void widenCImmType(MachineOperand &MOP) {
457 const ConstantInt *CImmVal = MOP.getCImm();
458 unsigned CurrentWidth = CImmVal->getBitWidth();
459 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
460 if (NewWidth != CurrentWidth) {
461 // Replace the immediate value with the widened version
462 MOP.setCImm(ConstantInt::get(Context&: CImmVal->getType()->getContext(),
463 V: CImmVal->getValue().zextOrTrunc(width: NewWidth)));
464 }
465}
466
467static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
468 MachineBasicBlock &MBB = *Def->getParent();
469 MachineBasicBlock::iterator DefIt =
470 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();
471 // Skip all the PHI and debug instructions.
472 while (DefIt != MBB.end() &&
473 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))
474 DefIt = std::next(x: DefIt);
475 MIB.setInsertPt(MBB, II: DefIt);
476}
477
478namespace llvm {
479void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType,
480 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
481 MachineRegisterInfo &MRI) {
482 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
483 MachineInstr *Def = MRI.getVRegDef(Reg);
484 setInsertPtAfterDef(MIB, Def);
485 if (!SpvType)
486 SpvType = GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder&: MIB,
487 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
488 if (!MRI.getRegClassOrNull(Reg))
489 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType));
490 if (!MRI.getType(Reg).isValid())
491 MRI.setType(VReg: Reg, Ty: GR->getRegType(SpvType));
492 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
493}
494
495void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
496 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
497 SPIRVTypeInst KnownResType) {
498 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
499 for (auto &Op : MI.operands()) {
500 if (!Op.isReg() || Op.isDef())
501 continue;
502 Register OpReg = Op.getReg();
503 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OpReg);
504 if (!SpvType && KnownResType) {
505 SpvType = KnownResType;
506 GR->assignSPIRVTypeToVReg(Type: KnownResType, VReg: OpReg, MF: *MI.getMF());
507 }
508 assert(SpvType);
509 if (!MRI.getRegClassOrNull(Reg: OpReg))
510 MRI.setRegClass(Reg: OpReg, RC: GR->getRegClass(SpvType));
511 if (!MRI.getType(Reg: OpReg).isValid())
512 MRI.setType(VReg: OpReg, Ty: GR->getRegType(SpvType));
513 }
514}
515} // namespace llvm
516
517// Sign-sensitive integer ops: their result depends on the value of the input
518// sign bit at position (width-1). On sub-pow2 widths the general widening
519// loop is a pure LLT relabel, which leaves the sign bit at the *original*
520// position instead of the widened MSB. These ops therefore need an explicit
521// G_SEXT_INREG on each value operand to move the sign bit up.
522//
523// Signed-vs-unsigned G_ICMP is distinguished by its predicate operand.
524//
525// TODO: follow-up PRs will add the remaining sign-sensitive opcodes
526// (e.g. G_SADDSAT/G_SSUBSAT, signed overflow ops).
527static bool isSignSensitiveOp(const MachineInstr &MI) {
528 switch (MI.getOpcode()) {
529 case TargetOpcode::G_ASHR:
530 case TargetOpcode::G_SDIV:
531 case TargetOpcode::G_SREM:
532 case TargetOpcode::G_SITOFP:
533 case TargetOpcode::G_SMIN:
534 case TargetOpcode::G_SMAX:
535 return true;
536 case TargetOpcode::G_ICMP:
537 return CmpInst::isSigned(
538 Pred: static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate()));
539 default:
540 return false;
541 }
542}
543
544struct NarrowWideningInfo {
545 // Width before widening of each sign-sensitive value-operand vreg (one entry
546 // per vreg).
547 DenseMap<Register, unsigned> OrigWidth;
548 // Sign-sensitive ops whose value operand(s) need replacing, ordered for
549 // reproducible vreg numbering.
550 SmallVector<MachineInstr *> SignSensitiveWorklist;
551 // Keyed by instruction, not vreg: G_TRUNC handling can replace the source.
552 SmallVector<std::pair<MachineInstr *, unsigned>> BitCountWorklist;
553};
554
555// G_CTTZ_ZERO_POISON is absent because its low bits are known non-zero, G_CTLS
556// because the backend does not select it.
557static bool isWidthSensitiveBitCountOp(unsigned Opcode) {
558 switch (Opcode) {
559 case TargetOpcode::G_CTLZ:
560 case TargetOpcode::G_CTLZ_ZERO_POISON:
561 case TargetOpcode::G_CTTZ:
562 case TargetOpcode::G_CTPOP:
563 return true;
564 default:
565 return false;
566 }
567}
568
569// Collect ops whose semantics depend on the operand width along with their
570// pre-widening widths, before later passes retype those vregs to pow2 LLTs
571// and the original width is no longer recoverable.
572static NarrowWideningInfo
573recordNarrowOperandWidths(MachineFunction &MF, const MachineRegisterInfo &MRI) {
574 NarrowWideningInfo Info;
575 auto RecordIfNarrow = [&](Register Reg) {
576 std::optional<unsigned> W = getNarrowScalarWidth(Reg, MRI);
577 if (!W)
578 return false;
579 Info.OrigWidth.try_emplace(Key: Reg, Args&: *W);
580 return true;
581 };
582 for (MachineBasicBlock &MBB : MF) {
583 for (MachineInstr &MI : MBB) {
584 if (isWidthSensitiveBitCountOp(Opcode: MI.getOpcode())) {
585 if (std::optional<unsigned> W =
586 getNarrowScalarWidth(Reg: MI.getOperand(i: 1).getReg(), MRI))
587 Info.BitCountWorklist.emplace_back(Args: &MI, Args&: *W);
588 continue;
589 }
590 if (!isSignSensitiveOp(MI))
591 continue;
592 bool NeedsRewrite = false;
593 for (const MachineOperand &MO : MI.all_uses())
594 NeedsRewrite = RecordIfNarrow(MO.getReg()) || NeedsRewrite;
595 if (NeedsRewrite) {
596 Info.SignSensitiveWorklist.push_back(Elt: &MI);
597 // Record the result too so it gets masked.
598 RecordIfNarrow(MI.getOperand(i: 0).getReg());
599 }
600 }
601 }
602 return Info;
603}
604
605// For every recorded sign-sensitive op, insert G_SEXT_INREG on each value
606// operand whose original width was narrower than the widened pow2 width and
607// retype the operand's vreg LLT in place to the widened width.
608//
609// Info must have been populated by recordNarrowOperandWidths before
610// other passes retyped the vregs; otherwise the narrow widths needed here
611// are lost.
612//
613// TODO: handle vector operands.
614static void widenSignSensitiveOps(MachineFunction &MF, SPIRVGlobalRegistry *GR,
615 MachineIRBuilder &MIB,
616 MachineRegisterInfo &MRI,
617 const NarrowWideningInfo &Info) {
618 // Emit G_SEXT_INREG from Reg's recorded narrow width; retypes Reg to the
619 // widened width and returns the sign-extended vreg.
620 auto SignExtendReg = [&](Register Reg, unsigned OldW,
621 MachineInstr &MI) -> Register {
622 unsigned NewW = widenBitWidthToNextPow2(BitWidth: OldW);
623 LLT NewLLT = LLT::scalar(SizeInBits: NewW);
624 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
625 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewW, MIRBuilder&: MIB);
626 Register SExted = MRI.createGenericVirtualRegister(Ty: NewLLT);
627 GR->assignSPIRVTypeToVReg(Type: SpvTy, VReg: SExted, MF);
628 MRI.setRegClass(Reg: SExted, RC: GR->getRegClass(SpvType: SpvTy));
629 MRI.setType(VReg: Reg, Ty: NewLLT);
630 MIB.buildSExtInReg(Res: SExted, Op: Reg, ImmOp: OldW);
631 return SExted;
632 };
633
634 // The wide op yields a sign-extended value, mask it back to OldW bits because
635 // later users assume the upper bits are zero.
636 auto MaskResult = [&](MachineInstr &MI, unsigned OldW) {
637 Register DstReg = MI.getOperand(i: 0).getReg();
638 widenScalarType(Reg: DstReg, MRI);
639 MIB.setInstrAndDebugLoc(MI);
640 SPIRVTypeInst SpvTy =
641 GR->getOrCreateSPIRVIntegerType(BitWidth: widenBitWidthToNextPow2(BitWidth: OldW), MIRBuilder&: MIB);
642 Register Result = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
643 MI.getOperand(i: 0).setReg(Result);
644 setInsertPtAfterDef(MIB, Def: &MI);
645 MIB.buildZExtInReg(Res: DstReg, Op: Result, ImmOp: OldW);
646 };
647
648 // TODO: when the same narrow vreg feeds multiple sign-sensitive ops (e.g.
649 // sdiv %x, %y and srem %x, %y), emit one shared G_SEXT_INREG instead of one
650 // per use.
651 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
652 for (MachineInstr *MI : Info.SignSensitiveWorklist) {
653 for (const MachineOperand &MO : MI->all_uses()) {
654 Register Reg = MO.getReg();
655 auto It = Info.OrigWidth.find(Val: Reg);
656 if (It == Info.OrigWidth.end())
657 continue;
658 // substituteRegister fills every slot holding Reg at once: SignExtendReg
659 // retypes Reg in place, so a second sext would read the widened width.
660 MI->substituteRegister(FromReg: Reg, ToReg: SignExtendReg(Reg, It->second, *MI),
661 /*SubIdx=*/0, RegInfo: TRI);
662 }
663 Register Dst = MI->getOperand(i: 0).getReg();
664 if (auto It = Info.OrigWidth.find(Val: Dst); It != Info.OrigWidth.end())
665 MaskResult(*MI, It->second);
666 }
667}
668
669// LegalizerHelper::widenScalar has the same cases but cannot be reached: the
670// relabel retypes every narrow scalar to a pow2 LLT, so no illegal narrow type
671// ever reaches the legalizer.
672//
673// TODO: handle vector operands.
674static void widenBitCountOps(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
675 MachineRegisterInfo &MRI,
676 const NarrowWideningInfo &Info) {
677 for (auto [MI, OldWidth] : Info.BitCountWorklist) {
678 Register SrcReg = MI->getOperand(i: 1).getReg();
679 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: OldWidth);
680 LLT NewTy = LLT::scalar(SizeInBits: NewWidth);
681 widenScalarType(Reg: SrcReg, MRI);
682 MIB.setInstrAndDebugLoc(*MI);
683 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewWidth, MIRBuilder&: MIB);
684
685 // The G_TRUNC lowering masks its result to the narrow width, so a source
686 // coming from it needs no second mask.
687 APInt Cst;
688 bool HighBitsAlreadyZero =
689 mi_match(R: SrcReg, MRI, P: m_GAnd(L: m_Reg(), R: m_ICst(Cst))) &&
690 Cst.isSubsetOf(RHS: APInt::getLowBitsSet(numBits: Cst.getBitWidth(), loBitsSet: OldWidth));
691 auto ClearHighBits = [&](unsigned Width) -> Register {
692 if (HighBitsAlreadyZero)
693 return SrcReg;
694 Register Masked = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
695 MIB.buildZExtInReg(Res: Masked, Op: SrcReg, ImmOp: Width);
696 return Masked;
697 };
698
699 Register Input;
700 switch (MI->getOpcode()) {
701 case TargetOpcode::G_CTLZ_ZERO_POISON: {
702 // Shifting up to the widened MSB moves the poison out too, so no
703 // adjustment.
704 Input = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
705 auto Diff = MIB.buildConstant(Res: NewTy, Val: NewWidth - OldWidth);
706 MIB.buildShl(Dst: Input, Src0: SrcReg, Src1: Diff);
707 break;
708 }
709 case TargetOpcode::G_CTTZ: {
710 // Keeps an all-zero narrow value counting exactly OldWidth zeros.
711 Input = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
712 auto TopBit =
713 MIB.buildConstant(Res: NewTy, Val: APInt::getOneBitSet(numBits: NewWidth, BitNo: OldWidth));
714 MIB.buildOr(Dst: Input, Src0: SrcReg, Src1: TopBit);
715 break;
716 }
717 case TargetOpcode::G_CTPOP:
718 Input = ClearHighBits(OldWidth);
719 break;
720 case TargetOpcode::G_CTLZ: {
721 // Clearing the extra bits adds leading zeros the count has to drop.
722 Input = ClearHighBits(OldWidth);
723 Register DstReg = MI->getOperand(i: 0).getReg();
724 widenScalarType(Reg: DstReg, MRI);
725 Register Count = createVirtualRegister(SpvType: SpvTy, GR, MIRBuilder&: MIB);
726 MI->getOperand(i: 0).setReg(Count);
727 setInsertPtAfterDef(MIB, Def: MI);
728 auto Diff = MIB.buildConstant(Res: NewTy, Val: NewWidth - OldWidth);
729 MIB.buildSub(Dst: DstReg, Src0: Count, Src1: Diff);
730 break;
731 }
732 default:
733 llvm_unreachable("unexpected width-sensitive bit-count opcode");
734 }
735 MI->getOperand(i: 1).setReg(Input);
736 }
737}
738
739static void
740generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
741 MachineIRBuilder MIB,
742 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
743 // Get access to information about available extensions
744 const SPIRVSubtarget *ST =
745 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
746
747 MachineRegisterInfo &MRI = MF.getRegInfo();
748 SmallVector<MachineInstr *, 10> ToErase;
749 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
750
751 bool IsExtendedInts =
752 ST->canUseExtension(
753 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
754 ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
755 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
756
757 if (!IsExtendedInts) {
758 // Without arbitrary precision integer extensions, SPIR-V only supports
759 // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40)
760 // must be widened to the next power of two.
761 //
762 // Record the original widths of width-sensitive operands before either
763 // the G_TRUNC handling or the general widening loop retypes vregs, then
764 // rewrite those ops after G_TRUNC processing using the recorded widths.
765 NarrowWideningInfo WideningInfo = recordNarrowOperandWidths(MF, MRI);
766
767 // G_TRUNC requires special handling because its semantics depend on the
768 // original destination width. For example:
769 // %dst:s24 = G_TRUNC %src:s64
770 // After widening s24 to s32, we cannot simply do:
771 // %dst:s32 = G_TRUNC %src:s64
772 // because this would keep 32 bits instead of 24. Instead, we insert a
773 // G_AND to mask the value to the original width:
774 // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask
775 // %masked:s64 = G_AND %src:s64, %mask
776 // %dst:s32 = G_TRUNC %masked:s64
777 // If src and dst widen to the same size, G_TRUNC is replaced entirely:
778 // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask
779 // %dst:s64 = G_AND %src:s64, %mask
780 SmallVector<MachineInstr *, 8> TruncToRemove;
781 for (MachineBasicBlock &MBB : MF) {
782 for (MachineInstr &MI : MBB) {
783 unsigned MIOp = MI.getOpcode();
784 if (MIOp != TargetOpcode::G_TRUNC)
785 continue;
786 assert(MI.getNumOperands() == 2);
787 assert(MI.getOperand(0).isReg());
788 assert(MI.getOperand(1).isReg());
789
790 Register DstReg = MI.getOperand(i: 0).getReg();
791 Register SrcReg = MI.getOperand(i: 1).getReg();
792
793 LLT DstTy = MRI.getType(Reg: DstReg);
794 LLT SrcTy = MRI.getType(Reg: SrcReg);
795 assert((DstTy.isScalar() || DstTy.isVector()) &&
796 (SrcTy.isScalar() || SrcTy.isVector()) &&
797 "Expected scalar or vector G_TRUNC types");
798 assert(DstTy.isVector() == SrcTy.isVector() &&
799 "Expected matching scalar/vector G_TRUNC types");
800 assert((!DstTy.isVector() ||
801 DstTy.getElementCount() == SrcTy.getElementCount()) &&
802 "Expected equal vector element counts");
803
804 unsigned OriginalDstWidth = DstTy.getScalarSizeInBits();
805 unsigned OriginalSrcWidth = SrcTy.getScalarSizeInBits();
806
807 unsigned NewDstWidth = widenBitWidthToNextPow2(BitWidth: OriginalDstWidth);
808 unsigned NewSrcWidth = widenBitWidthToNextPow2(BitWidth: OriginalSrcWidth);
809 LLT NewDstTy = DstTy.changeElementSize(NewEltSize: NewDstWidth);
810 LLT NewSrcTy = SrcTy.changeElementSize(NewEltSize: NewSrcWidth);
811
812 // No Dst width change means no truncation semantics change, but the
813 // source still needs a legal type.
814 if (OriginalDstWidth == NewDstWidth) {
815 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
816 continue;
817 }
818
819 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
820 MRI.setType(VReg: DstReg, Ty: NewDstTy);
821
822 MIB.setInsertPt(MBB, II: MI.getIterator());
823 APInt Mask = APInt::getLowBitsSet(numBits: NewSrcWidth, loBitsSet: OriginalDstWidth);
824 MachineInstrBuilder MaskReg =
825 DstTy.isVector()
826 ? MIB.buildBuildVectorConstant(
827 Res: NewSrcTy,
828 Ops: SmallVector<APInt, 4>(DstTy.getNumElements(), Mask))
829 : MIB.buildConstant(Res: NewSrcTy, Val: Mask);
830 Register MaskedReg = MRI.createGenericVirtualRegister(Ty: NewSrcTy);
831 MIB.buildAnd(Dst: MaskedReg, Src0: SrcReg, Src1: MaskReg);
832
833 if (NewSrcWidth == NewDstWidth) {
834 // Rekey OrigWidth from DstReg to MaskedReg so widenSignSensitiveOps
835 // still sees the narrow original width after replaceRegWith.
836 if (auto It = WideningInfo.OrigWidth.find(Val: DstReg);
837 It != WideningInfo.OrigWidth.end()) {
838 unsigned W = It->second;
839 WideningInfo.OrigWidth.erase(I: It);
840 WideningInfo.OrigWidth.try_emplace(Key: MaskedReg, Args&: W);
841 }
842 MRI.replaceRegWith(FromReg: DstReg, ToReg: MaskedReg);
843 TruncToRemove.push_back(Elt: &MI);
844 } else {
845 MI.getOperand(i: 1).setReg(MaskedReg);
846 }
847 }
848 }
849 for (MachineInstr *MI : TruncToRemove)
850 MI->eraseFromParent();
851
852 widenSignSensitiveOps(MF, GR, MIB, MRI, Info: WideningInfo);
853 widenBitCountOps(GR, MIB, MRI, Info: WideningInfo);
854 }
855
856 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
857 if (MBB->empty())
858 continue;
859
860 bool ReachedBegin = false;
861 for (auto MII = std::prev(x: MBB->end()), Begin = MBB->begin();
862 !ReachedBegin;) {
863 MachineInstr &MI = *MII;
864 unsigned MIOp = MI.getOpcode();
865
866 if (!IsExtendedInts) {
867 // validate bit width of scalar registers and constant immediates
868 for (auto &MOP : MI.operands()) {
869 if (MOP.isReg())
870 widenScalarType(Reg: MOP.getReg(), MRI);
871 else if (MOP.isCImm())
872 widenCImmType(MOP);
873 }
874 }
875
876 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) {
877 Register Reg = MI.getOperand(i: 1).getReg();
878 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
879 Type *ElementTy = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
880 auto SC = addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 3).getImm(), STI: *ST);
881 if (SC == SPIRV::StorageClass::Function &&
882 isa<FunctionType>(Val: ElementTy) &&
883 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
884 SC = SPIRV::StorageClass::CodeSectionINTEL;
885 SPIRVTypeInst AssignedPtrType =
886 GR->getOrCreateSPIRVPointerType(BaseType: ElementTy, I&: MI, SC);
887
888 // For untyped pointers, store the element type for later use.
889 if (ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers) &&
890 !ST->isShader()) {
891 SPIRVTypeInst ElemSpvType = GR->getOrCreateSPIRVType(
892 Type: ElementTy, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
893 /*EmitIR=*/true);
894 GR->setUntypedPtrElementType(Reg, ElemType: ElemSpvType);
895 }
896
897 // The intrinsic also carries vector-of-pointer values produced by
898 // scalarized vector GEPs; wrap the pointer in OpTypeVector to match
899 // the vreg's LLT.
900 LLT RegTy = MRI.getType(Reg);
901 if (RegTy.isValid() && RegTy.isVector())
902 AssignedPtrType = GR->getOrCreateSPIRVVectorType(
903 BaseType: AssignedPtrType, NumElements: RegTy.getNumElements(), MIRBuilder&: MIB,
904 /*EmitIR=*/true);
905 MachineInstr *Def = MRI.getVRegDef(Reg);
906 assert(Def && "Expecting an instruction that defines the register");
907 // G_GLOBAL_VALUE already has type info.
908 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
909 updateRegType(Reg, Ty: nullptr, SpvType: AssignedPtrType, GR, MIB,
910 MRI&: MF.getRegInfo());
911 ToErase.push_back(Elt: &MI);
912 } else if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_type)) {
913 Register Reg = MI.getOperand(i: 1).getReg();
914 Type *Ty = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
915 MachineInstr *Def = MRI.getVRegDef(Reg);
916 assert(Def && "Expecting an instruction that defines the register");
917 // G_GLOBAL_VALUE already has type info.
918 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
919 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
920 if (Def->getOpcode() == TargetOpcode::COPY && isVector1(Ty))
921 updateRegType(Reg: passCopy(Def, MRI: &MF.getRegInfo())->getOperand(i: 0).getReg(),
922 Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
923 ToErase.push_back(Elt: &MI);
924 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
925 MachineInstr *MdMI = MI.getPrevNode();
926 if (MdMI && isSpvIntrinsic(MI: *MdMI, IntrinsicID: Intrinsic::spv_value_md)) {
927 // It's an internal service info from before IRTranslator passes.
928 MachineInstr *Def = getVRegDef(MRI, Reg: MI.getOperand(i: 0).getReg());
929 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
930 if (getVRegDef(MRI, Reg: MI.getOperand(i: I).getReg()) != Def)
931 Def = nullptr;
932 if (Def) {
933 const MDNode *MD = MdMI->getOperand(i: 1).getMetadata();
934 StringRef ValueName =
935 cast<MDString>(Val: MD->getOperand(I: 1))->getString();
936 const MDNode *TypeMD = cast<MDNode>(Val: MD->getOperand(I: 0));
937 Type *ValueTy = getMDOperandAsType(N: TypeMD, I: 0);
938 GR->addValueAttrs(Key: Def, Val: std::make_pair(x&: ValueTy, y: ValueName.str()));
939 // Type results that nothing extracts.
940 auto *STy = dyn_cast<StructType>(Val: ValueTy);
941 if (STy && STy->getNumElements() == Def->getNumDefs())
942 for (auto [Op, ElemTy] : zip(t: Def->defs(), u: STy->elements()))
943 if (!GR->getSPIRVTypeForVReg(VReg: Op.getReg()))
944 updateRegType(Reg: Op.getReg(), Ty: ElemTy, SpvType: nullptr, GR, MIB, MRI);
945 }
946 ToErase.push_back(Elt: MdMI);
947 }
948 ToErase.push_back(Elt: &MI);
949 } else if (MIOp == TargetOpcode::G_CONSTANT ||
950 MIOp == TargetOpcode::G_FCONSTANT ||
951 MIOp == TargetOpcode::G_BUILD_VECTOR) {
952 // %rc = G_CONSTANT ty Val
953 // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
954 Register Reg = MI.getOperand(i: 0).getReg();
955 bool NeedAssignType = !GR->getSPIRVTypeForVReg(VReg: Reg);
956 Type *Ty = nullptr;
957 if (MIOp == TargetOpcode::G_CONSTANT) {
958 auto TargetExtIt = TargetExtConstTypes.find(Val: &MI);
959 Ty = TargetExtIt == TargetExtConstTypes.end()
960 ? MI.getOperand(i: 1).getCImm()->getType()
961 : TargetExtIt->second;
962 const ConstantInt *OpCI = MI.getOperand(i: 1).getCImm();
963 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =
964 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe
965 // at this point not i64/i32 constant but null pointer in the
966 // corresponding address space of RegType.getAddressSpace(). This may
967 // help to successfully validate the case when a OpConstantComposite's
968 // constituent has type that does not match Result Type of
969 // OpConstantComposite (see, for example,
970 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).
971 Register PrimaryReg = GR->find(V: OpCI, MF: &MF);
972 if (!PrimaryReg.isValid()) {
973 GR->add(V: OpCI, MI: &MI);
974 } else if (PrimaryReg != Reg &&
975 MRI.getType(Reg) == MRI.getType(Reg: PrimaryReg)) {
976 auto *RCReg = MRI.getRegClassOrNull(Reg);
977 auto *RCPrimary = MRI.getRegClassOrNull(Reg: PrimaryReg);
978 if (!RCReg || RCPrimary == RCReg) {
979 RegsAlreadyAddedToDT[&MI] = PrimaryReg;
980 ToErase.push_back(Elt: &MI);
981 NeedAssignType = false;
982 }
983 }
984 } else if (MIOp == TargetOpcode::G_FCONSTANT) {
985 Ty = MI.getOperand(i: 1).getFPImm()->getType();
986 } else {
987 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
988 Type *ElemTy = nullptr;
989 MachineInstr *ElemMI = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
990 assert(ElemMI);
991
992 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
993 ElemTy = ElemMI->getOperand(i: 1).getCImm()->getType();
994 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
995 ElemTy = ElemMI->getOperand(i: 1).getFPImm()->getType();
996 } else {
997 if (SPIRVTypeInst ElemSpvType =
998 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg(), MF: &MF))
999 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(Ty: ElemSpvType));
1000 }
1001 if (ElemTy)
1002 Ty = VectorType::get(
1003 ElementType: ElemTy, NumElements: MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
1004 Scalable: false);
1005 else
1006 NeedAssignType = false;
1007 }
1008 if (NeedAssignType)
1009 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI);
1010 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
1011 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
1012 }
1013
1014 if (MII == Begin)
1015 ReachedBegin = true;
1016 else
1017 --MII;
1018 }
1019 }
1020 for (MachineInstr *MI : ToErase) {
1021 auto It = RegsAlreadyAddedToDT.find(Val: MI);
1022 if (It != RegsAlreadyAddedToDT.end())
1023 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: It->second);
1024 invalidateAndEraseMI(GR, MI);
1025 }
1026
1027 // Address the case when IRTranslator introduces instructions with new
1028 // registers without associated SPIRV type.
1029 for (MachineBasicBlock &MBB : MF) {
1030 for (MachineInstr &MI : MBB) {
1031 switch (MI.getOpcode()) {
1032 case TargetOpcode::G_TRUNC:
1033 case TargetOpcode::G_ANYEXT:
1034 case TargetOpcode::G_SEXT:
1035 case TargetOpcode::G_ZEXT:
1036 case TargetOpcode::G_PTRTOINT:
1037 case TargetOpcode::COPY:
1038 case TargetOpcode::G_ADDRSPACE_CAST:
1039 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
1040 break;
1041 }
1042 }
1043 }
1044}
1045
1046static void processInstrsWithTypeFolding(MachineFunction &MF,
1047 SPIRVGlobalRegistry *GR,
1048 MachineIRBuilder MIB) {
1049 MachineRegisterInfo &MRI = MF.getRegInfo();
1050 for (MachineBasicBlock &MBB : MF)
1051 for (MachineInstr &MI : MBB)
1052 if (isTypeFoldingSupported(Opcode: MI.getOpcode()))
1053 processInstr(MI, MIB, MRI, GR, KnownResType: nullptr);
1054}
1055
1056static Register
1057collectInlineAsmInstrOperands(MachineInstr *MI,
1058 SmallVector<unsigned, 4> *Ops = nullptr) {
1059 Register DefReg;
1060 unsigned StartOp = InlineAsm::MIOp_FirstOperand,
1061 AsmDescOp = InlineAsm::MIOp_FirstOperand;
1062 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
1063 ++Idx) {
1064 const MachineOperand &MO = MI->getOperand(i: Idx);
1065 if (MO.isMetadata())
1066 continue;
1067 if (Idx == AsmDescOp && MO.isImm()) {
1068 // compute the index of the next operand descriptor
1069 const InlineAsm::Flag F(MO.getImm());
1070 AsmDescOp += 1 + F.getNumOperandRegisters();
1071 continue;
1072 }
1073 if (MO.isReg() && MO.isDef()) {
1074 if (!Ops)
1075 return MO.getReg();
1076 DefReg = MO.getReg();
1077 } else if (Ops) {
1078 Ops->push_back(Elt: Idx);
1079 }
1080 }
1081 return DefReg;
1082}
1083
1084static void
1085insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1086 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
1087 const SmallVector<MachineInstr *> &ToProcess) {
1088 MachineRegisterInfo &MRI = MF.getRegInfo();
1089 Register AsmTargetReg;
1090 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
1091 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
1092 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
1093 MIRBuilder.setInsertPt(MBB&: *I2->getParent(), II: *I2);
1094
1095 if (!AsmTargetReg.isValid()) {
1096 // define vendor specific assembly target or dialect
1097 AsmTargetReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1098 MRI.setRegClass(Reg: AsmTargetReg, RC: &SPIRV::iIDRegClass);
1099 auto AsmTargetMIB =
1100 MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmTargetINTEL).addDef(RegNo: AsmTargetReg);
1101 addStringImm(Str: ST.getTargetTripleAsStr(), MIB&: AsmTargetMIB);
1102 GR->add(Obj: AsmTargetMIB.getInstr(), MI: AsmTargetMIB);
1103 }
1104
1105 // create types
1106 const MDNode *IAMD = I1->getOperand(i: 1).getMetadata();
1107 FunctionType *FTy = cast<FunctionType>(Val: getMDOperandAsType(N: IAMD, I: 0));
1108 SmallVector<SPIRVTypeInst, 4> ArgTypes;
1109 for (const auto &ArgTy : FTy->params())
1110 ArgTypes.push_back(Elt: GR->getOrCreateSPIRVType(
1111 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true));
1112 SPIRVTypeInst RetType =
1113 GR->getOrCreateSPIRVType(Type: FTy->getReturnType(), MIRBuilder,
1114 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1115 SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
1116 Ty: FTy, RetType, ArgTypes, MIRBuilder);
1117
1118 // define vendor specific assembly instructions string
1119 Register AsmReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1120 MRI.setRegClass(Reg: AsmReg, RC: &SPIRV::iIDRegClass);
1121 auto AsmMIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmINTEL)
1122 .addDef(RegNo: AsmReg)
1123 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1124 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncType))
1125 .addUse(RegNo: AsmTargetReg);
1126 // inline asm string:
1127 addStringImm(Str: I2->getOperand(i: InlineAsm::MIOp_AsmString).getSymbolName(),
1128 MIB&: AsmMIB);
1129 // inline asm constraint string:
1130 addStringImm(Str: cast<MDString>(Val: I1->getOperand(i: 2).getMetadata()->getOperand(I: 0))
1131 ->getString(),
1132 MIB&: AsmMIB);
1133 GR->add(Obj: AsmMIB.getInstr(), MI: AsmMIB);
1134
1135 // calls the inline assembly instruction
1136 unsigned ExtraInfo = I2->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
1137 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
1138 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
1139 .addUse(RegNo: AsmReg)
1140 .addImm(Val: static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
1141
1142 Register DefReg = collectInlineAsmInstrOperands(MI: I2);
1143 if (!DefReg.isValid()) {
1144 DefReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
1145 MRI.setRegClass(Reg: DefReg, RC: &SPIRV::iIDRegClass);
1146 SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType(
1147 Type: Type::getVoidTy(C&: MF.getFunction().getContext()), MIRBuilder,
1148 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1149 GR->assignSPIRVTypeToVReg(Type: VoidType, VReg: DefReg, MF);
1150 }
1151
1152 auto AsmCall = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmCallINTEL)
1153 .addDef(RegNo: DefReg)
1154 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
1155 .addUse(RegNo: AsmReg);
1156 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
1157 AsmCall.addUse(RegNo: I1->getOperand(i: IntrIdx).getReg());
1158 AsmCall->setDebugLoc(I2->getDebugLoc());
1159
1160 // IRTranslator gets a bit confused when lowering inline ASM with outputs
1161 // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
1162 // we have to clean that up here to prevent erroneous trunc casts either on
1163 // a struct (for multiple outputs) or same width integers to get lowered
1164 // into SPIR-V
1165 if (MRI.hasOneUse(RegNo: DefReg)) {
1166 MachineInstr &CopyMI = *MRI.use_instr_begin(RegNo: DefReg);
1167 if (CopyMI.getOpcode() == TargetOpcode::COPY) {
1168 Register CopyDst = CopyMI.getOperand(i: 0).getReg();
1169 if (MRI.hasOneUse(RegNo: CopyDst)) {
1170 MachineInstr &TruncMI = *MRI.use_instr_begin(RegNo: CopyDst);
1171 if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
1172 MRI.setType(VReg: DefReg, Ty: GR->getRegType(SpvType: RetType));
1173 Register TruncReg = TruncMI.defs().begin()->getReg();
1174 MRI.replaceRegWith(FromReg: TruncReg, ToReg: DefReg);
1175 invalidateAndEraseMI(GR, MI: &TruncMI);
1176 invalidateAndEraseMI(GR, MI: &CopyMI);
1177 }
1178 }
1179 }
1180 }
1181 }
1182 for (MachineInstr *MI : ToProcess)
1183 invalidateAndEraseMI(GR, MI);
1184}
1185
1186static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1187 const SPIRVSubtarget &ST,
1188 MachineIRBuilder MIRBuilder) {
1189 SmallVector<MachineInstr *> ToProcess;
1190 for (MachineBasicBlock &MBB : MF) {
1191 for (MachineInstr &MI : MBB) {
1192 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_inline_asm) ||
1193 MI.getOpcode() == TargetOpcode::INLINEASM)
1194 ToProcess.push_back(Elt: &MI);
1195 }
1196 }
1197 if (ToProcess.size() == 0)
1198 return;
1199
1200 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly))
1201 report_fatal_error(reason: "Inline assembly instructions require the "
1202 "following SPIR-V extension: SPV_INTEL_inline_assembly",
1203 gen_crash_diag: false);
1204
1205 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
1206}
1207
1208static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1209 MachineIRBuilder MIB) {
1210 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MIB.getMF().getSubtarget());
1211 SmallVector<MachineInstr *, 10> ToErase;
1212 for (MachineBasicBlock &MBB : MF) {
1213 for (MachineInstr &MI : MBB) {
1214 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration) &&
1215 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_aliasing_decoration) &&
1216 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration))
1217 continue;
1218 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getNextNode());
1219 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration)) {
1220 buildOpSpirvDecorations(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1221 GVarMD: MI.getOperand(i: 2).getMetadata(), ST);
1222 } else if (isSpvIntrinsic(MI,
1223 IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration)) {
1224 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(
1225 MD: MI.getOperand(i: 2).getMetadata()->getOperand(I: 0));
1226 uint32_t OpValue = OpV->getValueAPF().bitcastToAPInt().getZExtValue();
1227
1228 buildOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1229 Dec: SPIRV::Decoration::FPMaxErrorDecorationINTEL,
1230 DecArgs: {OpValue});
1231 } else {
1232 GR->buildMemAliasingOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1233 Dec: MI.getOperand(i: 2).getImm(),
1234 GVarMD: MI.getOperand(i: 3).getMetadata());
1235 }
1236
1237 ToErase.push_back(Elt: &MI);
1238 }
1239 }
1240 for (MachineInstr *MI : ToErase)
1241 invalidateAndEraseMI(GR, MI);
1242}
1243
1244// Returns the value of the switch case operand in Reg. The case value stays a
1245// G_CONSTANT until the module emits a SPIR-V constant for the same value, at
1246// which point the case register is replaced with the one defining that
1247// constant, which keeps its value in literal operands rather than in a CImm.
1248static const ConstantInt *getSwitchCaseValue(Register Reg,
1249 const MachineRegisterInfo &MRI,
1250 LLVMContext &Ctx) {
1251 APInt Val;
1252 if (mi_match(R: Reg, MRI, P: m_ICst(Cst&: Val)))
1253 return ConstantInt::get(Context&: Ctx, V: Val);
1254
1255 const MachineInstr *Def = nullptr;
1256 if (!mi_match(R: Reg, MRI, P: m_MInstr(MI&: Def)))
1257 llvm_unreachable("Switch case operand has no definition");
1258
1259 LLT Ty = MRI.getType(Reg);
1260 assert(Ty.isValid() && "Expected a typed switch case value");
1261 Val = APInt(Ty.getScalarSizeInBits(), 0);
1262
1263 switch (Def->getOpcode()) {
1264 case SPIRV::OpConstantNull:
1265 case SPIRV::OpConstantI:
1266 // The operands after the type are 32-bit literal words, least significant
1267 // first, as written by addNumImm(). OpConstantNull carries none, so it
1268 // decodes to zero without a case of its own.
1269 for (unsigned I = 2, E = Def->getNumExplicitOperands(); I != E; ++I) {
1270 uint32_t Word = static_cast<uint32_t>(Def->getOperand(i: I).getImm());
1271 Val |= APInt(Val.getBitWidth(), Word).shl(shiftAmt: (I - 2) * 32);
1272 }
1273 break;
1274 default:
1275 llvm_unreachable("Unexpected definition of a switch case value");
1276 }
1277 return ConstantInt::get(Context&: Ctx, V: Val);
1278}
1279
1280// LLVM allows the switches to use registers as cases, while SPIR-V required
1281// those to be immediate values. This function replaces such operands with the
1282// equivalent immediate constant.
1283static void processSwitchesConstants(MachineFunction &MF,
1284 SPIRVGlobalRegistry *GR,
1285 MachineIRBuilder MIB) {
1286 MachineRegisterInfo &MRI = MF.getRegInfo();
1287 LLVMContext &Ctx = MF.getFunction().getContext();
1288 for (MachineBasicBlock &MBB : MF) {
1289 for (MachineInstr &MI : MBB) {
1290 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch))
1291 continue;
1292
1293 SmallVector<MachineOperand, 8> NewOperands;
1294 NewOperands.push_back(Elt: MI.getOperand(i: 0)); // Opcode
1295 NewOperands.push_back(Elt: MI.getOperand(i: 1)); // Condition
1296 NewOperands.push_back(Elt: MI.getOperand(i: 2)); // Default
1297 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
1298 Register Reg = MI.getOperand(i).getReg();
1299 NewOperands.push_back(
1300 Elt: MachineOperand::CreateCImm(CI: getSwitchCaseValue(Reg, MRI, Ctx)));
1301
1302 NewOperands.push_back(Elt: MI.getOperand(i: i + 1));
1303 }
1304
1305 assert(MI.getNumOperands() == NewOperands.size());
1306 while (MI.getNumOperands() > 0)
1307 MI.removeOperand(OpNo: 0);
1308 for (auto &MO : NewOperands)
1309 MI.addOperand(Op: MO);
1310 }
1311 }
1312}
1313
1314// Some instructions are used during CodeGen but should never be emitted.
1315// Cleaning up those.
1316static void cleanupHelperInstructions(MachineFunction &MF,
1317 SPIRVGlobalRegistry *GR) {
1318 SmallVector<MachineInstr *, 8> ToEraseMI;
1319 for (MachineBasicBlock &MBB : MF) {
1320 for (MachineInstr &MI : MBB) {
1321 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant) ||
1322 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
1323 ToEraseMI.push_back(Elt: &MI);
1324 }
1325 }
1326
1327 for (MachineInstr *MI : ToEraseMI)
1328 invalidateAndEraseMI(GR, MI);
1329}
1330
1331// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
1332// operands/registers by the actual MBB it references.
1333static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1334 MachineIRBuilder MIB) {
1335 // Gather the reverse-mapping BB -> MBB.
1336 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
1337 for (MachineBasicBlock &MBB : MF)
1338 BB2MBB[MBB.getBasicBlock()] = &MBB;
1339
1340 // Gather instructions requiring patching. For now, only those can use
1341 // G_BLOCK_ADDR.
1342 SmallVector<MachineInstr *, 8> InstructionsToPatch;
1343 for (MachineBasicBlock &MBB : MF) {
1344 for (MachineInstr &MI : MBB) {
1345 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch) ||
1346 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_loop_merge) ||
1347 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_selection_merge))
1348 InstructionsToPatch.push_back(Elt: &MI);
1349 }
1350 }
1351
1352 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
1353 // the actual MBB it references. Once those references have been updated, we
1354 // can cleanup remaining G_BLOCK_ADDR references.
1355 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
1356 SmallPtrSet<MachineInstr *, 8> ToEraseMI;
1357 MachineRegisterInfo &MRI = MF.getRegInfo();
1358 for (MachineInstr *MI : InstructionsToPatch) {
1359 SmallVector<MachineOperand, 8> NewOps;
1360 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1361 // The operand is not a register, keep as-is.
1362 if (!MI->getOperand(i).isReg()) {
1363 NewOps.push_back(Elt: MI->getOperand(i));
1364 continue;
1365 }
1366
1367 Register Reg = MI->getOperand(i).getReg();
1368 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
1369 // The register is not the result of G_BLOCK_ADDR, keep as-is.
1370 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
1371 NewOps.push_back(Elt: MI->getOperand(i));
1372 continue;
1373 }
1374
1375 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
1376 BuildMBB->getOperand(1).isBlockAddress() &&
1377 BuildMBB->getOperand(1).getBlockAddress());
1378 BasicBlock *BB =
1379 BuildMBB->getOperand(i: 1).getBlockAddress()->getBasicBlock();
1380 auto It = BB2MBB.find(Val: BB);
1381 if (It == BB2MBB.end())
1382 report_fatal_error(reason: "cannot find a machine basic block by a basic block "
1383 "in a switch statement");
1384 MachineBasicBlock *ReferencedBlock = It->second;
1385 NewOps.push_back(Elt: MachineOperand::CreateMBB(MBB: ReferencedBlock));
1386
1387 ClearAddressTaken.insert(Ptr: ReferencedBlock);
1388 ToEraseMI.insert(Ptr: BuildMBB);
1389 }
1390
1391 // Replace the operands.
1392 assert(MI->getNumOperands() == NewOps.size());
1393 while (MI->getNumOperands() > 0)
1394 MI->removeOperand(OpNo: 0);
1395 for (auto &MO : NewOps)
1396 MI->addOperand(Op: MO);
1397
1398 if (MachineInstr *Next = MI->getNextNode()) {
1399 if (isSpvIntrinsic(MI: *Next, IntrinsicID: Intrinsic::spv_track_constant)) {
1400 ToEraseMI.insert(Ptr: Next);
1401 Next = MI->getNextNode();
1402 }
1403 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
1404 ToEraseMI.insert(Ptr: Next);
1405 }
1406 }
1407
1408 // BlockAddress operands were used to keep information between passes,
1409 // let's undo the "address taken" status to reflect that Succ doesn't
1410 // actually correspond to an IR-level basic block.
1411 for (MachineBasicBlock *Succ : ClearAddressTaken)
1412 Succ->setAddressTakenIRBlock(nullptr);
1413
1414 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
1415 // this leaves their BasicBlock counterparts in a "address taken" status. This
1416 // would make AsmPrinter to generate a series of unneeded labels of a "Address
1417 // of block that was removed by CodeGen" kind. Let's first ensure that we
1418 // don't have a dangling BlockAddress constants by zapping the BlockAddress
1419 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
1420 Constant *Replacement =
1421 ConstantInt::get(Ty: Type::getInt32Ty(C&: MF.getFunction().getContext()), V: 1);
1422 for (MachineInstr *BlockAddrI : ToEraseMI) {
1423 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
1424 BlockAddress *BA = const_cast<BlockAddress *>(
1425 BlockAddrI->getOperand(i: 1).getBlockAddress());
1426 BA->replaceAllUsesWith(
1427 V: ConstantExpr::getIntToPtr(C: Replacement, Ty: BA->getType()));
1428 BA->destroyConstant();
1429 }
1430 invalidateAndEraseMI(GR, MI: BlockAddrI);
1431 }
1432}
1433
1434static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
1435 if (MBB.empty())
1436 return MBB.getNextNode() != nullptr;
1437
1438 // Branching SPIR-V intrinsics are not detected by this generic method.
1439 // Thus, we can only trust negative result.
1440 if (!MBB.canFallThrough())
1441 return false;
1442
1443 // Otherwise, we must manually check if we have a SPIR-V intrinsic which
1444 // prevent an implicit fallthrough.
1445 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
1446 It != E; ++It) {
1447 if (isSpvIntrinsic(MI: *It, IntrinsicID: Intrinsic::spv_switch))
1448 return false;
1449 }
1450 return true;
1451}
1452
1453static void removeImplicitFallthroughs(MachineFunction &MF,
1454 MachineIRBuilder MIB) {
1455 // It is valid for MachineBasicBlocks to not finish with a branch instruction.
1456 // In such cases, they will simply fallthrough their immediate successor.
1457 for (MachineBasicBlock &MBB : MF) {
1458 if (!isImplicitFallthrough(MBB))
1459 continue;
1460
1461 assert(MBB.succ_size() == 1);
1462 MIB.setInsertPt(MBB, II: MBB.end());
1463 MIB.buildBr(Dest&: **MBB.successors().begin());
1464 }
1465}
1466
1467static bool runPreLegalizer(MachineFunction &MF) {
1468 // Initialize the type registry.
1469 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
1470 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1471 GR->setCurrentFunc(MF);
1472 MachineIRBuilder MIB(MF);
1473 // a registry of target extension constants
1474 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
1475 // to keep record of tracked constants
1476 addConstantsToTrack(MF, GR, STI: ST, TargetExtConstTypes);
1477 foldConstantsIntoIntrinsics(MF, GR, MIB);
1478 insertBitcasts(MF, GR, MIB);
1479 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
1480
1481 processSwitchesConstants(MF, GR, MIB);
1482 processBlockAddr(MF, GR, MIB);
1483 cleanupHelperInstructions(MF, GR);
1484
1485 processInstrsWithTypeFolding(MF, GR, MIB);
1486 removeImplicitFallthroughs(MF, MIB);
1487 insertSpirvDecorations(MF, GR, MIB);
1488 insertInlineAsm(MF, GR, ST, MIRBuilder: MIB);
1489 lowerBitcasts(MF, GR, MIB);
1490
1491 return true;
1492}
1493
1494INITIALIZE_PASS(SPIRVPreLegalizerLegacy, DEBUG_TYPE, "SPIRV pre legalizer",
1495 false, false)
1496
1497char SPIRVPreLegalizerLegacy::ID = 0;
1498
1499FunctionPass *llvm::createSPIRVPreLegalizerLegacyPass() {
1500 return new SPIRVPreLegalizerLegacy();
1501}
1502
1503bool SPIRVPreLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
1504 return runPreLegalizer(MF);
1505}
1506
1507PreservedAnalyses
1508SPIRVPreLegalizerPass::run(MachineFunction &MF,
1509 MachineFunctionAnalysisManager &MFAM) {
1510 bool Changed = runPreLegalizer(MF);
1511 if (!Changed)
1512 return PreservedAnalyses::all();
1513
1514 return getMachineFunctionPassPreservedAnalyses()
1515 .preserve<GISelValueTrackingAnalysis>();
1516}
1517