1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass prepares IR for legalization: it assigns SPIR-V types to registers
10// and removes intrinsics which holded these types during IR translation.
11// Also it processes constants and registers them in GR to avoid duplication.
12//
13//===----------------------------------------------------------------------===//
14
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
21#include "llvm/IR/Attributes.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
24
25#define DEBUG_TYPE "spirv-prelegalizer"
26
27using namespace llvm;
28
29namespace {
30class SPIRVPreLegalizer : public MachineFunctionPass {
31public:
32 static char ID;
33 SPIRVPreLegalizer() : MachineFunctionPass(ID) {}
34 bool runOnMachineFunction(MachineFunction &MF) override;
35 void getAnalysisUsage(AnalysisUsage &AU) const override;
36};
37} // namespace
38
39void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const {
40 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
41 MachineFunctionPass::getAnalysisUsage(AU);
42}
43
44static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
45 MachineInstr *MI) {
46 GR->invalidateMachineInstr(MI);
47 MI->eraseFromParent();
48}
49
50static void
51addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
52 const SPIRVSubtarget &STI,
53 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
54 MachineRegisterInfo &MRI = MF.getRegInfo();
55 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
56 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
57 for (MachineBasicBlock &MBB : MF) {
58 for (MachineInstr &MI : MBB) {
59 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant))
60 continue;
61 ToErase.push_back(Elt: &MI);
62 Register SrcReg = MI.getOperand(i: 2).getReg();
63 auto *Const =
64 cast<Constant>(Val: cast<ConstantAsMetadata>(
65 Val: MI.getOperand(i: 3).getMetadata()->getOperand(I: 0))
66 ->getValue());
67 if (auto *GV = dyn_cast<GlobalValue>(Val: Const)) {
68 Register Reg = GR->find(V: GV, MF: &MF);
69 if (!Reg.isValid()) {
70 GR->add(V: GV, MI: MRI.getVRegDef(Reg: SrcReg));
71 GR->addGlobalObject(V: GV, MF: &MF, R: SrcReg);
72 } else
73 RegsAlreadyAddedToDT[&MI] = Reg;
74 } else {
75 Register Reg = GR->find(V: Const, MF: &MF);
76 if (!Reg.isValid()) {
77 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Val: Const)) {
78 auto *BuildVec = MRI.getVRegDef(Reg: SrcReg);
79 assert(BuildVec &&
80 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
81 GR->add(V: Const, MI: BuildVec);
82 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
83 // Ensure that OpConstantComposite reuses a constant when it's
84 // already created and available in the same machine function.
85 Constant *ElemConst = ConstVec->getElementAsConstant(i);
86 Register ElemReg = GR->find(V: ElemConst, MF: &MF);
87 if (!ElemReg.isValid())
88 GR->add(V: ElemConst,
89 MI: MRI.getVRegDef(Reg: BuildVec->getOperand(i: 1 + i).getReg()));
90 else
91 BuildVec->getOperand(i: 1 + i).setReg(ElemReg);
92 }
93 }
94 if (Const->getType()->isTargetExtTy()) {
95 // remember association so that we can restore it when assign types
96 MachineInstr *SrcMI = MRI.getVRegDef(Reg: SrcReg);
97 if (SrcMI)
98 GR->add(V: Const, MI: SrcMI);
99 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
100 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
101 TargetExtConstTypes[SrcMI] = Const->getType();
102 if (Const->isNullValue()) {
103 MachineBasicBlock &DepMBB = MF.front();
104 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());
105 SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType(
106 Type: Const->getType(), MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
107 EmitIR: true);
108 assert(SrcMI && "Expected source instruction to be valid");
109 SrcMI->setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
110 SrcMI->addOperand(Op: MachineOperand::CreateReg(
111 Reg: GR->getSPIRVTypeID(SpirvType: ExtType), isDef: false));
112 }
113 }
114 } else {
115 RegsAlreadyAddedToDT[&MI] = Reg;
116 // This MI is unused and will be removed. If the MI uses
117 // const_composite, it will be unused and should be removed too.
118 assert(MI.getOperand(2).isReg() && "Reg operand is expected");
119 MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
120 if (SrcMI && isSpvIntrinsic(MI: *SrcMI, IntrinsicID: Intrinsic::spv_const_composite))
121 ToEraseComposites.push_back(Elt: SrcMI);
122 }
123 }
124 }
125 }
126 for (MachineInstr *MI : ToErase) {
127 Register Reg = MI->getOperand(i: 2).getReg();
128 auto It = RegsAlreadyAddedToDT.find(Val: MI);
129 if (It != RegsAlreadyAddedToDT.end())
130 Reg = It->second;
131 auto *RC = MRI.getRegClassOrNull(Reg: MI->getOperand(i: 0).getReg());
132 if (!MRI.getRegClassOrNull(Reg) && RC)
133 MRI.setRegClass(Reg, RC);
134 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: Reg);
135 invalidateAndEraseMI(GR, MI);
136 }
137 for (MachineInstr *MI : ToEraseComposites)
138 invalidateAndEraseMI(GR, MI);
139}
140
141static void foldConstantsIntoIntrinsics(MachineFunction &MF,
142 SPIRVGlobalRegistry *GR,
143 MachineIRBuilder MIB) {
144 SmallVector<MachineInstr *, 64> ToErase;
145 for (MachineBasicBlock &MBB : MF) {
146 for (MachineInstr &MI : MBB) {
147 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_name))
148 continue;
149 const MDNode *MD = MI.getOperand(i: 2).getMetadata();
150 StringRef ValueName = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
151 if (ValueName.size() > 0) {
152 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
153 buildOpName(Target: MI.getOperand(i: 1).getReg(), Name: ValueName, MIRBuilder&: MIB);
154 }
155 ToErase.push_back(Elt: &MI);
156 }
157 for (MachineInstr *MI : ToErase)
158 invalidateAndEraseMI(GR, MI);
159 ToErase.clear();
160 }
161}
162
163static MachineInstr *findAssignTypeInstr(Register Reg,
164 MachineRegisterInfo *MRI) {
165 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(RegNo: Reg),
166 IE = MRI->use_instr_end();
167 I != IE; ++I) {
168 MachineInstr *UseMI = &*I;
169 if ((isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_ptr_type) ||
170 isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_type)) &&
171 UseMI->getOperand(i: 1).getReg() == Reg)
172 return UseMI;
173 }
174 return nullptr;
175}
176
177static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
178 Register ResVReg, Register OpReg) {
179 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResVReg);
180 SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg);
181 assert(ResType && OpType && "Operand types are expected");
182 if (!GR->isBitcastCompatible(Type1: ResType, Type2: OpType))
183 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
184 MachineRegisterInfo *MRI = MIB.getMRI();
185 if (!MRI->getRegClassOrNull(Reg: ResVReg))
186 MRI->setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: ResType));
187 if (ResType == OpType)
188 MIB.buildInstr(Opcode: TargetOpcode::COPY).addDef(RegNo: ResVReg).addUse(RegNo: OpReg);
189 else
190 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
191 .addDef(RegNo: ResVReg)
192 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResType))
193 .addUse(RegNo: OpReg);
194}
195
196// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.
197// The verifier checks if the source and destination LLTs of a G_BITCAST are
198// different, but this check is too strict for SPIR-V's typed pointers, which
199// may have the same LLT but different SPIRV type (e.g. pointers to different
200// pointee types). By lowering to OpBitcast here, we bypass the verifier's
201// check. See discussion in https://github.com/llvm/llvm-project/pull/110270
202// for more context.
203//
204// We also handle the llvm.spv.bitcast intrinsic here. If the source and
205// destination SPIR-V types are the same, we lower it to a COPY to enable
206// further optimizations like copy propagation.
207static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
208 MachineIRBuilder MIB) {
209 SmallVector<MachineInstr *, 16> ToErase;
210 for (MachineBasicBlock &MBB : MF) {
211 for (MachineInstr &MI : MBB) {
212 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_bitcast)) {
213 Register DstReg = MI.getOperand(i: 0).getReg();
214 Register SrcReg = MI.getOperand(i: 2).getReg();
215 SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(VReg: DstReg);
216 assert(
217 DstType &&
218 "Expected destination SPIR-V type to have been assigned already.");
219 SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(VReg: SrcReg);
220 assert(SrcType &&
221 "Expected source SPIR-V type to have been assigned already.");
222 if (DstType == SrcType) {
223 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
224 MIB.buildCopy(Res: DstReg, Op: SrcReg);
225 ToErase.push_back(Elt: &MI);
226 continue;
227 }
228 }
229
230 if (MI.getOpcode() != TargetOpcode::G_BITCAST)
231 continue;
232
233 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
234 buildOpBitcast(GR, MIB, ResVReg: MI.getOperand(i: 0).getReg(),
235 OpReg: MI.getOperand(i: 1).getReg());
236 ToErase.push_back(Elt: &MI);
237 }
238 }
239 for (MachineInstr *MI : ToErase)
240 invalidateAndEraseMI(GR, MI);
241}
242
243static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
244 MachineIRBuilder MIB) {
245 // Get access to information about available extensions
246 const SPIRVSubtarget *ST =
247 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
248 SmallVector<MachineInstr *, 10> ToErase;
249 for (MachineBasicBlock &MBB : MF) {
250 for (MachineInstr &MI : MBB) {
251 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_ptrcast))
252 continue;
253 assert(MI.getOperand(2).isReg());
254 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
255 ToErase.push_back(Elt: &MI);
256 Register Def = MI.getOperand(i: 0).getReg();
257 Register Source = MI.getOperand(i: 2).getReg();
258 Type *ElemTy = getMDOperandAsType(N: MI.getOperand(i: 3).getMetadata(), I: 0);
259 auto SC =
260 isa<FunctionType>(Val: ElemTy)
261 ? SPIRV::StorageClass::CodeSectionINTEL
262 : addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 4).getImm(), STI: *ST);
263 SPIRVTypeInst AssignedPtrType =
264 GR->getOrCreateSPIRVPointerType(BaseType: ElemTy, I&: MI, SC);
265
266 // If the ptrcast would be redundant, replace all uses with the source
267 // register.
268 MachineRegisterInfo *MRI = MIB.getMRI();
269 if (GR->getSPIRVTypeForVReg(VReg: Source) == AssignedPtrType) {
270 // Erase Def's assign type instruction if we are going to replace Def.
271 if (MachineInstr *AssignMI = findAssignTypeInstr(Reg: Def, MRI))
272 ToErase.push_back(Elt: AssignMI);
273 MRI->replaceRegWith(FromReg: Def, ToReg: Source);
274 } else {
275 if (!GR->getSPIRVTypeForVReg(VReg: Def, MF: &MF))
276 GR->assignSPIRVTypeToVReg(Type: AssignedPtrType, VReg: Def, MF);
277 MIB.buildBitcast(Dst: Def, Src: Source);
278 }
279 }
280 }
281 for (MachineInstr *MI : ToErase)
282 invalidateAndEraseMI(GR, MI);
283}
284
285// Translating GV, IRTranslator sometimes generates following IR:
286// %1 = G_GLOBAL_VALUE
287// %2 = COPY %1
288// %3 = G_ADDRSPACE_CAST %2
289//
290// or
291//
292// %1 = G_ZEXT %2
293// G_MEMCPY ... %2 ...
294//
295// New registers have no SPIRV type and no register class info.
296//
297// Set SPIRV type for GV, propagate it from GV to other instructions,
298// also set register classes.
299static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
300 SPIRVGlobalRegistry *GR,
301 MachineRegisterInfo &MRI,
302 MachineIRBuilder &MIB) {
303 SPIRVTypeInst SpvType = nullptr;
304 assert(MI && "Machine instr is expected");
305 if (MI->getOperand(i: 0).isReg()) {
306 Register Reg = MI->getOperand(i: 0).getReg();
307 SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
308 if (!SpvType) {
309 switch (MI->getOpcode()) {
310 case TargetOpcode::G_FCONSTANT:
311 case TargetOpcode::G_CONSTANT: {
312 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
313 Type *Ty = MI->getOperand(i: 1).getCImm()->getType();
314 SpvType = GR->getOrCreateSPIRVType(
315 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
316 break;
317 }
318 case TargetOpcode::G_GLOBAL_VALUE: {
319 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
320 const GlobalValue *Global = MI->getOperand(i: 1).getGlobal();
321 Type *ElementTy = toTypedPointer(Ty: GR->getDeducedGlobalValueType(Global));
322 auto *Ty = TypedPointerType::get(ElementType: ElementTy,
323 AddressSpace: Global->getType()->getAddressSpace());
324 SpvType = GR->getOrCreateSPIRVType(
325 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
326 break;
327 }
328 case TargetOpcode::G_ANYEXT:
329 case TargetOpcode::G_SEXT:
330 case TargetOpcode::G_ZEXT: {
331 if (MI->getOperand(i: 1).isReg()) {
332 if (MachineInstr *DefInstr =
333 MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
334 if (SPIRVTypeInst Def =
335 propagateSPIRVType(MI: DefInstr, GR, MRI, MIB)) {
336 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Type: Def);
337 unsigned ExpectedBW =
338 std::max(a: MRI.getType(Reg).getScalarSizeInBits(), b: CurrentBW);
339 unsigned NumElements = GR->getScalarOrVectorComponentCount(Type: Def);
340 SpvType = GR->getOrCreateSPIRVIntegerType(BitWidth: ExpectedBW, MIRBuilder&: MIB);
341 if (NumElements > 1)
342 SpvType = GR->getOrCreateSPIRVVectorType(BaseType: SpvType, NumElements,
343 MIRBuilder&: MIB, EmitIR: true);
344 }
345 }
346 }
347 break;
348 }
349 case TargetOpcode::G_PTRTOINT:
350 SpvType = GR->getOrCreateSPIRVIntegerType(
351 BitWidth: MRI.getType(Reg).getScalarSizeInBits(), MIRBuilder&: MIB);
352 break;
353 case TargetOpcode::G_TRUNC:
354 case TargetOpcode::G_ADDRSPACE_CAST:
355 case TargetOpcode::G_PTR_ADD:
356 case TargetOpcode::COPY: {
357 MachineOperand &Op = MI->getOperand(i: 1);
358 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Reg: Op.getReg()) : nullptr;
359 if (Def)
360 SpvType = propagateSPIRVType(MI: Def, GR, MRI, MIB);
361 break;
362 }
363 default:
364 break;
365 }
366 if (SpvType) {
367 // check if the address space needs correction
368 LLT RegType = MRI.getType(Reg);
369 if (SpvType->getOpcode() == SPIRV::OpTypePointer &&
370 RegType.isPointer() &&
371 storageClassToAddressSpace(SC: GR->getPointerStorageClass(Type: SpvType)) !=
372 RegType.getAddressSpace()) {
373 const SPIRVSubtarget &ST =
374 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
375 auto TSC = addressSpaceToStorageClass(AddrSpace: RegType.getAddressSpace(), STI: ST);
376 SpvType = GR->changePointerStorageClass(PtrType: SpvType, SC: TSC, I&: *MI);
377 }
378 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
379 }
380 if (!MRI.getRegClassOrNull(Reg))
381 MRI.setRegClass(Reg, RC: SpvType ? GR->getRegClass(SpvType)
382 : &SPIRV::iIDRegClass);
383 }
384 }
385 return SpvType;
386}
387
388// To support current approach and limitations wrt. bit width here we widen a
389// scalar register with a bit width greater than 1 to valid sizes and cap it to
390// 128 width.
391static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {
392 if (BitWidth == 1)
393 return 1; // No need to widen 1-bit values
394 return std::min(a: std::max(a: 1u << Log2_32_Ceil(Value: BitWidth), b: 8u), b: 128u);
395}
396
397static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {
398 LLT RegType = MRI.getType(Reg);
399 if (!RegType.isScalar())
400 return;
401 unsigned CurrentWidth = RegType.getScalarSizeInBits();
402 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
403 if (NewWidth != CurrentWidth)
404 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: NewWidth));
405}
406
407static void widenCImmType(MachineOperand &MOP) {
408 const ConstantInt *CImmVal = MOP.getCImm();
409 unsigned CurrentWidth = CImmVal->getBitWidth();
410 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
411 if (NewWidth != CurrentWidth) {
412 // Replace the immediate value with the widened version
413 MOP.setCImm(ConstantInt::get(Context&: CImmVal->getType()->getContext(),
414 V: CImmVal->getValue().zextOrTrunc(width: NewWidth)));
415 }
416}
417
418static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
419 MachineBasicBlock &MBB = *Def->getParent();
420 MachineBasicBlock::iterator DefIt =
421 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();
422 // Skip all the PHI and debug instructions.
423 while (DefIt != MBB.end() &&
424 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))
425 DefIt = std::next(x: DefIt);
426 MIB.setInsertPt(MBB, II: DefIt);
427}
428
429namespace llvm {
430void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType,
431 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
432 MachineRegisterInfo &MRI) {
433 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
434 MachineInstr *Def = MRI.getVRegDef(Reg);
435 setInsertPtAfterDef(MIB, Def);
436 if (!SpvType)
437 SpvType = GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder&: MIB,
438 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
439 if (!MRI.getRegClassOrNull(Reg))
440 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType));
441 if (!MRI.getType(Reg).isValid())
442 MRI.setType(VReg: Reg, Ty: GR->getRegType(SpvType));
443 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
444}
445
446void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
447 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
448 SPIRVTypeInst KnownResType) {
449 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
450 for (auto &Op : MI.operands()) {
451 if (!Op.isReg() || Op.isDef())
452 continue;
453 Register OpReg = Op.getReg();
454 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OpReg);
455 if (!SpvType && KnownResType) {
456 SpvType = KnownResType;
457 GR->assignSPIRVTypeToVReg(Type: KnownResType, VReg: OpReg, MF: *MI.getMF());
458 }
459 assert(SpvType);
460 if (!MRI.getRegClassOrNull(Reg: OpReg))
461 MRI.setRegClass(Reg: OpReg, RC: GR->getRegClass(SpvType));
462 if (!MRI.getType(Reg: OpReg).isValid())
463 MRI.setType(VReg: OpReg, Ty: GR->getRegType(SpvType));
464 }
465}
466} // namespace llvm
467
468static void
469generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
470 MachineIRBuilder MIB,
471 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
472 // Get access to information about available extensions
473 const SPIRVSubtarget *ST =
474 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
475
476 MachineRegisterInfo &MRI = MF.getRegInfo();
477 SmallVector<MachineInstr *, 10> ToErase;
478 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
479
480 bool IsExtendedInts =
481 ST->canUseExtension(
482 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
483 ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
484 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
485
486 if (!IsExtendedInts) {
487 // Without arbitrary precision integer extensions, SPIR-V only supports
488 // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40)
489 // must be widened to the next power of two.
490 //
491 // G_TRUNC requires special handling because its semantics depend on the
492 // original destination width. For example:
493 // %dst:s24 = G_TRUNC %src:s64
494 // After widening s24 to s32, we cannot simply do:
495 // %dst:s32 = G_TRUNC %src:s64
496 // because this would keep 32 bits instead of 24. Instead, we insert a
497 // G_AND to mask the value to the original width:
498 // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask
499 // %masked:s64 = G_AND %src:s64, %mask
500 // %dst:s32 = G_TRUNC %masked:s64
501 // If src and dst widen to the same size, G_TRUNC is replaced entirely:
502 // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask
503 // %dst:s64 = G_AND %src:s64, %mask
504 SmallVector<MachineInstr *, 8> TruncToRemove;
505 for (MachineBasicBlock &MBB : MF) {
506 for (MachineInstr &MI : MBB) {
507 unsigned MIOp = MI.getOpcode();
508 if (MIOp != TargetOpcode::G_TRUNC)
509 continue;
510 assert(MI.getNumOperands() == 2);
511 assert(MI.getOperand(0).isReg());
512 assert(MI.getOperand(1).isReg());
513
514 Register DstReg = MI.getOperand(i: 0).getReg();
515 Register SrcReg = MI.getOperand(i: 1).getReg();
516
517 LLT DstTy = MRI.getType(Reg: DstReg);
518 LLT SrcTy = MRI.getType(Reg: SrcReg);
519 assert((DstTy.isScalar() || DstTy.isVector()) &&
520 (SrcTy.isScalar() || SrcTy.isVector()) &&
521 "Expected scalar or vector G_TRUNC types");
522 assert(DstTy.isVector() == SrcTy.isVector() &&
523 "Expected matching scalar/vector G_TRUNC types");
524 assert((!DstTy.isVector() ||
525 DstTy.getElementCount() == SrcTy.getElementCount()) &&
526 "Expected equal vector element counts");
527
528 unsigned OriginalDstWidth = DstTy.getScalarSizeInBits();
529 unsigned OriginalSrcWidth = SrcTy.getScalarSizeInBits();
530
531 unsigned NewDstWidth = widenBitWidthToNextPow2(BitWidth: OriginalDstWidth);
532 unsigned NewSrcWidth = widenBitWidthToNextPow2(BitWidth: OriginalSrcWidth);
533 LLT NewDstTy = DstTy.changeElementSize(NewEltSize: NewDstWidth);
534 LLT NewSrcTy = SrcTy.changeElementSize(NewEltSize: NewSrcWidth);
535
536 // No Dst width change means no truncation semantics change, but the
537 // source still needs a legal type.
538 if (OriginalDstWidth == NewDstWidth) {
539 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
540 continue;
541 }
542
543 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
544 MRI.setType(VReg: DstReg, Ty: NewDstTy);
545
546 MIB.setInsertPt(MBB, II: MI.getIterator());
547 APInt Mask = APInt::getLowBitsSet(numBits: NewSrcWidth, loBitsSet: OriginalDstWidth);
548 MachineInstrBuilder MaskReg =
549 DstTy.isVector()
550 ? MIB.buildBuildVectorConstant(
551 Res: NewSrcTy,
552 Ops: SmallVector<APInt, 4>(DstTy.getNumElements(), Mask))
553 : MIB.buildConstant(Res: NewSrcTy, Val: Mask);
554 Register MaskedReg = MRI.createGenericVirtualRegister(Ty: NewSrcTy);
555 MIB.buildAnd(Dst: MaskedReg, Src0: SrcReg, Src1: MaskReg);
556
557 if (NewSrcWidth == NewDstWidth) {
558 MRI.replaceRegWith(FromReg: DstReg, ToReg: MaskedReg);
559 TruncToRemove.push_back(Elt: &MI);
560 } else {
561 MI.getOperand(i: 1).setReg(MaskedReg);
562 }
563 }
564 }
565 for (MachineInstr *MI : TruncToRemove)
566 MI->eraseFromParent();
567 }
568
569 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
570 if (MBB->empty())
571 continue;
572
573 bool ReachedBegin = false;
574 for (auto MII = std::prev(x: MBB->end()), Begin = MBB->begin();
575 !ReachedBegin;) {
576 MachineInstr &MI = *MII;
577 unsigned MIOp = MI.getOpcode();
578
579 if (!IsExtendedInts) {
580 // validate bit width of scalar registers and constant immediates
581 for (auto &MOP : MI.operands()) {
582 if (MOP.isReg())
583 widenScalarType(Reg: MOP.getReg(), MRI);
584 else if (MOP.isCImm())
585 widenCImmType(MOP);
586 }
587 }
588
589 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) {
590 Register Reg = MI.getOperand(i: 1).getReg();
591 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
592 Type *ElementTy = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
593 SPIRVTypeInst AssignedPtrType = GR->getOrCreateSPIRVPointerType(
594 BaseType: ElementTy, I&: MI,
595 SC: addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 3).getImm(), STI: *ST));
596 // The intrinsic also carries vector-of-pointer values produced by
597 // scalarized vector GEPs; wrap the pointer in OpTypeVector to match
598 // the vreg's LLT.
599 LLT RegTy = MRI.getType(Reg);
600 if (RegTy.isValid() && RegTy.isVector())
601 AssignedPtrType = GR->getOrCreateSPIRVVectorType(
602 BaseType: AssignedPtrType, NumElements: RegTy.getNumElements(), MIRBuilder&: MIB, EmitIR: true);
603 MachineInstr *Def = MRI.getVRegDef(Reg);
604 assert(Def && "Expecting an instruction that defines the register");
605 // G_GLOBAL_VALUE already has type info.
606 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
607 updateRegType(Reg, Ty: nullptr, SpvType: AssignedPtrType, GR, MIB,
608 MRI&: MF.getRegInfo());
609 ToErase.push_back(Elt: &MI);
610 } else if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_type)) {
611 Register Reg = MI.getOperand(i: 1).getReg();
612 Type *Ty = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
613 MachineInstr *Def = MRI.getVRegDef(Reg);
614 assert(Def && "Expecting an instruction that defines the register");
615 // G_GLOBAL_VALUE already has type info.
616 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
617 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
618 ToErase.push_back(Elt: &MI);
619 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
620 MachineInstr *MdMI = MI.getPrevNode();
621 if (MdMI && isSpvIntrinsic(MI: *MdMI, IntrinsicID: Intrinsic::spv_value_md)) {
622 // It's an internal service info from before IRTranslator passes.
623 MachineInstr *Def = getVRegDef(MRI, Reg: MI.getOperand(i: 0).getReg());
624 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
625 if (getVRegDef(MRI, Reg: MI.getOperand(i: I).getReg()) != Def)
626 Def = nullptr;
627 if (Def) {
628 const MDNode *MD = MdMI->getOperand(i: 1).getMetadata();
629 StringRef ValueName =
630 cast<MDString>(Val: MD->getOperand(I: 1))->getString();
631 const MDNode *TypeMD = cast<MDNode>(Val: MD->getOperand(I: 0));
632 Type *ValueTy = getMDOperandAsType(N: TypeMD, I: 0);
633 GR->addValueAttrs(Key: Def, Val: std::make_pair(x&: ValueTy, y: ValueName.str()));
634 }
635 ToErase.push_back(Elt: MdMI);
636 }
637 ToErase.push_back(Elt: &MI);
638 } else if (MIOp == TargetOpcode::G_CONSTANT ||
639 MIOp == TargetOpcode::G_FCONSTANT ||
640 MIOp == TargetOpcode::G_BUILD_VECTOR) {
641 // %rc = G_CONSTANT ty Val
642 // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
643 Register Reg = MI.getOperand(i: 0).getReg();
644 bool NeedAssignType = !GR->getSPIRVTypeForVReg(VReg: Reg);
645 Type *Ty = nullptr;
646 if (MIOp == TargetOpcode::G_CONSTANT) {
647 auto TargetExtIt = TargetExtConstTypes.find(Val: &MI);
648 Ty = TargetExtIt == TargetExtConstTypes.end()
649 ? MI.getOperand(i: 1).getCImm()->getType()
650 : TargetExtIt->second;
651 const ConstantInt *OpCI = MI.getOperand(i: 1).getCImm();
652 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =
653 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe
654 // at this point not i64/i32 constant but null pointer in the
655 // corresponding address space of RegType.getAddressSpace(). This may
656 // help to successfully validate the case when a OpConstantComposite's
657 // constituent has type that does not match Result Type of
658 // OpConstantComposite (see, for example,
659 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).
660 Register PrimaryReg = GR->find(V: OpCI, MF: &MF);
661 if (!PrimaryReg.isValid()) {
662 GR->add(V: OpCI, MI: &MI);
663 } else if (PrimaryReg != Reg &&
664 MRI.getType(Reg) == MRI.getType(Reg: PrimaryReg)) {
665 auto *RCReg = MRI.getRegClassOrNull(Reg);
666 auto *RCPrimary = MRI.getRegClassOrNull(Reg: PrimaryReg);
667 if (!RCReg || RCPrimary == RCReg) {
668 RegsAlreadyAddedToDT[&MI] = PrimaryReg;
669 ToErase.push_back(Elt: &MI);
670 NeedAssignType = false;
671 }
672 }
673 } else if (MIOp == TargetOpcode::G_FCONSTANT) {
674 Ty = MI.getOperand(i: 1).getFPImm()->getType();
675 } else {
676 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
677 Type *ElemTy = nullptr;
678 MachineInstr *ElemMI = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
679 assert(ElemMI);
680
681 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
682 ElemTy = ElemMI->getOperand(i: 1).getCImm()->getType();
683 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
684 ElemTy = ElemMI->getOperand(i: 1).getFPImm()->getType();
685 } else {
686 if (SPIRVTypeInst ElemSpvType =
687 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg(), MF: &MF))
688 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(Ty: ElemSpvType));
689 }
690 if (ElemTy)
691 Ty = VectorType::get(
692 ElementType: ElemTy, NumElements: MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
693 Scalable: false);
694 else
695 NeedAssignType = false;
696 }
697 if (NeedAssignType)
698 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI);
699 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
700 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
701 }
702
703 if (MII == Begin)
704 ReachedBegin = true;
705 else
706 --MII;
707 }
708 }
709 for (MachineInstr *MI : ToErase) {
710 auto It = RegsAlreadyAddedToDT.find(Val: MI);
711 if (It != RegsAlreadyAddedToDT.end())
712 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: It->second);
713 invalidateAndEraseMI(GR, MI);
714 }
715
716 // Address the case when IRTranslator introduces instructions with new
717 // registers without associated SPIRV type.
718 for (MachineBasicBlock &MBB : MF) {
719 for (MachineInstr &MI : MBB) {
720 switch (MI.getOpcode()) {
721 case TargetOpcode::G_TRUNC:
722 case TargetOpcode::G_ANYEXT:
723 case TargetOpcode::G_SEXT:
724 case TargetOpcode::G_ZEXT:
725 case TargetOpcode::G_PTRTOINT:
726 case TargetOpcode::COPY:
727 case TargetOpcode::G_ADDRSPACE_CAST:
728 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
729 break;
730 }
731 }
732 }
733}
734
735static void processInstrsWithTypeFolding(MachineFunction &MF,
736 SPIRVGlobalRegistry *GR,
737 MachineIRBuilder MIB) {
738 MachineRegisterInfo &MRI = MF.getRegInfo();
739 for (MachineBasicBlock &MBB : MF)
740 for (MachineInstr &MI : MBB)
741 if (isTypeFoldingSupported(Opcode: MI.getOpcode()))
742 processInstr(MI, MIB, MRI, GR, KnownResType: nullptr);
743}
744
745static Register
746collectInlineAsmInstrOperands(MachineInstr *MI,
747 SmallVector<unsigned, 4> *Ops = nullptr) {
748 Register DefReg;
749 unsigned StartOp = InlineAsm::MIOp_FirstOperand,
750 AsmDescOp = InlineAsm::MIOp_FirstOperand;
751 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
752 ++Idx) {
753 const MachineOperand &MO = MI->getOperand(i: Idx);
754 if (MO.isMetadata())
755 continue;
756 if (Idx == AsmDescOp && MO.isImm()) {
757 // compute the index of the next operand descriptor
758 const InlineAsm::Flag F(MO.getImm());
759 AsmDescOp += 1 + F.getNumOperandRegisters();
760 continue;
761 }
762 if (MO.isReg() && MO.isDef()) {
763 if (!Ops)
764 return MO.getReg();
765 DefReg = MO.getReg();
766 } else if (Ops) {
767 Ops->push_back(Elt: Idx);
768 }
769 }
770 return DefReg;
771}
772
773static void
774insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
775 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
776 const SmallVector<MachineInstr *> &ToProcess) {
777 MachineRegisterInfo &MRI = MF.getRegInfo();
778 Register AsmTargetReg;
779 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
780 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
781 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
782 MIRBuilder.setInsertPt(MBB&: *I2->getParent(), II: *I2);
783
784 if (!AsmTargetReg.isValid()) {
785 // define vendor specific assembly target or dialect
786 AsmTargetReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
787 MRI.setRegClass(Reg: AsmTargetReg, RC: &SPIRV::iIDRegClass);
788 auto AsmTargetMIB =
789 MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmTargetINTEL).addDef(RegNo: AsmTargetReg);
790 addStringImm(Str: ST.getTargetTripleAsStr(), MIB&: AsmTargetMIB);
791 GR->add(Obj: AsmTargetMIB.getInstr(), MI: AsmTargetMIB);
792 }
793
794 // create types
795 const MDNode *IAMD = I1->getOperand(i: 1).getMetadata();
796 FunctionType *FTy = cast<FunctionType>(Val: getMDOperandAsType(N: IAMD, I: 0));
797 SmallVector<SPIRVTypeInst, 4> ArgTypes;
798 for (const auto &ArgTy : FTy->params())
799 ArgTypes.push_back(Elt: GR->getOrCreateSPIRVType(
800 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true));
801 SPIRVTypeInst RetType =
802 GR->getOrCreateSPIRVType(Type: FTy->getReturnType(), MIRBuilder,
803 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
804 SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
805 Ty: FTy, RetType, ArgTypes, MIRBuilder);
806
807 // define vendor specific assembly instructions string
808 Register AsmReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
809 MRI.setRegClass(Reg: AsmReg, RC: &SPIRV::iIDRegClass);
810 auto AsmMIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmINTEL)
811 .addDef(RegNo: AsmReg)
812 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
813 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncType))
814 .addUse(RegNo: AsmTargetReg);
815 // inline asm string:
816 addStringImm(Str: I2->getOperand(i: InlineAsm::MIOp_AsmString).getSymbolName(),
817 MIB&: AsmMIB);
818 // inline asm constraint string:
819 addStringImm(Str: cast<MDString>(Val: I1->getOperand(i: 2).getMetadata()->getOperand(I: 0))
820 ->getString(),
821 MIB&: AsmMIB);
822 GR->add(Obj: AsmMIB.getInstr(), MI: AsmMIB);
823
824 // calls the inline assembly instruction
825 unsigned ExtraInfo = I2->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
826 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
827 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
828 .addUse(RegNo: AsmReg)
829 .addImm(Val: static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
830
831 Register DefReg = collectInlineAsmInstrOperands(MI: I2);
832 if (!DefReg.isValid()) {
833 DefReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
834 MRI.setRegClass(Reg: DefReg, RC: &SPIRV::iIDRegClass);
835 SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType(
836 Type: Type::getVoidTy(C&: MF.getFunction().getContext()), MIRBuilder,
837 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
838 GR->assignSPIRVTypeToVReg(Type: VoidType, VReg: DefReg, MF);
839 }
840
841 auto AsmCall = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmCallINTEL)
842 .addDef(RegNo: DefReg)
843 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
844 .addUse(RegNo: AsmReg);
845 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
846 AsmCall.addUse(RegNo: I1->getOperand(i: IntrIdx).getReg());
847
848 // IRTranslator gets a bit confused when lowering inline ASM with outputs
849 // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
850 // we have to clean that up here to prevent erroneous trunc casts either on
851 // a struct (for multiple outputs) or same width integers to get lowered
852 // into SPIR-V
853 if (MRI.hasOneUse(RegNo: DefReg)) {
854 MachineInstr &CopyMI = *MRI.use_instr_begin(RegNo: DefReg);
855 if (CopyMI.getOpcode() == TargetOpcode::COPY) {
856 Register CopyDst = CopyMI.getOperand(i: 0).getReg();
857 if (MRI.hasOneUse(RegNo: CopyDst)) {
858 MachineInstr &TruncMI = *MRI.use_instr_begin(RegNo: CopyDst);
859 if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
860 MRI.setType(VReg: DefReg, Ty: GR->getRegType(SpvType: RetType));
861 Register TruncReg = TruncMI.defs().begin()->getReg();
862 MRI.replaceRegWith(FromReg: TruncReg, ToReg: DefReg);
863 invalidateAndEraseMI(GR, MI: &TruncMI);
864 invalidateAndEraseMI(GR, MI: &CopyMI);
865 }
866 }
867 }
868 }
869 }
870 for (MachineInstr *MI : ToProcess)
871 invalidateAndEraseMI(GR, MI);
872}
873
874static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
875 const SPIRVSubtarget &ST,
876 MachineIRBuilder MIRBuilder) {
877 SmallVector<MachineInstr *> ToProcess;
878 for (MachineBasicBlock &MBB : MF) {
879 for (MachineInstr &MI : MBB) {
880 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_inline_asm) ||
881 MI.getOpcode() == TargetOpcode::INLINEASM)
882 ToProcess.push_back(Elt: &MI);
883 }
884 }
885 if (ToProcess.size() == 0)
886 return;
887
888 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly))
889 report_fatal_error(reason: "Inline assembly instructions require the "
890 "following SPIR-V extension: SPV_INTEL_inline_assembly",
891 gen_crash_diag: false);
892
893 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
894}
895
896static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
897 MachineIRBuilder MIB) {
898 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MIB.getMF().getSubtarget());
899 SmallVector<MachineInstr *, 10> ToErase;
900 for (MachineBasicBlock &MBB : MF) {
901 for (MachineInstr &MI : MBB) {
902 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration) &&
903 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_aliasing_decoration) &&
904 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration))
905 continue;
906 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getNextNode());
907 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration)) {
908 buildOpSpirvDecorations(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
909 GVarMD: MI.getOperand(i: 2).getMetadata(), ST);
910 } else if (isSpvIntrinsic(MI,
911 IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration)) {
912 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(
913 MD: MI.getOperand(i: 2).getMetadata()->getOperand(I: 0));
914 uint32_t OpValue = OpV->getValueAPF().bitcastToAPInt().getZExtValue();
915
916 buildOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
917 Dec: SPIRV::Decoration::FPMaxErrorDecorationINTEL,
918 DecArgs: {OpValue});
919 } else {
920 GR->buildMemAliasingOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
921 Dec: MI.getOperand(i: 2).getImm(),
922 GVarMD: MI.getOperand(i: 3).getMetadata());
923 }
924
925 ToErase.push_back(Elt: &MI);
926 }
927 }
928 for (MachineInstr *MI : ToErase)
929 invalidateAndEraseMI(GR, MI);
930}
931
932// LLVM allows the switches to use registers as cases, while SPIR-V required
933// those to be immediate values. This function replaces such operands with the
934// equivalent immediate constant.
935static void processSwitchesConstants(MachineFunction &MF,
936 SPIRVGlobalRegistry *GR,
937 MachineIRBuilder MIB) {
938 MachineRegisterInfo &MRI = MF.getRegInfo();
939 for (MachineBasicBlock &MBB : MF) {
940 for (MachineInstr &MI : MBB) {
941 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch))
942 continue;
943
944 SmallVector<MachineOperand, 8> NewOperands;
945 NewOperands.push_back(Elt: MI.getOperand(i: 0)); // Opcode
946 NewOperands.push_back(Elt: MI.getOperand(i: 1)); // Condition
947 NewOperands.push_back(Elt: MI.getOperand(i: 2)); // Default
948 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
949 Register Reg = MI.getOperand(i).getReg();
950 MachineInstr *ConstInstr = getDefInstrMaybeConstant(ConstReg&: Reg, MRI: &MRI);
951 NewOperands.push_back(
952 Elt: MachineOperand::CreateCImm(CI: ConstInstr->getOperand(i: 1).getCImm()));
953
954 NewOperands.push_back(Elt: MI.getOperand(i: i + 1));
955 }
956
957 assert(MI.getNumOperands() == NewOperands.size());
958 while (MI.getNumOperands() > 0)
959 MI.removeOperand(OpNo: 0);
960 for (auto &MO : NewOperands)
961 MI.addOperand(Op: MO);
962 }
963 }
964}
965
966// Some instructions are used during CodeGen but should never be emitted.
967// Cleaning up those.
968static void cleanupHelperInstructions(MachineFunction &MF,
969 SPIRVGlobalRegistry *GR) {
970 SmallVector<MachineInstr *, 8> ToEraseMI;
971 for (MachineBasicBlock &MBB : MF) {
972 for (MachineInstr &MI : MBB) {
973 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant) ||
974 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
975 ToEraseMI.push_back(Elt: &MI);
976 }
977 }
978
979 for (MachineInstr *MI : ToEraseMI)
980 invalidateAndEraseMI(GR, MI);
981}
982
983// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
984// operands/registers by the actual MBB it references.
985static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
986 MachineIRBuilder MIB) {
987 // Gather the reverse-mapping BB -> MBB.
988 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
989 for (MachineBasicBlock &MBB : MF)
990 BB2MBB[MBB.getBasicBlock()] = &MBB;
991
992 // Gather instructions requiring patching. For now, only those can use
993 // G_BLOCK_ADDR.
994 SmallVector<MachineInstr *, 8> InstructionsToPatch;
995 for (MachineBasicBlock &MBB : MF) {
996 for (MachineInstr &MI : MBB) {
997 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch) ||
998 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_loop_merge) ||
999 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_selection_merge))
1000 InstructionsToPatch.push_back(Elt: &MI);
1001 }
1002 }
1003
1004 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
1005 // the actual MBB it references. Once those references have been updated, we
1006 // can cleanup remaining G_BLOCK_ADDR references.
1007 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
1008 SmallPtrSet<MachineInstr *, 8> ToEraseMI;
1009 MachineRegisterInfo &MRI = MF.getRegInfo();
1010 for (MachineInstr *MI : InstructionsToPatch) {
1011 SmallVector<MachineOperand, 8> NewOps;
1012 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1013 // The operand is not a register, keep as-is.
1014 if (!MI->getOperand(i).isReg()) {
1015 NewOps.push_back(Elt: MI->getOperand(i));
1016 continue;
1017 }
1018
1019 Register Reg = MI->getOperand(i).getReg();
1020 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
1021 // The register is not the result of G_BLOCK_ADDR, keep as-is.
1022 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
1023 NewOps.push_back(Elt: MI->getOperand(i));
1024 continue;
1025 }
1026
1027 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
1028 BuildMBB->getOperand(1).isBlockAddress() &&
1029 BuildMBB->getOperand(1).getBlockAddress());
1030 BasicBlock *BB =
1031 BuildMBB->getOperand(i: 1).getBlockAddress()->getBasicBlock();
1032 auto It = BB2MBB.find(Val: BB);
1033 if (It == BB2MBB.end())
1034 report_fatal_error(reason: "cannot find a machine basic block by a basic block "
1035 "in a switch statement");
1036 MachineBasicBlock *ReferencedBlock = It->second;
1037 NewOps.push_back(Elt: MachineOperand::CreateMBB(MBB: ReferencedBlock));
1038
1039 ClearAddressTaken.insert(Ptr: ReferencedBlock);
1040 ToEraseMI.insert(Ptr: BuildMBB);
1041 }
1042
1043 // Replace the operands.
1044 assert(MI->getNumOperands() == NewOps.size());
1045 while (MI->getNumOperands() > 0)
1046 MI->removeOperand(OpNo: 0);
1047 for (auto &MO : NewOps)
1048 MI->addOperand(Op: MO);
1049
1050 if (MachineInstr *Next = MI->getNextNode()) {
1051 if (isSpvIntrinsic(MI: *Next, IntrinsicID: Intrinsic::spv_track_constant)) {
1052 ToEraseMI.insert(Ptr: Next);
1053 Next = MI->getNextNode();
1054 }
1055 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
1056 ToEraseMI.insert(Ptr: Next);
1057 }
1058 }
1059
1060 // BlockAddress operands were used to keep information between passes,
1061 // let's undo the "address taken" status to reflect that Succ doesn't
1062 // actually correspond to an IR-level basic block.
1063 for (MachineBasicBlock *Succ : ClearAddressTaken)
1064 Succ->setAddressTakenIRBlock(nullptr);
1065
1066 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
1067 // this leaves their BasicBlock counterparts in a "address taken" status. This
1068 // would make AsmPrinter to generate a series of unneeded labels of a "Address
1069 // of block that was removed by CodeGen" kind. Let's first ensure that we
1070 // don't have a dangling BlockAddress constants by zapping the BlockAddress
1071 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
1072 Constant *Replacement =
1073 ConstantInt::get(Ty: Type::getInt32Ty(C&: MF.getFunction().getContext()), V: 1);
1074 for (MachineInstr *BlockAddrI : ToEraseMI) {
1075 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
1076 BlockAddress *BA = const_cast<BlockAddress *>(
1077 BlockAddrI->getOperand(i: 1).getBlockAddress());
1078 BA->replaceAllUsesWith(
1079 V: ConstantExpr::getIntToPtr(C: Replacement, Ty: BA->getType()));
1080 BA->destroyConstant();
1081 }
1082 invalidateAndEraseMI(GR, MI: BlockAddrI);
1083 }
1084}
1085
1086static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
1087 if (MBB.empty())
1088 return MBB.getNextNode() != nullptr;
1089
1090 // Branching SPIR-V intrinsics are not detected by this generic method.
1091 // Thus, we can only trust negative result.
1092 if (!MBB.canFallThrough())
1093 return false;
1094
1095 // Otherwise, we must manually check if we have a SPIR-V intrinsic which
1096 // prevent an implicit fallthrough.
1097 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
1098 It != E; ++It) {
1099 if (isSpvIntrinsic(MI: *It, IntrinsicID: Intrinsic::spv_switch))
1100 return false;
1101 }
1102 return true;
1103}
1104
1105static void removeImplicitFallthroughs(MachineFunction &MF,
1106 MachineIRBuilder MIB) {
1107 // It is valid for MachineBasicBlocks to not finish with a branch instruction.
1108 // In such cases, they will simply fallthrough their immediate successor.
1109 for (MachineBasicBlock &MBB : MF) {
1110 if (!isImplicitFallthrough(MBB))
1111 continue;
1112
1113 assert(MBB.succ_size() == 1);
1114 MIB.setInsertPt(MBB, II: MBB.end());
1115 MIB.buildBr(Dest&: **MBB.successors().begin());
1116 }
1117}
1118
1119bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
1120 // Initialize the type registry.
1121 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
1122 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1123 GR->setCurrentFunc(MF);
1124 MachineIRBuilder MIB(MF);
1125 // a registry of target extension constants
1126 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
1127 // to keep record of tracked constants
1128 addConstantsToTrack(MF, GR, STI: ST, TargetExtConstTypes);
1129 foldConstantsIntoIntrinsics(MF, GR, MIB);
1130 insertBitcasts(MF, GR, MIB);
1131 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
1132
1133 processSwitchesConstants(MF, GR, MIB);
1134 processBlockAddr(MF, GR, MIB);
1135 cleanupHelperInstructions(MF, GR);
1136
1137 processInstrsWithTypeFolding(MF, GR, MIB);
1138 removeImplicitFallthroughs(MF, MIB);
1139 insertSpirvDecorations(MF, GR, MIB);
1140 insertInlineAsm(MF, GR, ST, MIRBuilder: MIB);
1141 lowerBitcasts(MF, GR, MIB);
1142
1143 return true;
1144}
1145
1146INITIALIZE_PASS(SPIRVPreLegalizer, DEBUG_TYPE, "SPIRV pre legalizer", false,
1147 false)
1148
1149char SPIRVPreLegalizer::ID = 0;
1150
1151FunctionPass *llvm::createSPIRVPreLegalizerPass() {
1152 return new SPIRVPreLegalizer();
1153}
1154