1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass prepares IR for legalization: it assigns SPIR-V types to registers
10// and removes intrinsics which holded these types during IR translation.
11// Also it processes constants and registers them in GR to avoid duplication.
12//
13//===----------------------------------------------------------------------===//
14
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
21#include "llvm/IR/Attributes.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/InstrTypes.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/MathExtras.h"
26
27#define DEBUG_TYPE "spirv-prelegalizer"
28
29using namespace llvm;
30
31namespace {
32class SPIRVPreLegalizer : public MachineFunctionPass {
33public:
34 static char ID;
35 SPIRVPreLegalizer() : MachineFunctionPass(ID) {}
36 bool runOnMachineFunction(MachineFunction &MF) override;
37 void getAnalysisUsage(AnalysisUsage &AU) const override;
38};
39} // namespace
40
41void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const {
42 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
43 MachineFunctionPass::getAnalysisUsage(AU);
44}
45
46static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR,
47 MachineInstr *MI) {
48 GR->invalidateMachineInstr(MI);
49 MI->eraseFromParent();
50}
51
52static void
53addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
54 const SPIRVSubtarget &STI,
55 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
56 MachineRegisterInfo &MRI = MF.getRegInfo();
57 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
58 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
59 for (MachineBasicBlock &MBB : MF) {
60 for (MachineInstr &MI : MBB) {
61 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant))
62 continue;
63 ToErase.push_back(Elt: &MI);
64 Register SrcReg = MI.getOperand(i: 2).getReg();
65 auto *Const =
66 cast<Constant>(Val: cast<ConstantAsMetadata>(
67 Val: MI.getOperand(i: 3).getMetadata()->getOperand(I: 0))
68 ->getValue());
69 if (auto *GV = dyn_cast<GlobalValue>(Val: Const)) {
70 Register Reg = GR->find(V: GV, MF: &MF);
71 if (!Reg.isValid()) {
72 GR->add(V: GV, MI: MRI.getVRegDef(Reg: SrcReg));
73 GR->addGlobalObject(V: GV, MF: &MF, R: SrcReg);
74 } else
75 RegsAlreadyAddedToDT[&MI] = Reg;
76 } else {
77 Register Reg = GR->find(V: Const, MF: &MF);
78 if (!Reg.isValid()) {
79 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Val: Const)) {
80 auto *BuildVec = MRI.getVRegDef(Reg: SrcReg);
81 assert(BuildVec &&
82 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
83 GR->add(V: Const, MI: BuildVec);
84 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
85 // Ensure that OpConstantComposite reuses a constant when it's
86 // already created and available in the same machine function.
87 Constant *ElemConst = ConstVec->getElementAsConstant(i);
88 Register ElemReg = GR->find(V: ElemConst, MF: &MF);
89 if (!ElemReg.isValid())
90 GR->add(V: ElemConst,
91 MI: MRI.getVRegDef(Reg: BuildVec->getOperand(i: 1 + i).getReg()));
92 else
93 BuildVec->getOperand(i: 1 + i).setReg(ElemReg);
94 }
95 }
96 if (Const->getType()->isTargetExtTy()) {
97 // remember association so that we can restore it when assign types
98 MachineInstr *SrcMI = MRI.getVRegDef(Reg: SrcReg);
99 if (SrcMI)
100 GR->add(V: Const, MI: SrcMI);
101 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
102 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
103 TargetExtConstTypes[SrcMI] = Const->getType();
104 if (Const->isNullValue()) {
105 MachineBasicBlock &DepMBB = MF.front();
106 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());
107 SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType(
108 Type: Const->getType(), MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite,
109 EmitIR: true);
110 assert(SrcMI && "Expected source instruction to be valid");
111 SrcMI->setDesc(STI.getInstrInfo()->get(Opcode: SPIRV::OpConstantNull));
112 SrcMI->addOperand(Op: MachineOperand::CreateReg(
113 Reg: GR->getSPIRVTypeID(SpirvType: ExtType), isDef: false));
114 }
115 }
116 } else {
117 RegsAlreadyAddedToDT[&MI] = Reg;
118 // This MI is unused and will be removed. If the MI uses
119 // const_composite, it will be unused and should be removed too.
120 assert(MI.getOperand(2).isReg() && "Reg operand is expected");
121 MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
122 if (SrcMI && isSpvIntrinsic(MI: *SrcMI, IntrinsicID: Intrinsic::spv_const_composite))
123 ToEraseComposites.push_back(Elt: SrcMI);
124 }
125 }
126 }
127 }
128 for (MachineInstr *MI : ToErase) {
129 Register Reg = MI->getOperand(i: 2).getReg();
130 auto It = RegsAlreadyAddedToDT.find(Val: MI);
131 if (It != RegsAlreadyAddedToDT.end())
132 Reg = It->second;
133 auto *RC = MRI.getRegClassOrNull(Reg: MI->getOperand(i: 0).getReg());
134 if (!MRI.getRegClassOrNull(Reg) && RC)
135 MRI.setRegClass(Reg, RC);
136 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: Reg);
137 invalidateAndEraseMI(GR, MI);
138 }
139 for (MachineInstr *MI : ToEraseComposites)
140 invalidateAndEraseMI(GR, MI);
141}
142
143static void foldConstantsIntoIntrinsics(MachineFunction &MF,
144 SPIRVGlobalRegistry *GR,
145 MachineIRBuilder MIB) {
146 SmallVector<MachineInstr *, 64> ToErase;
147 for (MachineBasicBlock &MBB : MF) {
148 for (MachineInstr &MI : MBB) {
149 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_name))
150 continue;
151 const MDNode *MD = MI.getOperand(i: 2).getMetadata();
152 StringRef ValueName = cast<MDString>(Val: MD->getOperand(I: 0))->getString();
153 if (ValueName.size() > 0) {
154 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
155 buildOpName(Target: MI.getOperand(i: 1).getReg(), Name: ValueName, MIRBuilder&: MIB);
156 }
157 ToErase.push_back(Elt: &MI);
158 }
159 for (MachineInstr *MI : ToErase)
160 invalidateAndEraseMI(GR, MI);
161 ToErase.clear();
162 }
163}
164
165static MachineInstr *findAssignTypeInstr(Register Reg,
166 MachineRegisterInfo *MRI) {
167 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(RegNo: Reg),
168 IE = MRI->use_instr_end();
169 I != IE; ++I) {
170 MachineInstr *UseMI = &*I;
171 if ((isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_ptr_type) ||
172 isSpvIntrinsic(MI: *UseMI, IntrinsicID: Intrinsic::spv_assign_type)) &&
173 UseMI->getOperand(i: 1).getReg() == Reg)
174 return UseMI;
175 }
176 return nullptr;
177}
178
179static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
180 Register ResVReg, Register OpReg) {
181 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResVReg);
182 SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg);
183 assert(ResType && OpType && "Operand types are expected");
184 if (!GR->isBitcastCompatible(Type1: ResType, Type2: OpType))
185 report_fatal_error(reason: "incompatible result and operand types in a bitcast");
186 MachineRegisterInfo *MRI = MIB.getMRI();
187 if (!MRI->getRegClassOrNull(Reg: ResVReg))
188 MRI->setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: ResType));
189 if (ResType == OpType)
190 MIB.buildInstr(Opcode: TargetOpcode::COPY).addDef(RegNo: ResVReg).addUse(RegNo: OpReg);
191 else
192 MIB.buildInstr(Opcode: SPIRV::OpBitcast)
193 .addDef(RegNo: ResVReg)
194 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResType))
195 .addUse(RegNo: OpReg);
196}
197
198// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.
199// The verifier checks if the source and destination LLTs of a G_BITCAST are
200// different, but this check is too strict for SPIR-V's typed pointers, which
201// may have the same LLT but different SPIRV type (e.g. pointers to different
202// pointee types). By lowering to OpBitcast here, we bypass the verifier's
203// check. See discussion in https://github.com/llvm/llvm-project/pull/110270
204// for more context.
205//
206// We also handle the llvm.spv.bitcast intrinsic here. If the source and
207// destination SPIR-V types are the same, we lower it to a COPY to enable
208// further optimizations like copy propagation.
209static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
210 MachineIRBuilder MIB) {
211 SmallVector<MachineInstr *, 16> ToErase;
212 for (MachineBasicBlock &MBB : MF) {
213 for (MachineInstr &MI : MBB) {
214 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_bitcast)) {
215 Register DstReg = MI.getOperand(i: 0).getReg();
216 Register SrcReg = MI.getOperand(i: 2).getReg();
217 SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(VReg: DstReg);
218 assert(
219 DstType &&
220 "Expected destination SPIR-V type to have been assigned already.");
221 SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(VReg: SrcReg);
222 assert(SrcType &&
223 "Expected source SPIR-V type to have been assigned already.");
224 if (DstType == SrcType) {
225 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
226 MIB.buildCopy(Res: DstReg, Op: SrcReg);
227 ToErase.push_back(Elt: &MI);
228 continue;
229 }
230 }
231
232 if (MI.getOpcode() != TargetOpcode::G_BITCAST)
233 continue;
234
235 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
236 buildOpBitcast(GR, MIB, ResVReg: MI.getOperand(i: 0).getReg(),
237 OpReg: MI.getOperand(i: 1).getReg());
238 ToErase.push_back(Elt: &MI);
239 }
240 }
241 for (MachineInstr *MI : ToErase)
242 invalidateAndEraseMI(GR, MI);
243}
244
245static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
246 MachineIRBuilder MIB) {
247 // Get access to information about available extensions
248 const SPIRVSubtarget *ST =
249 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
250 SmallVector<MachineInstr *, 10> ToErase;
251 for (MachineBasicBlock &MBB : MF) {
252 for (MachineInstr &MI : MBB) {
253 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_ptrcast))
254 continue;
255 assert(MI.getOperand(2).isReg());
256 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI);
257 ToErase.push_back(Elt: &MI);
258 Register Def = MI.getOperand(i: 0).getReg();
259 Register Source = MI.getOperand(i: 2).getReg();
260 Type *ElemTy = getMDOperandAsType(N: MI.getOperand(i: 3).getMetadata(), I: 0);
261 auto SC =
262 isa<FunctionType>(Val: ElemTy) &&
263 ST->canUseExtension(
264 E: SPIRV::Extension::SPV_INTEL_function_pointers)
265 ? SPIRV::StorageClass::CodeSectionINTEL
266 : addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 4).getImm(), STI: *ST);
267 SPIRVTypeInst AssignedPtrType =
268 GR->getOrCreateSPIRVPointerType(BaseType: ElemTy, I&: MI, SC);
269
270 // If the ptrcast would be redundant, replace all uses with the source
271 // register.
272 MachineRegisterInfo *MRI = MIB.getMRI();
273 if (GR->getSPIRVTypeForVReg(VReg: Source) == AssignedPtrType) {
274 // Erase Def's assign type instruction if we are going to replace Def.
275 if (MachineInstr *AssignMI = findAssignTypeInstr(Reg: Def, MRI))
276 ToErase.push_back(Elt: AssignMI);
277 MRI->replaceRegWith(FromReg: Def, ToReg: Source);
278 } else {
279 if (!GR->getSPIRVTypeForVReg(VReg: Def, MF: &MF))
280 GR->assignSPIRVTypeToVReg(Type: AssignedPtrType, VReg: Def, MF);
281 MIB.buildBitcast(Dst: Def, Src: Source);
282 }
283 }
284 }
285 for (MachineInstr *MI : ToErase)
286 invalidateAndEraseMI(GR, MI);
287}
288
289// Translating GV, IRTranslator sometimes generates following IR:
290// %1 = G_GLOBAL_VALUE
291// %2 = COPY %1
292// %3 = G_ADDRSPACE_CAST %2
293//
294// or
295//
296// %1 = G_ZEXT %2
297// G_MEMCPY ... %2 ...
298//
299// New registers have no SPIRV type and no register class info.
300//
301// Set SPIRV type for GV, propagate it from GV to other instructions,
302// also set register classes.
303static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI,
304 SPIRVGlobalRegistry *GR,
305 MachineRegisterInfo &MRI,
306 MachineIRBuilder &MIB) {
307 SPIRVTypeInst SpvType = nullptr;
308 assert(MI && "Machine instr is expected");
309 if (MI->getOperand(i: 0).isReg()) {
310 Register Reg = MI->getOperand(i: 0).getReg();
311 SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
312 if (!SpvType) {
313 switch (MI->getOpcode()) {
314 case TargetOpcode::G_FCONSTANT:
315 case TargetOpcode::G_CONSTANT: {
316 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
317 Type *Ty = MI->getOperand(i: 1).getCImm()->getType();
318 SpvType = GR->getOrCreateSPIRVType(
319 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
320 break;
321 }
322 case TargetOpcode::G_GLOBAL_VALUE: {
323 MIB.setInsertPt(MBB&: *MI->getParent(), II: MI);
324 const GlobalValue *Global = MI->getOperand(i: 1).getGlobal();
325 Type *ElementTy = toTypedPointer(Ty: GR->getDeducedGlobalValueType(Global));
326 auto *Ty = TypedPointerType::get(ElementType: ElementTy,
327 AddressSpace: Global->getType()->getAddressSpace());
328 SpvType = GR->getOrCreateSPIRVType(
329 Type: Ty, MIRBuilder&: MIB, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
330 break;
331 }
332 case TargetOpcode::G_ANYEXT:
333 case TargetOpcode::G_SEXT:
334 case TargetOpcode::G_ZEXT: {
335 if (MI->getOperand(i: 1).isReg()) {
336 if (MachineInstr *DefInstr =
337 MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
338 if (SPIRVTypeInst Def =
339 propagateSPIRVType(MI: DefInstr, GR, MRI, MIB)) {
340 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Type: Def);
341 unsigned ExpectedBW =
342 std::max(a: MRI.getType(Reg).getScalarSizeInBits(), b: CurrentBW);
343 unsigned NumElements = GR->getScalarOrVectorComponentCount(Type: Def);
344 SpvType = GR->getOrCreateSPIRVIntegerType(BitWidth: ExpectedBW, MIRBuilder&: MIB);
345 if (NumElements > 1)
346 SpvType = GR->getOrCreateSPIRVVectorType(BaseType: SpvType, NumElements,
347 MIRBuilder&: MIB, EmitIR: true);
348 }
349 }
350 }
351 break;
352 }
353 case TargetOpcode::G_PTRTOINT:
354 SpvType = GR->getOrCreateSPIRVIntegerType(
355 BitWidth: MRI.getType(Reg).getScalarSizeInBits(), MIRBuilder&: MIB);
356 break;
357 case TargetOpcode::G_TRUNC:
358 case TargetOpcode::G_ADDRSPACE_CAST:
359 case TargetOpcode::G_PTR_ADD:
360 case TargetOpcode::COPY: {
361 MachineOperand &Op = MI->getOperand(i: 1);
362 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Reg: Op.getReg()) : nullptr;
363 if (Def)
364 SpvType = propagateSPIRVType(MI: Def, GR, MRI, MIB);
365 break;
366 }
367 default:
368 break;
369 }
370 if (SpvType) {
371 // check if the address space needs correction
372 LLT RegType = MRI.getType(Reg);
373 if (SpvType->getOpcode() == SPIRV::OpTypePointer &&
374 RegType.isPointer() &&
375 storageClassToAddressSpace(SC: GR->getPointerStorageClass(Type: SpvType)) !=
376 RegType.getAddressSpace()) {
377 const SPIRVSubtarget &ST =
378 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();
379 auto TSC = addressSpaceToStorageClass(AddrSpace: RegType.getAddressSpace(), STI: ST);
380 SpvType = GR->changePointerStorageClass(PtrType: SpvType, SC: TSC, I&: *MI);
381 }
382 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
383 }
384 if (!MRI.getRegClassOrNull(Reg))
385 MRI.setRegClass(Reg, RC: SpvType ? GR->getRegClass(SpvType)
386 : &SPIRV::iIDRegClass);
387 }
388 }
389 return SpvType;
390}
391
392// To support current approach and limitations wrt. bit width here we widen a
393// scalar register with a bit width greater than 1 to valid sizes and cap it to
394// 128 width.
395static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {
396 if (BitWidth == 1)
397 return 1; // No need to widen 1-bit values
398 return std::min(a: std::max<unsigned>(a: PowerOf2Ceil(A: BitWidth), b: 8u), b: 128u);
399}
400
401static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {
402 LLT RegType = MRI.getType(Reg);
403 if (!RegType.isScalar())
404 return;
405 unsigned CurrentWidth = RegType.getScalarSizeInBits();
406 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
407 if (NewWidth != CurrentWidth)
408 MRI.setType(VReg: Reg, Ty: LLT::scalar(SizeInBits: NewWidth));
409}
410
411static void widenCImmType(MachineOperand &MOP) {
412 const ConstantInt *CImmVal = MOP.getCImm();
413 unsigned CurrentWidth = CImmVal->getBitWidth();
414 unsigned NewWidth = widenBitWidthToNextPow2(BitWidth: CurrentWidth);
415 if (NewWidth != CurrentWidth) {
416 // Replace the immediate value with the widened version
417 MOP.setCImm(ConstantInt::get(Context&: CImmVal->getType()->getContext(),
418 V: CImmVal->getValue().zextOrTrunc(width: NewWidth)));
419 }
420}
421
422static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
423 MachineBasicBlock &MBB = *Def->getParent();
424 MachineBasicBlock::iterator DefIt =
425 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();
426 // Skip all the PHI and debug instructions.
427 while (DefIt != MBB.end() &&
428 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))
429 DefIt = std::next(x: DefIt);
430 MIB.setInsertPt(MBB, II: DefIt);
431}
432
433namespace llvm {
434void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType,
435 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
436 MachineRegisterInfo &MRI) {
437 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
438 MachineInstr *Def = MRI.getVRegDef(Reg);
439 setInsertPtAfterDef(MIB, Def);
440 if (!SpvType)
441 SpvType = GR->getOrCreateSPIRVType(Type: Ty, MIRBuilder&: MIB,
442 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
443 if (!MRI.getRegClassOrNull(Reg))
444 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType));
445 if (!MRI.getType(Reg).isValid())
446 MRI.setType(VReg: Reg, Ty: GR->getRegType(SpvType));
447 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: Reg, MF: MIB.getMF());
448}
449
450void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
451 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
452 SPIRVTypeInst KnownResType) {
453 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
454 for (auto &Op : MI.operands()) {
455 if (!Op.isReg() || Op.isDef())
456 continue;
457 Register OpReg = Op.getReg();
458 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OpReg);
459 if (!SpvType && KnownResType) {
460 SpvType = KnownResType;
461 GR->assignSPIRVTypeToVReg(Type: KnownResType, VReg: OpReg, MF: *MI.getMF());
462 }
463 assert(SpvType);
464 if (!MRI.getRegClassOrNull(Reg: OpReg))
465 MRI.setRegClass(Reg: OpReg, RC: GR->getRegClass(SpvType));
466 if (!MRI.getType(Reg: OpReg).isValid())
467 MRI.setType(VReg: OpReg, Ty: GR->getRegType(SpvType));
468 }
469}
470} // namespace llvm
471
472// Sign-sensitive integer ops: their result depends on the value of the input
473// sign bit at position (width-1). On sub-pow2 widths the general widening
474// loop is a pure LLT relabel, which leaves the sign bit at the *original*
475// position instead of the widened MSB. These ops therefore need an explicit
476// G_SEXT_INREG on each value operand to move the sign bit up.
477//
478// Signed-vs-unsigned G_ICMP is distinguished by its predicate operand.
479//
480// TODO: follow-up PRs will add the remaining sign-sensitive opcodes
481// (e.g. G_SMIN/G_SMAX, G_SADDSAT/G_SSUBSAT, signed overflow ops).
482static bool isSignSensitiveOp(const MachineInstr &MI) {
483 switch (MI.getOpcode()) {
484 case TargetOpcode::G_ASHR:
485 case TargetOpcode::G_SDIV:
486 case TargetOpcode::G_SREM:
487 return true;
488 case TargetOpcode::G_ICMP:
489 return CmpInst::isSigned(
490 Pred: static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate()));
491 default:
492 return false;
493 }
494}
495
496struct SignSensitiveWideningInfo {
497 // Width before widening of each value-operand vreg (one entry per vreg).
498 DenseMap<Register, unsigned> OrigWidth;
499 // Ops whose value operand(s) need replacing, ordered for reproducible vreg
500 // numbering.
501 SmallVector<MachineInstr *> Worklist;
502};
503
504// Collect sign-sensitive ops with narrow scalar value operands and their
505// pre-widening widths, before later passes retype those vregs to pow2 LLTs
506// and the original width is no longer recoverable.
507static SignSensitiveWideningInfo
508recordSignSensitiveOperandWidths(MachineFunction &MF,
509 MachineRegisterInfo &MRI) {
510 SignSensitiveWideningInfo Info;
511 auto RecordIfNarrow = [&](Register Reg) {
512 LLT Ty = MRI.getType(Reg);
513 if (!Ty.isScalar())
514 return false;
515 unsigned W = Ty.getScalarSizeInBits();
516 if (widenBitWidthToNextPow2(BitWidth: W) == W)
517 return false;
518 Info.OrigWidth.try_emplace(Key: Reg, Args&: W);
519 return true;
520 };
521 for (MachineBasicBlock &MBB : MF) {
522 for (MachineInstr &MI : MBB) {
523 if (!isSignSensitiveOp(MI))
524 continue;
525 // Value operands are the trailing two, past any def or predicate.
526 unsigned N = MI.getNumOperands();
527 const MachineOperand &LHS = MI.getOperand(i: N - 2);
528 const MachineOperand &RHS = MI.getOperand(i: N - 1);
529 // Sign-sensitive opcodes carry register operands only.
530 assert(LHS.isReg() && RHS.isReg());
531 bool NeedsRewrite = RecordIfNarrow(LHS.getReg());
532 NeedsRewrite = RecordIfNarrow(RHS.getReg()) || NeedsRewrite;
533 if (NeedsRewrite)
534 Info.Worklist.push_back(Elt: &MI);
535 }
536 }
537 return Info;
538}
539
540// For every recorded sign-sensitive op, insert G_SEXT_INREG on each value
541// operand whose original width was narrower than the widened pow2 width and
542// retype the operand's vreg LLT in place to the widened width.
543//
544// Info must have been populated by recordSignSensitiveOperandWidths before
545// other passes retyped the vregs; otherwise the narrow widths needed here
546// are lost.
547//
548// TODO: handle vector operands.
549static void widenSignSensitiveOps(MachineFunction &MF, SPIRVGlobalRegistry *GR,
550 MachineIRBuilder &MIB,
551 MachineRegisterInfo &MRI,
552 const SignSensitiveWideningInfo &Info) {
553 // Emit G_SEXT_INREG from Reg's recorded narrow width; retypes Reg to the
554 // widened width and returns the sign-extended vreg.
555 auto SignExtendReg = [&](Register Reg, unsigned OldW,
556 MachineInstr &MI) -> Register {
557 unsigned NewW = widenBitWidthToNextPow2(BitWidth: OldW);
558 LLT NewLLT = LLT::scalar(SizeInBits: NewW);
559 SPIRVTypeInst SpvTy = GR->getOrCreateSPIRVIntegerType(BitWidth: NewW, MIRBuilder&: MIB);
560 Register SExted = MRI.createGenericVirtualRegister(Ty: NewLLT);
561 GR->assignSPIRVTypeToVReg(Type: SpvTy, VReg: SExted, MF);
562 MRI.setRegClass(Reg: SExted, RC: GR->getRegClass(SpvType: SpvTy));
563 MRI.setType(VReg: Reg, Ty: NewLLT);
564 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
565 MIB.buildSExtInReg(Res: SExted, Op: Reg, ImmOp: OldW);
566 return SExted;
567 };
568
569 // TODO: when the same narrow vreg feeds multiple sign-sensitive ops (e.g.
570 // sdiv %x, %y and srem %x, %y), emit one shared G_SEXT_INREG instead of one
571 // per use.
572 for (MachineInstr *MI : Info.Worklist) {
573 unsigned N = MI->getNumOperands();
574 MachineOperand &LHS = MI->getOperand(i: N - 2);
575 MachineOperand &RHS = MI->getOperand(i: N - 1);
576 Register LHSReg = LHS.getReg();
577 Register RHSReg = RHS.getReg();
578 if (auto It = Info.OrigWidth.find(Val: LHSReg); It != Info.OrigWidth.end())
579 LHS.setReg(SignExtendReg(LHSReg, It->second, *MI));
580 // Same vreg on both sides (e.g. G_ICMP slt %x, %x): reuse the sext just
581 // emitted for LHS instead of emitting a second one.
582 if (RHSReg == LHSReg) {
583 RHS.setReg(LHS.getReg());
584 continue;
585 }
586 if (auto It = Info.OrigWidth.find(Val: RHSReg); It != Info.OrigWidth.end())
587 RHS.setReg(SignExtendReg(RHSReg, It->second, *MI));
588 }
589}
590
591static void
592generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
593 MachineIRBuilder MIB,
594 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
595 // Get access to information about available extensions
596 const SPIRVSubtarget *ST =
597 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
598
599 MachineRegisterInfo &MRI = MF.getRegInfo();
600 SmallVector<MachineInstr *, 10> ToErase;
601 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
602
603 bool IsExtendedInts =
604 ST->canUseExtension(
605 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
606 ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
607 ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
608
609 if (!IsExtendedInts) {
610 // Without arbitrary precision integer extensions, SPIR-V only supports
611 // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40)
612 // must be widened to the next power of two.
613 //
614 // Record the original widths of sign-sensitive operands before either
615 // the G_TRUNC handling or the general widening loop retypes vregs, then
616 // rewrite those ops after G_TRUNC processing using the recorded widths.
617 SignSensitiveWideningInfo SignSensitiveInfo =
618 recordSignSensitiveOperandWidths(MF, MRI);
619
620 // G_TRUNC requires special handling because its semantics depend on the
621 // original destination width. For example:
622 // %dst:s24 = G_TRUNC %src:s64
623 // After widening s24 to s32, we cannot simply do:
624 // %dst:s32 = G_TRUNC %src:s64
625 // because this would keep 32 bits instead of 24. Instead, we insert a
626 // G_AND to mask the value to the original width:
627 // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask
628 // %masked:s64 = G_AND %src:s64, %mask
629 // %dst:s32 = G_TRUNC %masked:s64
630 // If src and dst widen to the same size, G_TRUNC is replaced entirely:
631 // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask
632 // %dst:s64 = G_AND %src:s64, %mask
633 SmallVector<MachineInstr *, 8> TruncToRemove;
634 for (MachineBasicBlock &MBB : MF) {
635 for (MachineInstr &MI : MBB) {
636 unsigned MIOp = MI.getOpcode();
637 if (MIOp != TargetOpcode::G_TRUNC)
638 continue;
639 assert(MI.getNumOperands() == 2);
640 assert(MI.getOperand(0).isReg());
641 assert(MI.getOperand(1).isReg());
642
643 Register DstReg = MI.getOperand(i: 0).getReg();
644 Register SrcReg = MI.getOperand(i: 1).getReg();
645
646 LLT DstTy = MRI.getType(Reg: DstReg);
647 LLT SrcTy = MRI.getType(Reg: SrcReg);
648 assert((DstTy.isScalar() || DstTy.isVector()) &&
649 (SrcTy.isScalar() || SrcTy.isVector()) &&
650 "Expected scalar or vector G_TRUNC types");
651 assert(DstTy.isVector() == SrcTy.isVector() &&
652 "Expected matching scalar/vector G_TRUNC types");
653 assert((!DstTy.isVector() ||
654 DstTy.getElementCount() == SrcTy.getElementCount()) &&
655 "Expected equal vector element counts");
656
657 unsigned OriginalDstWidth = DstTy.getScalarSizeInBits();
658 unsigned OriginalSrcWidth = SrcTy.getScalarSizeInBits();
659
660 unsigned NewDstWidth = widenBitWidthToNextPow2(BitWidth: OriginalDstWidth);
661 unsigned NewSrcWidth = widenBitWidthToNextPow2(BitWidth: OriginalSrcWidth);
662 LLT NewDstTy = DstTy.changeElementSize(NewEltSize: NewDstWidth);
663 LLT NewSrcTy = SrcTy.changeElementSize(NewEltSize: NewSrcWidth);
664
665 // No Dst width change means no truncation semantics change, but the
666 // source still needs a legal type.
667 if (OriginalDstWidth == NewDstWidth) {
668 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
669 continue;
670 }
671
672 MRI.setType(VReg: SrcReg, Ty: NewSrcTy);
673 MRI.setType(VReg: DstReg, Ty: NewDstTy);
674
675 MIB.setInsertPt(MBB, II: MI.getIterator());
676 APInt Mask = APInt::getLowBitsSet(numBits: NewSrcWidth, loBitsSet: OriginalDstWidth);
677 MachineInstrBuilder MaskReg =
678 DstTy.isVector()
679 ? MIB.buildBuildVectorConstant(
680 Res: NewSrcTy,
681 Ops: SmallVector<APInt, 4>(DstTy.getNumElements(), Mask))
682 : MIB.buildConstant(Res: NewSrcTy, Val: Mask);
683 Register MaskedReg = MRI.createGenericVirtualRegister(Ty: NewSrcTy);
684 MIB.buildAnd(Dst: MaskedReg, Src0: SrcReg, Src1: MaskReg);
685
686 if (NewSrcWidth == NewDstWidth) {
687 MRI.replaceRegWith(FromReg: DstReg, ToReg: MaskedReg);
688 TruncToRemove.push_back(Elt: &MI);
689 } else {
690 MI.getOperand(i: 1).setReg(MaskedReg);
691 }
692 }
693 }
694 for (MachineInstr *MI : TruncToRemove)
695 MI->eraseFromParent();
696
697 widenSignSensitiveOps(MF, GR, MIB, MRI, Info: SignSensitiveInfo);
698 }
699
700 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
701 if (MBB->empty())
702 continue;
703
704 bool ReachedBegin = false;
705 for (auto MII = std::prev(x: MBB->end()), Begin = MBB->begin();
706 !ReachedBegin;) {
707 MachineInstr &MI = *MII;
708 unsigned MIOp = MI.getOpcode();
709
710 if (!IsExtendedInts) {
711 // validate bit width of scalar registers and constant immediates
712 for (auto &MOP : MI.operands()) {
713 if (MOP.isReg())
714 widenScalarType(Reg: MOP.getReg(), MRI);
715 else if (MOP.isCImm())
716 widenCImmType(MOP);
717 }
718 }
719
720 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) {
721 Register Reg = MI.getOperand(i: 1).getReg();
722 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
723 Type *ElementTy = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
724 SPIRVTypeInst AssignedPtrType = GR->getOrCreateSPIRVPointerType(
725 BaseType: ElementTy, I&: MI,
726 SC: addressSpaceToStorageClass(AddrSpace: MI.getOperand(i: 3).getImm(), STI: *ST));
727 // The intrinsic also carries vector-of-pointer values produced by
728 // scalarized vector GEPs; wrap the pointer in OpTypeVector to match
729 // the vreg's LLT.
730 LLT RegTy = MRI.getType(Reg);
731 if (RegTy.isValid() && RegTy.isVector())
732 AssignedPtrType = GR->getOrCreateSPIRVVectorType(
733 BaseType: AssignedPtrType, NumElements: RegTy.getNumElements(), MIRBuilder&: MIB, EmitIR: true);
734 MachineInstr *Def = MRI.getVRegDef(Reg);
735 assert(Def && "Expecting an instruction that defines the register");
736 // G_GLOBAL_VALUE already has type info.
737 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
738 updateRegType(Reg, Ty: nullptr, SpvType: AssignedPtrType, GR, MIB,
739 MRI&: MF.getRegInfo());
740 ToErase.push_back(Elt: &MI);
741 } else if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_type)) {
742 Register Reg = MI.getOperand(i: 1).getReg();
743 Type *Ty = getMDOperandAsType(N: MI.getOperand(i: 2).getMetadata(), I: 0);
744 MachineInstr *Def = MRI.getVRegDef(Reg);
745 assert(Def && "Expecting an instruction that defines the register");
746 // G_GLOBAL_VALUE already has type info.
747 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
748 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI&: MF.getRegInfo());
749 ToErase.push_back(Elt: &MI);
750 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
751 MachineInstr *MdMI = MI.getPrevNode();
752 if (MdMI && isSpvIntrinsic(MI: *MdMI, IntrinsicID: Intrinsic::spv_value_md)) {
753 // It's an internal service info from before IRTranslator passes.
754 MachineInstr *Def = getVRegDef(MRI, Reg: MI.getOperand(i: 0).getReg());
755 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
756 if (getVRegDef(MRI, Reg: MI.getOperand(i: I).getReg()) != Def)
757 Def = nullptr;
758 if (Def) {
759 const MDNode *MD = MdMI->getOperand(i: 1).getMetadata();
760 StringRef ValueName =
761 cast<MDString>(Val: MD->getOperand(I: 1))->getString();
762 const MDNode *TypeMD = cast<MDNode>(Val: MD->getOperand(I: 0));
763 Type *ValueTy = getMDOperandAsType(N: TypeMD, I: 0);
764 GR->addValueAttrs(Key: Def, Val: std::make_pair(x&: ValueTy, y: ValueName.str()));
765 }
766 ToErase.push_back(Elt: MdMI);
767 }
768 ToErase.push_back(Elt: &MI);
769 } else if (MIOp == TargetOpcode::G_CONSTANT ||
770 MIOp == TargetOpcode::G_FCONSTANT ||
771 MIOp == TargetOpcode::G_BUILD_VECTOR) {
772 // %rc = G_CONSTANT ty Val
773 // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
774 Register Reg = MI.getOperand(i: 0).getReg();
775 bool NeedAssignType = !GR->getSPIRVTypeForVReg(VReg: Reg);
776 Type *Ty = nullptr;
777 if (MIOp == TargetOpcode::G_CONSTANT) {
778 auto TargetExtIt = TargetExtConstTypes.find(Val: &MI);
779 Ty = TargetExtIt == TargetExtConstTypes.end()
780 ? MI.getOperand(i: 1).getCImm()->getType()
781 : TargetExtIt->second;
782 const ConstantInt *OpCI = MI.getOperand(i: 1).getCImm();
783 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =
784 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe
785 // at this point not i64/i32 constant but null pointer in the
786 // corresponding address space of RegType.getAddressSpace(). This may
787 // help to successfully validate the case when a OpConstantComposite's
788 // constituent has type that does not match Result Type of
789 // OpConstantComposite (see, for example,
790 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).
791 Register PrimaryReg = GR->find(V: OpCI, MF: &MF);
792 if (!PrimaryReg.isValid()) {
793 GR->add(V: OpCI, MI: &MI);
794 } else if (PrimaryReg != Reg &&
795 MRI.getType(Reg) == MRI.getType(Reg: PrimaryReg)) {
796 auto *RCReg = MRI.getRegClassOrNull(Reg);
797 auto *RCPrimary = MRI.getRegClassOrNull(Reg: PrimaryReg);
798 if (!RCReg || RCPrimary == RCReg) {
799 RegsAlreadyAddedToDT[&MI] = PrimaryReg;
800 ToErase.push_back(Elt: &MI);
801 NeedAssignType = false;
802 }
803 }
804 } else if (MIOp == TargetOpcode::G_FCONSTANT) {
805 Ty = MI.getOperand(i: 1).getFPImm()->getType();
806 } else {
807 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
808 Type *ElemTy = nullptr;
809 MachineInstr *ElemMI = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
810 assert(ElemMI);
811
812 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
813 ElemTy = ElemMI->getOperand(i: 1).getCImm()->getType();
814 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
815 ElemTy = ElemMI->getOperand(i: 1).getFPImm()->getType();
816 } else {
817 if (SPIRVTypeInst ElemSpvType =
818 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg(), MF: &MF))
819 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(Ty: ElemSpvType));
820 }
821 if (ElemTy)
822 Ty = VectorType::get(
823 ElementType: ElemTy, NumElements: MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
824 Scalable: false);
825 else
826 NeedAssignType = false;
827 }
828 if (NeedAssignType)
829 updateRegType(Reg, Ty, SpvType: nullptr, GR, MIB, MRI);
830 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
831 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
832 }
833
834 if (MII == Begin)
835 ReachedBegin = true;
836 else
837 --MII;
838 }
839 }
840 for (MachineInstr *MI : ToErase) {
841 auto It = RegsAlreadyAddedToDT.find(Val: MI);
842 if (It != RegsAlreadyAddedToDT.end())
843 MRI.replaceRegWith(FromReg: MI->getOperand(i: 0).getReg(), ToReg: It->second);
844 invalidateAndEraseMI(GR, MI);
845 }
846
847 // Address the case when IRTranslator introduces instructions with new
848 // registers without associated SPIRV type.
849 for (MachineBasicBlock &MBB : MF) {
850 for (MachineInstr &MI : MBB) {
851 switch (MI.getOpcode()) {
852 case TargetOpcode::G_TRUNC:
853 case TargetOpcode::G_ANYEXT:
854 case TargetOpcode::G_SEXT:
855 case TargetOpcode::G_ZEXT:
856 case TargetOpcode::G_PTRTOINT:
857 case TargetOpcode::COPY:
858 case TargetOpcode::G_ADDRSPACE_CAST:
859 propagateSPIRVType(MI: &MI, GR, MRI, MIB);
860 break;
861 }
862 }
863 }
864}
865
866static void processInstrsWithTypeFolding(MachineFunction &MF,
867 SPIRVGlobalRegistry *GR,
868 MachineIRBuilder MIB) {
869 MachineRegisterInfo &MRI = MF.getRegInfo();
870 for (MachineBasicBlock &MBB : MF)
871 for (MachineInstr &MI : MBB)
872 if (isTypeFoldingSupported(Opcode: MI.getOpcode()))
873 processInstr(MI, MIB, MRI, GR, KnownResType: nullptr);
874}
875
876static Register
877collectInlineAsmInstrOperands(MachineInstr *MI,
878 SmallVector<unsigned, 4> *Ops = nullptr) {
879 Register DefReg;
880 unsigned StartOp = InlineAsm::MIOp_FirstOperand,
881 AsmDescOp = InlineAsm::MIOp_FirstOperand;
882 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
883 ++Idx) {
884 const MachineOperand &MO = MI->getOperand(i: Idx);
885 if (MO.isMetadata())
886 continue;
887 if (Idx == AsmDescOp && MO.isImm()) {
888 // compute the index of the next operand descriptor
889 const InlineAsm::Flag F(MO.getImm());
890 AsmDescOp += 1 + F.getNumOperandRegisters();
891 continue;
892 }
893 if (MO.isReg() && MO.isDef()) {
894 if (!Ops)
895 return MO.getReg();
896 DefReg = MO.getReg();
897 } else if (Ops) {
898 Ops->push_back(Elt: Idx);
899 }
900 }
901 return DefReg;
902}
903
904static void
905insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
906 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
907 const SmallVector<MachineInstr *> &ToProcess) {
908 MachineRegisterInfo &MRI = MF.getRegInfo();
909 Register AsmTargetReg;
910 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
911 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
912 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
913 MIRBuilder.setInsertPt(MBB&: *I2->getParent(), II: *I2);
914
915 if (!AsmTargetReg.isValid()) {
916 // define vendor specific assembly target or dialect
917 AsmTargetReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
918 MRI.setRegClass(Reg: AsmTargetReg, RC: &SPIRV::iIDRegClass);
919 auto AsmTargetMIB =
920 MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmTargetINTEL).addDef(RegNo: AsmTargetReg);
921 addStringImm(Str: ST.getTargetTripleAsStr(), MIB&: AsmTargetMIB);
922 GR->add(Obj: AsmTargetMIB.getInstr(), MI: AsmTargetMIB);
923 }
924
925 // create types
926 const MDNode *IAMD = I1->getOperand(i: 1).getMetadata();
927 FunctionType *FTy = cast<FunctionType>(Val: getMDOperandAsType(N: IAMD, I: 0));
928 SmallVector<SPIRVTypeInst, 4> ArgTypes;
929 for (const auto &ArgTy : FTy->params())
930 ArgTypes.push_back(Elt: GR->getOrCreateSPIRVType(
931 Type: ArgTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true));
932 SPIRVTypeInst RetType =
933 GR->getOrCreateSPIRVType(Type: FTy->getReturnType(), MIRBuilder,
934 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
935 SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
936 Ty: FTy, RetType, ArgTypes, MIRBuilder);
937
938 // define vendor specific assembly instructions string
939 Register AsmReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
940 MRI.setRegClass(Reg: AsmReg, RC: &SPIRV::iIDRegClass);
941 auto AsmMIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmINTEL)
942 .addDef(RegNo: AsmReg)
943 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
944 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: FuncType))
945 .addUse(RegNo: AsmTargetReg);
946 // inline asm string:
947 addStringImm(Str: I2->getOperand(i: InlineAsm::MIOp_AsmString).getSymbolName(),
948 MIB&: AsmMIB);
949 // inline asm constraint string:
950 addStringImm(Str: cast<MDString>(Val: I1->getOperand(i: 2).getMetadata()->getOperand(I: 0))
951 ->getString(),
952 MIB&: AsmMIB);
953 GR->add(Obj: AsmMIB.getInstr(), MI: AsmMIB);
954
955 // calls the inline assembly instruction
956 unsigned ExtraInfo = I2->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
957 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
958 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorate)
959 .addUse(RegNo: AsmReg)
960 .addImm(Val: static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
961
962 Register DefReg = collectInlineAsmInstrOperands(MI: I2);
963 if (!DefReg.isValid()) {
964 DefReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
965 MRI.setRegClass(Reg: DefReg, RC: &SPIRV::iIDRegClass);
966 SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType(
967 Type: Type::getVoidTy(C&: MF.getFunction().getContext()), MIRBuilder,
968 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
969 GR->assignSPIRVTypeToVReg(Type: VoidType, VReg: DefReg, MF);
970 }
971
972 auto AsmCall = MIRBuilder.buildInstr(Opcode: SPIRV::OpAsmCallINTEL)
973 .addDef(RegNo: DefReg)
974 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
975 .addUse(RegNo: AsmReg);
976 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
977 AsmCall.addUse(RegNo: I1->getOperand(i: IntrIdx).getReg());
978
979 // IRTranslator gets a bit confused when lowering inline ASM with outputs
980 // and inserts a spurious COPY & TRUNC as registers are assumed to be i64;
981 // we have to clean that up here to prevent erroneous trunc casts either on
982 // a struct (for multiple outputs) or same width integers to get lowered
983 // into SPIR-V
984 if (MRI.hasOneUse(RegNo: DefReg)) {
985 MachineInstr &CopyMI = *MRI.use_instr_begin(RegNo: DefReg);
986 if (CopyMI.getOpcode() == TargetOpcode::COPY) {
987 Register CopyDst = CopyMI.getOperand(i: 0).getReg();
988 if (MRI.hasOneUse(RegNo: CopyDst)) {
989 MachineInstr &TruncMI = *MRI.use_instr_begin(RegNo: CopyDst);
990 if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) {
991 MRI.setType(VReg: DefReg, Ty: GR->getRegType(SpvType: RetType));
992 Register TruncReg = TruncMI.defs().begin()->getReg();
993 MRI.replaceRegWith(FromReg: TruncReg, ToReg: DefReg);
994 invalidateAndEraseMI(GR, MI: &TruncMI);
995 invalidateAndEraseMI(GR, MI: &CopyMI);
996 }
997 }
998 }
999 }
1000 }
1001 for (MachineInstr *MI : ToProcess)
1002 invalidateAndEraseMI(GR, MI);
1003}
1004
1005static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1006 const SPIRVSubtarget &ST,
1007 MachineIRBuilder MIRBuilder) {
1008 SmallVector<MachineInstr *> ToProcess;
1009 for (MachineBasicBlock &MBB : MF) {
1010 for (MachineInstr &MI : MBB) {
1011 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_inline_asm) ||
1012 MI.getOpcode() == TargetOpcode::INLINEASM)
1013 ToProcess.push_back(Elt: &MI);
1014 }
1015 }
1016 if (ToProcess.size() == 0)
1017 return;
1018
1019 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly))
1020 report_fatal_error(reason: "Inline assembly instructions require the "
1021 "following SPIR-V extension: SPV_INTEL_inline_assembly",
1022 gen_crash_diag: false);
1023
1024 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
1025}
1026
1027static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1028 MachineIRBuilder MIB) {
1029 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MIB.getMF().getSubtarget());
1030 SmallVector<MachineInstr *, 10> ToErase;
1031 for (MachineBasicBlock &MBB : MF) {
1032 for (MachineInstr &MI : MBB) {
1033 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration) &&
1034 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_aliasing_decoration) &&
1035 !isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration))
1036 continue;
1037 MIB.setInsertPt(MBB&: *MI.getParent(), II: MI.getNextNode());
1038 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_assign_decoration)) {
1039 buildOpSpirvDecorations(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1040 GVarMD: MI.getOperand(i: 2).getMetadata(), ST);
1041 } else if (isSpvIntrinsic(MI,
1042 IntrinsicID: Intrinsic::spv_assign_fpmaxerror_decoration)) {
1043 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(
1044 MD: MI.getOperand(i: 2).getMetadata()->getOperand(I: 0));
1045 uint32_t OpValue = OpV->getValueAPF().bitcastToAPInt().getZExtValue();
1046
1047 buildOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1048 Dec: SPIRV::Decoration::FPMaxErrorDecorationINTEL,
1049 DecArgs: {OpValue});
1050 } else {
1051 GR->buildMemAliasingOpDecorate(Reg: MI.getOperand(i: 1).getReg(), MIRBuilder&: MIB,
1052 Dec: MI.getOperand(i: 2).getImm(),
1053 GVarMD: MI.getOperand(i: 3).getMetadata());
1054 }
1055
1056 ToErase.push_back(Elt: &MI);
1057 }
1058 }
1059 for (MachineInstr *MI : ToErase)
1060 invalidateAndEraseMI(GR, MI);
1061}
1062
1063// LLVM allows the switches to use registers as cases, while SPIR-V required
1064// those to be immediate values. This function replaces such operands with the
1065// equivalent immediate constant.
1066static void processSwitchesConstants(MachineFunction &MF,
1067 SPIRVGlobalRegistry *GR,
1068 MachineIRBuilder MIB) {
1069 MachineRegisterInfo &MRI = MF.getRegInfo();
1070 for (MachineBasicBlock &MBB : MF) {
1071 for (MachineInstr &MI : MBB) {
1072 if (!isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch))
1073 continue;
1074
1075 SmallVector<MachineOperand, 8> NewOperands;
1076 NewOperands.push_back(Elt: MI.getOperand(i: 0)); // Opcode
1077 NewOperands.push_back(Elt: MI.getOperand(i: 1)); // Condition
1078 NewOperands.push_back(Elt: MI.getOperand(i: 2)); // Default
1079 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {
1080 Register Reg = MI.getOperand(i).getReg();
1081 MachineInstr *ConstInstr = getDefInstrMaybeConstant(ConstReg&: Reg, MRI: &MRI);
1082 NewOperands.push_back(
1083 Elt: MachineOperand::CreateCImm(CI: ConstInstr->getOperand(i: 1).getCImm()));
1084
1085 NewOperands.push_back(Elt: MI.getOperand(i: i + 1));
1086 }
1087
1088 assert(MI.getNumOperands() == NewOperands.size());
1089 while (MI.getNumOperands() > 0)
1090 MI.removeOperand(OpNo: 0);
1091 for (auto &MO : NewOperands)
1092 MI.addOperand(Op: MO);
1093 }
1094 }
1095}
1096
1097// Some instructions are used during CodeGen but should never be emitted.
1098// Cleaning up those.
1099static void cleanupHelperInstructions(MachineFunction &MF,
1100 SPIRVGlobalRegistry *GR) {
1101 SmallVector<MachineInstr *, 8> ToEraseMI;
1102 for (MachineBasicBlock &MBB : MF) {
1103 for (MachineInstr &MI : MBB) {
1104 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_track_constant) ||
1105 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)
1106 ToEraseMI.push_back(Elt: &MI);
1107 }
1108 }
1109
1110 for (MachineInstr *MI : ToEraseMI)
1111 invalidateAndEraseMI(GR, MI);
1112}
1113
1114// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those
1115// operands/registers by the actual MBB it references.
1116static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,
1117 MachineIRBuilder MIB) {
1118 // Gather the reverse-mapping BB -> MBB.
1119 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
1120 for (MachineBasicBlock &MBB : MF)
1121 BB2MBB[MBB.getBasicBlock()] = &MBB;
1122
1123 // Gather instructions requiring patching. For now, only those can use
1124 // G_BLOCK_ADDR.
1125 SmallVector<MachineInstr *, 8> InstructionsToPatch;
1126 for (MachineBasicBlock &MBB : MF) {
1127 for (MachineInstr &MI : MBB) {
1128 if (isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_switch) ||
1129 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_loop_merge) ||
1130 isSpvIntrinsic(MI, IntrinsicID: Intrinsic::spv_selection_merge))
1131 InstructionsToPatch.push_back(Elt: &MI);
1132 }
1133 }
1134
1135 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by
1136 // the actual MBB it references. Once those references have been updated, we
1137 // can cleanup remaining G_BLOCK_ADDR references.
1138 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
1139 SmallPtrSet<MachineInstr *, 8> ToEraseMI;
1140 MachineRegisterInfo &MRI = MF.getRegInfo();
1141 for (MachineInstr *MI : InstructionsToPatch) {
1142 SmallVector<MachineOperand, 8> NewOps;
1143 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1144 // The operand is not a register, keep as-is.
1145 if (!MI->getOperand(i).isReg()) {
1146 NewOps.push_back(Elt: MI->getOperand(i));
1147 continue;
1148 }
1149
1150 Register Reg = MI->getOperand(i).getReg();
1151 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
1152 // The register is not the result of G_BLOCK_ADDR, keep as-is.
1153 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {
1154 NewOps.push_back(Elt: MI->getOperand(i));
1155 continue;
1156 }
1157
1158 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
1159 BuildMBB->getOperand(1).isBlockAddress() &&
1160 BuildMBB->getOperand(1).getBlockAddress());
1161 BasicBlock *BB =
1162 BuildMBB->getOperand(i: 1).getBlockAddress()->getBasicBlock();
1163 auto It = BB2MBB.find(Val: BB);
1164 if (It == BB2MBB.end())
1165 report_fatal_error(reason: "cannot find a machine basic block by a basic block "
1166 "in a switch statement");
1167 MachineBasicBlock *ReferencedBlock = It->second;
1168 NewOps.push_back(Elt: MachineOperand::CreateMBB(MBB: ReferencedBlock));
1169
1170 ClearAddressTaken.insert(Ptr: ReferencedBlock);
1171 ToEraseMI.insert(Ptr: BuildMBB);
1172 }
1173
1174 // Replace the operands.
1175 assert(MI->getNumOperands() == NewOps.size());
1176 while (MI->getNumOperands() > 0)
1177 MI->removeOperand(OpNo: 0);
1178 for (auto &MO : NewOps)
1179 MI->addOperand(Op: MO);
1180
1181 if (MachineInstr *Next = MI->getNextNode()) {
1182 if (isSpvIntrinsic(MI: *Next, IntrinsicID: Intrinsic::spv_track_constant)) {
1183 ToEraseMI.insert(Ptr: Next);
1184 Next = MI->getNextNode();
1185 }
1186 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
1187 ToEraseMI.insert(Ptr: Next);
1188 }
1189 }
1190
1191 // BlockAddress operands were used to keep information between passes,
1192 // let's undo the "address taken" status to reflect that Succ doesn't
1193 // actually correspond to an IR-level basic block.
1194 for (MachineBasicBlock *Succ : ClearAddressTaken)
1195 Succ->setAddressTakenIRBlock(nullptr);
1196
1197 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
1198 // this leaves their BasicBlock counterparts in a "address taken" status. This
1199 // would make AsmPrinter to generate a series of unneeded labels of a "Address
1200 // of block that was removed by CodeGen" kind. Let's first ensure that we
1201 // don't have a dangling BlockAddress constants by zapping the BlockAddress
1202 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
1203 Constant *Replacement =
1204 ConstantInt::get(Ty: Type::getInt32Ty(C&: MF.getFunction().getContext()), V: 1);
1205 for (MachineInstr *BlockAddrI : ToEraseMI) {
1206 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
1207 BlockAddress *BA = const_cast<BlockAddress *>(
1208 BlockAddrI->getOperand(i: 1).getBlockAddress());
1209 BA->replaceAllUsesWith(
1210 V: ConstantExpr::getIntToPtr(C: Replacement, Ty: BA->getType()));
1211 BA->destroyConstant();
1212 }
1213 invalidateAndEraseMI(GR, MI: BlockAddrI);
1214 }
1215}
1216
1217static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
1218 if (MBB.empty())
1219 return MBB.getNextNode() != nullptr;
1220
1221 // Branching SPIR-V intrinsics are not detected by this generic method.
1222 // Thus, we can only trust negative result.
1223 if (!MBB.canFallThrough())
1224 return false;
1225
1226 // Otherwise, we must manually check if we have a SPIR-V intrinsic which
1227 // prevent an implicit fallthrough.
1228 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
1229 It != E; ++It) {
1230 if (isSpvIntrinsic(MI: *It, IntrinsicID: Intrinsic::spv_switch))
1231 return false;
1232 }
1233 return true;
1234}
1235
1236static void removeImplicitFallthroughs(MachineFunction &MF,
1237 MachineIRBuilder MIB) {
1238 // It is valid for MachineBasicBlocks to not finish with a branch instruction.
1239 // In such cases, they will simply fallthrough their immediate successor.
1240 for (MachineBasicBlock &MBB : MF) {
1241 if (!isImplicitFallthrough(MBB))
1242 continue;
1243
1244 assert(MBB.succ_size() == 1);
1245 MIB.setInsertPt(MBB, II: MBB.end());
1246 MIB.buildBr(Dest&: **MBB.successors().begin());
1247 }
1248}
1249
1250bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
1251 // Initialize the type registry.
1252 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
1253 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1254 GR->setCurrentFunc(MF);
1255 MachineIRBuilder MIB(MF);
1256 // a registry of target extension constants
1257 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
1258 // to keep record of tracked constants
1259 addConstantsToTrack(MF, GR, STI: ST, TargetExtConstTypes);
1260 foldConstantsIntoIntrinsics(MF, GR, MIB);
1261 insertBitcasts(MF, GR, MIB);
1262 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
1263
1264 processSwitchesConstants(MF, GR, MIB);
1265 processBlockAddr(MF, GR, MIB);
1266 cleanupHelperInstructions(MF, GR);
1267
1268 processInstrsWithTypeFolding(MF, GR, MIB);
1269 removeImplicitFallthroughs(MF, MIB);
1270 insertSpirvDecorations(MF, GR, MIB);
1271 insertInlineAsm(MF, GR, ST, MIRBuilder: MIB);
1272 lowerBitcasts(MF, GR, MIB);
1273
1274 return true;
1275}
1276
1277INITIALIZE_PASS(SPIRVPreLegalizer, DEBUG_TYPE, "SPIRV pre legalizer", false,
1278 false)
1279
1280char SPIRVPreLegalizer::ID = 0;
1281
1282FunctionPass *llvm::createSPIRVPreLegalizerPass() {
1283 return new SPIRVPreLegalizer();
1284}
1285