1//===-- SPIRVPostLegalizer.cpp - amend info after legalization -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass partially applies pre-legalization logic to new instructions
10// inserted as a result of legalization:
11// - assigns SPIR-V types to registers for new instructions.
12// - inserts ASSIGN_TYPE pseudo-instructions required for type folding.
13//
14//===----------------------------------------------------------------------===//
15
16#include "SPIRV.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVUtils.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
23#include "llvm/CodeGen/MachinePassManager.h"
24#include "llvm/IR/Analysis.h"
25#include "llvm/IR/IntrinsicsSPIRV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "spirv-postlegalizer"
29
30using namespace llvm;
31
32namespace {
33class SPIRVPostLegalizerLegacy : public MachineFunctionPass {
34public:
35 static char ID;
36 SPIRVPostLegalizerLegacy() : MachineFunctionPass(ID) {}
37 bool runOnMachineFunction(MachineFunction &MF) override;
38 MachineFunctionProperties getSetProperties() const override {
39 return MachineFunctionProperties().setRegBankSelected();
40 }
41};
42} // namespace
43
44namespace llvm {
45// Defined in SPIRVPreLegalizer.cpp.
46extern void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpirvTy,
47 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
48 MachineRegisterInfo &MRI);
49extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
50 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
51 SPIRVTypeInst KnownResType);
52} // namespace llvm
53
54static SPIRVTypeInst deduceIntTypeFromResult(Register ResVReg,
55 MachineIRBuilder &MIB,
56 SPIRVGlobalRegistry *GR) {
57 const LLT &Ty = MIB.getMRI()->getType(Reg: ResVReg);
58 SPIRVTypeInst ScalarType =
59 GR->getOrCreateSPIRVIntegerType(BitWidth: Ty.getScalarSizeInBits(), MIRBuilder&: MIB);
60 if (Ty.isVector())
61 return GR->getOrCreateSPIRVVectorType(BaseType: ScalarType, NumElements: Ty.getNumElements(), MIRBuilder&: MIB,
62 EmitIR: false);
63 return ScalarType;
64}
65
66static SPIRVTypeInst deduceTypeFromSingleOperand(MachineInstr *I,
67 MachineIRBuilder &MIB,
68 SPIRVGlobalRegistry *GR,
69 unsigned OpIdx) {
70 Register OpReg = I->getOperand(i: OpIdx).getReg();
71 if (SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg)) {
72 if (SPIRVTypeInst CompType = GR->getScalarOrVectorComponentType(Type: OpType)) {
73 Register ResVReg = I->getOperand(i: 0).getReg();
74 const LLT &ResLLT = MIB.getMRI()->getType(Reg: ResVReg);
75 if (ResLLT.isVector())
76 return GR->getOrCreateSPIRVVectorType(BaseType: CompType, NumElements: ResLLT.getNumElements(),
77 MIRBuilder&: MIB, EmitIR: false);
78 return CompType;
79 }
80 }
81 return nullptr;
82}
83
84static SPIRVTypeInst deduceTypeFromOperandRange(MachineInstr *I,
85 MachineIRBuilder &MIB,
86 SPIRVGlobalRegistry *GR,
87 unsigned StartOp,
88 unsigned EndOp) {
89 SPIRVTypeInst ResType = nullptr;
90 for (unsigned i = StartOp; i < EndOp; ++i) {
91 if (SPIRVTypeInst Type = deduceTypeFromSingleOperand(I, MIB, GR, OpIdx: i)) {
92#ifdef EXPENSIVE_CHECKS
93 assert(!ResType || Type == ResType && "Conflicting type from operands.");
94 ResType = Type;
95#else
96 return Type;
97#endif
98 }
99 }
100 return ResType;
101}
102
103static SPIRVTypeInst deduceTypeFromResultRegister(MachineInstr *Use,
104 Register UseRegister,
105 SPIRVGlobalRegistry *GR,
106 MachineIRBuilder &MIB) {
107 for (const MachineOperand &MO : Use->defs()) {
108 if (!MO.isReg())
109 continue;
110 if (SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: MO.getReg())) {
111 if (SPIRVTypeInst CompType = GR->getScalarOrVectorComponentType(Type: OpType)) {
112 const LLT &ResLLT = MIB.getMRI()->getType(Reg: UseRegister);
113 if (ResLLT.isVector())
114 return GR->getOrCreateSPIRVVectorType(
115 BaseType: CompType, NumElements: ResLLT.getNumElements(), MIRBuilder&: MIB, EmitIR: false);
116 return CompType;
117 }
118 }
119 }
120 return nullptr;
121}
122
123static SPIRVTypeInst
124deducePointerTypeFromResultRegister(MachineInstr *Use, Register UseRegister,
125 SPIRVGlobalRegistry *GR,
126 MachineIRBuilder &MIB) {
127 assert(Use->getOpcode() == TargetOpcode::G_LOAD ||
128 Use->getOpcode() == TargetOpcode::G_STORE);
129
130 Register ValueReg = Use->getOperand(i: 0).getReg();
131 SPIRVTypeInst ValueType = GR->getSPIRVTypeForVReg(VReg: ValueReg);
132 if (!ValueType)
133 return nullptr;
134
135 return GR->getOrCreateSPIRVPointerType(BaseType: ValueType, MIRBuilder&: MIB,
136 SC: SPIRV::StorageClass::Function);
137}
138
139static SPIRVTypeInst deduceTypeFromPointerOperand(MachineInstr *Use,
140 Register UseRegister,
141 SPIRVGlobalRegistry *GR,
142 MachineIRBuilder &MIB) {
143 assert(Use->getOpcode() == TargetOpcode::G_LOAD ||
144 Use->getOpcode() == TargetOpcode::G_STORE);
145
146 Register PtrReg = Use->getOperand(i: 1).getReg();
147 SPIRVTypeInst PtrType = GR->getSPIRVTypeForVReg(VReg: PtrReg);
148 if (!PtrType)
149 return nullptr;
150
151 return GR->getPointeeType(PtrType);
152}
153
154static SPIRVTypeInst deduceTypeFromUses(Register Reg, MachineFunction &MF,
155 SPIRVGlobalRegistry *GR,
156 MachineIRBuilder &MIB) {
157 MachineRegisterInfo &MRI = MF.getRegInfo();
158 for (MachineInstr &Use : MRI.use_nodbg_instructions(Reg)) {
159 SPIRVTypeInst ResType = nullptr;
160 LLVM_DEBUG(dbgs() << "Looking at use " << Use);
161 switch (Use.getOpcode()) {
162 case TargetOpcode::G_BUILD_VECTOR:
163 case TargetOpcode::G_CONCAT_VECTORS:
164 case TargetOpcode::G_SHUFFLE_VECTOR:
165 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
166 case TargetOpcode::G_UNMERGE_VALUES:
167 case TargetOpcode::G_ADD:
168 case TargetOpcode::G_SUB:
169 case TargetOpcode::G_MUL:
170 case TargetOpcode::G_SDIV:
171 case TargetOpcode::G_UDIV:
172 case TargetOpcode::G_SREM:
173 case TargetOpcode::G_UREM:
174 case TargetOpcode::G_FADD:
175 case TargetOpcode::G_FSUB:
176 case TargetOpcode::G_FMUL:
177 case TargetOpcode::G_FDIV:
178 case TargetOpcode::G_FEXP:
179 case TargetOpcode::G_FEXP2:
180 case TargetOpcode::G_FCEIL:
181 case TargetOpcode::G_FFLOOR:
182 case TargetOpcode::G_FREM:
183 case TargetOpcode::G_FMA:
184 case TargetOpcode::G_FACOS:
185 case TargetOpcode::G_FASIN:
186 case TargetOpcode::G_FATAN:
187 case TargetOpcode::G_FATAN2:
188 case TargetOpcode::G_FCOS:
189 case TargetOpcode::G_FSIN:
190 case TargetOpcode::G_FTAN:
191 case TargetOpcode::G_FCOSH:
192 case TargetOpcode::G_FSINH:
193 case TargetOpcode::G_FTANH:
194 case TargetOpcode::G_FLOG:
195 case TargetOpcode::G_FLOG2:
196 case TargetOpcode::G_FLOG10:
197 case TargetOpcode::G_FPOW:
198 case TargetOpcode::G_FMINNUM:
199 case TargetOpcode::G_FMAXNUM:
200 case TargetOpcode::G_FABS:
201 case TargetOpcode::G_FSQRT:
202 case TargetOpcode::COPY:
203 case TargetOpcode::G_STRICT_FMA:
204 case TargetOpcode::G_INTRINSIC_TRUNC:
205 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
206 ResType = deduceTypeFromResultRegister(Use: &Use, UseRegister: Reg, GR, MIB);
207 break;
208 case TargetOpcode::G_SELECT:
209 if (Reg == Use.getOperand(i: 2).getReg() ||
210 Reg == Use.getOperand(i: 3).getReg())
211 ResType = deduceTypeFromResultRegister(Use: &Use, UseRegister: Reg, GR, MIB);
212 break;
213 case TargetOpcode::G_LOAD:
214 case TargetOpcode::G_STORE:
215 if (Reg == Use.getOperand(i: 1).getReg())
216 ResType = deducePointerTypeFromResultRegister(Use: &Use, UseRegister: Reg, GR, MIB);
217 else
218 ResType = deduceTypeFromPointerOperand(Use: &Use, UseRegister: Reg, GR, MIB);
219 break;
220 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
221 case TargetOpcode::G_INTRINSIC: {
222 auto IntrinsicID = cast<GIntrinsic>(Val&: Use).getIntrinsicID();
223 if (IntrinsicID == Intrinsic::spv_insertelt) {
224 if (Reg == Use.getOperand(i: 2).getReg())
225 ResType = deduceTypeFromResultRegister(Use: &Use, UseRegister: Reg, GR, MIB);
226 } else if (IntrinsicID == Intrinsic::spv_extractelt) {
227 if (Reg == Use.getOperand(i: 2).getReg())
228 ResType = deduceTypeFromResultRegister(Use: &Use, UseRegister: Reg, GR, MIB);
229 }
230 break;
231 }
232 }
233 if (ResType) {
234 LLVM_DEBUG(dbgs() << "Deduced type from use " << *ResType);
235 return ResType;
236 }
237 }
238 return nullptr;
239}
240
241static SPIRVTypeInst deduceGEPType(MachineInstr *I, SPIRVGlobalRegistry *GR,
242 MachineIRBuilder &MIB) {
243 LLVM_DEBUG(dbgs() << "Deducing GEP type for: " << *I);
244 Register PtrReg = I->getOperand(i: 3).getReg();
245 SPIRVTypeInst PtrType = GR->getSPIRVTypeForVReg(VReg: PtrReg);
246 if (!PtrType) {
247 LLVM_DEBUG(dbgs() << " Could not get type for pointer operand.\n");
248 return nullptr;
249 }
250
251 SPIRVTypeInst PointeeType = GR->getPointeeType(PtrType);
252 if (!PointeeType) {
253 LLVM_DEBUG(dbgs() << " Could not get pointee type from pointer type.\n");
254 return nullptr;
255 }
256
257 MachineRegisterInfo *MRI = MIB.getMRI();
258
259 // The first index (operand 4) steps over the pointer, so the type doesn't
260 // change.
261 for (unsigned i = 5; i < I->getNumOperands(); ++i) {
262 LLVM_DEBUG(dbgs() << " Traversing index " << i
263 << ", current type: " << *PointeeType);
264 switch (PointeeType->getOpcode()) {
265 case SPIRV::OpTypeArray:
266 case SPIRV::OpTypeRuntimeArray:
267 case SPIRV::OpTypeVector:
268 case SPIRV::OpTypeVectorIdEXT: {
269 Register ElemTypeReg = PointeeType->getOperand(i: 1).getReg();
270 PointeeType = GR->getSPIRVTypeForVReg(VReg: ElemTypeReg);
271 break;
272 }
273 case SPIRV::OpTypeStruct: {
274 MachineOperand &IdxOp = I->getOperand(i);
275 if (!IdxOp.isReg()) {
276 LLVM_DEBUG(dbgs() << " Index is not a register.\n");
277 return nullptr;
278 }
279 MachineInstr *Def = MRI->getVRegDef(Reg: IdxOp.getReg());
280 if (!Def) {
281 LLVM_DEBUG(
282 dbgs() << " Could not find definition for index register.\n");
283 return nullptr;
284 }
285
286 uint64_t IndexVal = foldImm(MO: IdxOp, MRI);
287 if (IndexVal >= PointeeType->getNumOperands() - 1) {
288 LLVM_DEBUG(dbgs() << " Struct index out of bounds.\n");
289 return nullptr;
290 }
291
292 Register MemberTypeReg = PointeeType->getOperand(i: IndexVal + 1).getReg();
293 PointeeType = GR->getSPIRVTypeForVReg(VReg: MemberTypeReg);
294 break;
295 }
296 default:
297 LLVM_DEBUG(dbgs() << " Unknown type opcode for GEP traversal.\n");
298 return nullptr;
299 }
300
301 if (!PointeeType) {
302 LLVM_DEBUG(dbgs() << " Could not resolve next pointee type.\n");
303 return nullptr;
304 }
305 }
306 LLVM_DEBUG(dbgs() << " Final pointee type: " << *PointeeType);
307
308 SPIRV::StorageClass::StorageClass SC = GR->getPointerStorageClass(Type: PtrType);
309 SPIRVTypeInst Res = GR->getOrCreateSPIRVPointerType(BaseType: PointeeType, MIRBuilder&: MIB, SC);
310 LLVM_DEBUG(dbgs() << " Deduced GEP type: " << *Res);
311 return Res;
312}
313
314static SPIRVTypeInst deduceResultTypeFromOperands(MachineInstr *I,
315 SPIRVGlobalRegistry *GR,
316 MachineIRBuilder &MIB) {
317 Register ResVReg = I->getOperand(i: 0).getReg();
318 switch (I->getOpcode()) {
319 case TargetOpcode::G_CONSTANT:
320 case TargetOpcode::G_ANYEXT:
321 case TargetOpcode::G_SEXT:
322 case TargetOpcode::G_ZEXT:
323 case TargetOpcode::G_TRUNC:
324 case TargetOpcode::G_FPTOSI:
325 case TargetOpcode::G_FPTOUI:
326 return deduceIntTypeFromResult(ResVReg, MIB, GR);
327 case TargetOpcode::G_BUILD_VECTOR:
328 return deduceTypeFromOperandRange(I, MIB, GR, StartOp: 1, EndOp: I->getNumOperands());
329 case TargetOpcode::G_SHUFFLE_VECTOR:
330 return deduceTypeFromOperandRange(I, MIB, GR, StartOp: 1, EndOp: 3);
331 case TargetOpcode::G_SELECT:
332 return deduceTypeFromOperandRange(I, MIB, GR, StartOp: 2, EndOp: 4);
333 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
334 case TargetOpcode::G_INTRINSIC: {
335 auto IntrinsicID = cast<GIntrinsic>(Val: I)->getIntrinsicID();
336 if (IntrinsicID == Intrinsic::spv_gep)
337 return deduceGEPType(I, GR, MIB);
338 if (IntrinsicID == Intrinsic::spv_extractelt)
339 return deduceTypeFromSingleOperand(I, MIB, GR, OpIdx: 2);
340 break;
341 }
342 case TargetOpcode::G_LOAD: {
343 SPIRVTypeInst PtrType = deduceTypeFromSingleOperand(I, MIB, GR, OpIdx: 1);
344 return PtrType ? GR->getPointeeType(PtrType) : nullptr;
345 }
346 case TargetOpcode::G_PHI: {
347 for (unsigned Idx = 1; Idx < I->getNumOperands(); Idx += 2) {
348 Register OpReg = I->getOperand(i: Idx).getReg();
349 if (SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(VReg: OpReg))
350 return OpType;
351 }
352 return nullptr;
353 }
354 default:
355 if (I->getNumDefs() == 1 && I->getNumOperands() > 1 &&
356 I->getOperand(i: 1).isReg())
357 return deduceTypeFromSingleOperand(I, MIB, GR, OpIdx: 1);
358 }
359 return nullptr;
360}
361
362static bool deduceAndAssignTypeForGUnmerge(MachineInstr *I, MachineFunction &MF,
363 SPIRVGlobalRegistry *GR,
364 MachineIRBuilder &MIB) {
365 MachineRegisterInfo &MRI = MF.getRegInfo();
366 Register SrcReg = I->getOperand(i: I->getNumOperands() - 1).getReg();
367 SPIRVTypeInst ScalarType = nullptr;
368 if (SPIRVTypeInst DefType = GR->getSPIRVTypeForVReg(VReg: SrcReg)) {
369 assert(isVectorType(DefType));
370 ScalarType = GR->getScalarOrVectorComponentType(Type: DefType);
371 }
372
373 if (!ScalarType) {
374 // If we could not deduce the type from the source, try to deduce it from
375 // the uses of the results.
376 for (unsigned i = 0; i < I->getNumDefs(); ++i) {
377 Register DefReg = I->getOperand(i).getReg();
378 ScalarType = deduceTypeFromUses(Reg: DefReg, MF, GR, MIB);
379 if (ScalarType) {
380 ScalarType = GR->getScalarOrVectorComponentType(Type: ScalarType);
381 break;
382 }
383 }
384 }
385
386 if (!ScalarType)
387 return false;
388
389 for (unsigned i = 0; i < I->getNumOperands(); ++i) {
390 Register DefReg = I->getOperand(i).getReg();
391 if (GR->getSPIRVTypeForVReg(VReg: DefReg))
392 continue;
393
394 LLT DefLLT = MRI.getType(Reg: DefReg);
395 SPIRVTypeInst ResType =
396 DefLLT.isVector()
397 ? GR->getOrCreateSPIRVVectorType(
398 BaseType: ScalarType, NumElements: DefLLT.getNumElements(), I&: *I,
399 TII: *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo())
400 : ScalarType;
401 setRegClassType(Reg: DefReg, SpvType: ResType, GR, MRI: &MRI, MF);
402 }
403 return true;
404}
405
406static bool deduceAndAssignSpirvType(MachineInstr *I, MachineFunction &MF,
407 SPIRVGlobalRegistry *GR,
408 MachineIRBuilder &MIB) {
409 LLVM_DEBUG(dbgs() << "\nProcessing instruction: " << *I);
410 MachineRegisterInfo &MRI = MF.getRegInfo();
411 Register ResVReg = I->getOperand(i: 0).getReg();
412
413 // G_UNMERGE_VALUES is handled separately because it has multiple definitions,
414 // unlike the other instructions which have a single result register. The main
415 // deduction logic is designed for the single-definition case.
416 if (I->getOpcode() == TargetOpcode::G_UNMERGE_VALUES)
417 return deduceAndAssignTypeForGUnmerge(I, MF, GR, MIB);
418
419 LLVM_DEBUG(dbgs() << "Inferring type from operands\n");
420 SPIRVTypeInst ResType = deduceResultTypeFromOperands(I, GR, MIB);
421 if (!ResType) {
422 LLVM_DEBUG(dbgs() << "Inferring type from uses\n");
423 ResType = deduceTypeFromUses(Reg: ResVReg, MF, GR, MIB);
424 }
425
426 if (!ResType)
427 return false;
428
429 LLVM_DEBUG(dbgs() << "Assigned type to " << *I << ": " << *ResType);
430 setRegClassType(Reg: ResVReg, SpvType: ResType, GR, MRI: &MRI, MF);
431 return true;
432}
433
434static bool requiresSpirvType(MachineInstr &I, SPIRVGlobalRegistry *GR,
435 MachineRegisterInfo &MRI) {
436 LLVM_DEBUG(dbgs() << "Checking if instruction requires a SPIR-V type: "
437 << I;);
438 if (I.getNumDefs() == 0) {
439 LLVM_DEBUG(dbgs() << "Instruction does not have a definition.\n");
440 return false;
441 }
442
443 if (!I.isPreISelOpcode()) {
444 LLVM_DEBUG(dbgs() << "Instruction is not a generic instruction.\n");
445 return false;
446 }
447
448 Register ResultRegister = I.defs().begin()->getReg();
449 if (GR->getSPIRVTypeForVReg(VReg: ResultRegister)) {
450 LLVM_DEBUG(dbgs() << "Instruction already has a SPIR-V type.\n");
451 if (!MRI.getRegClassOrNull(Reg: ResultRegister)) {
452 LLVM_DEBUG(dbgs() << "Updating the register class.\n");
453 setRegClassType(Reg: ResultRegister, SpvType: GR->getSPIRVTypeForVReg(VReg: ResultRegister),
454 GR, MRI: &MRI, MF: *GR->CurMF, Force: true);
455 }
456 return false;
457 }
458
459 return true;
460}
461
462static void registerSpirvTypeForNewInstructions(MachineFunction &MF,
463 SPIRVGlobalRegistry *GR) {
464 MachineRegisterInfo &MRI = MF.getRegInfo();
465 SmallVector<MachineInstr *, 8> Worklist;
466 for (MachineBasicBlock &MBB : MF) {
467 for (MachineInstr &I : MBB) {
468 if (requiresSpirvType(I, GR, MRI)) {
469 Worklist.push_back(Elt: &I);
470 }
471 }
472 }
473
474 if (Worklist.empty()) {
475 LLVM_DEBUG(dbgs() << "Initial worklist is empty.\n");
476 return;
477 }
478
479 LLVM_DEBUG(dbgs() << "Initial worklist:\n";
480 for (auto *I : Worklist) { I->dump(); });
481
482 bool Changed;
483 do {
484 Changed = false;
485 SmallVector<MachineInstr *, 8> NextWorklist;
486
487 for (MachineInstr *I : Worklist) {
488 MachineIRBuilder MIB(*I);
489 if (deduceAndAssignSpirvType(I, MF, GR, MIB)) {
490 Changed = true;
491 } else {
492 NextWorklist.push_back(Elt: I);
493 }
494 }
495 Worklist = std::move(NextWorklist);
496 LLVM_DEBUG(dbgs() << "Worklist size: " << Worklist.size() << "\n");
497 } while (Changed);
498
499 if (Worklist.empty())
500 return;
501
502 for (auto *I : Worklist) {
503 MachineIRBuilder MIB(*I);
504 LLVM_DEBUG(dbgs() << "Assigning default type to results in " << *I);
505 for (unsigned Idx = 0; Idx < I->getNumDefs(); ++Idx) {
506 Register ResVReg = I->getOperand(i: Idx).getReg();
507 if (GR->getSPIRVTypeForVReg(VReg: ResVReg))
508 continue;
509 const LLT &ResLLT = MRI.getType(Reg: ResVReg);
510 SPIRVTypeInst ResType = nullptr;
511 if (ResLLT.isVector()) {
512 SPIRVTypeInst CompType = GR->getOrCreateSPIRVIntegerType(
513 BitWidth: ResLLT.getElementType().getSizeInBits(), MIRBuilder&: MIB);
514 ResType = GR->getOrCreateSPIRVVectorType(
515 BaseType: CompType, NumElements: ResLLT.getNumElements(), MIRBuilder&: MIB, EmitIR: false);
516 } else {
517 ResType = GR->getOrCreateSPIRVIntegerType(BitWidth: ResLLT.getSizeInBits(), MIRBuilder&: MIB);
518 }
519 setRegClassType(Reg: ResVReg, SpvType: ResType, GR, MRI: &MRI, MF, Force: true);
520 }
521 }
522}
523
524static bool hasAssignType(Register Reg, MachineRegisterInfo &MRI) {
525 for (MachineInstr &UseInstr : MRI.use_nodbg_instructions(Reg)) {
526 if (UseInstr.getOpcode() == SPIRV::ASSIGN_TYPE) {
527 return true;
528 }
529 }
530 return false;
531}
532
533static void generateAssignType(MachineInstr &MI, Register ResultRegister,
534 SPIRVTypeInst ResultType,
535 SPIRVGlobalRegistry *GR,
536 MachineRegisterInfo &MRI) {
537 LLVM_DEBUG(dbgs() << " Adding ASSIGN_TYPE for ResultRegister: "
538 << printReg(ResultRegister, MRI.getTargetRegisterInfo())
539 << " with type: " << *ResultType);
540 MachineIRBuilder MIB(MI);
541 updateRegType(Reg: ResultRegister, Ty: nullptr, SpirvTy: ResultType, GR, MIB, MRI);
542 MIB.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
543
544 // Tablegen definition assumes SPIRV::ASSIGN_TYPE pseudo-instruction is
545 // present after each auto-folded instruction to take a type reference
546 // from.
547 Register NewReg =
548 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: ResultRegister));
549 const auto *RegClass = GR->getRegClass(SpvType: ResultType);
550 MRI.setRegClass(Reg: NewReg, RC: RegClass);
551 MRI.setRegClass(Reg: ResultRegister, RC: RegClass);
552
553 GR->assignSPIRVTypeToVReg(Type: ResultType, VReg: ResultRegister, MF: MIB.getMF());
554 // This is to make it convenient for Legalizer to get the SPIRVType
555 // when processing the actual MI (i.e. not pseudo one).
556 GR->assignSPIRVTypeToVReg(Type: ResultType, VReg: NewReg, MF: MIB.getMF());
557 // Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to
558 // keep the flags after instruction selection.
559 const uint32_t Flags = MI.getFlags();
560 MIB.buildInstr(Opcode: SPIRV::ASSIGN_TYPE)
561 .addDef(RegNo: ResultRegister)
562 .addUse(RegNo: NewReg)
563 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: ResultType))
564 .setMIFlags(Flags);
565 for (unsigned I = 0, E = MI.getNumDefs(); I != E; ++I) {
566 MachineOperand &MO = MI.getOperand(i: I);
567 if (MO.getReg() == ResultRegister) {
568 MO.setReg(NewReg);
569 break;
570 }
571 }
572}
573
574static void ensureAssignTypeForTypeFolding(MachineFunction &MF,
575 SPIRVGlobalRegistry *GR) {
576 LLVM_DEBUG(dbgs() << "Entering ensureAssignTypeForTypeFolding for function "
577 << MF.getName() << "\n");
578 MachineRegisterInfo &MRI = MF.getRegInfo();
579 for (MachineBasicBlock &MBB : MF) {
580 for (MachineInstr &MI : MBB) {
581 if (!isTypeFoldingSupported(Opcode: MI.getOpcode()))
582 continue;
583
584 LLVM_DEBUG(dbgs() << "Processing instruction: " << MI);
585
586 Register ResultRegister = MI.defs().begin()->getReg();
587 if (hasAssignType(Reg: ResultRegister, MRI)) {
588 LLVM_DEBUG(dbgs() << " Instruction already has ASSIGN_TYPE\n");
589 continue;
590 }
591
592 SPIRVTypeInst ResultType = GR->getSPIRVTypeForVReg(VReg: ResultRegister);
593 generateAssignType(MI, ResultRegister, ResultType, GR, MRI);
594 }
595 }
596}
597
598static bool runPostLegalizer(MachineFunction &MF) {
599 // Initialize the type registry.
600 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
601 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
602 GR->setCurrentFunc(MF);
603 registerSpirvTypeForNewInstructions(MF, GR);
604 ensureAssignTypeForTypeFolding(MF, GR);
605 return true;
606}
607
608INITIALIZE_PASS(SPIRVPostLegalizerLegacy, DEBUG_TYPE, "SPIRV post legalizer",
609 false, false)
610
611char SPIRVPostLegalizerLegacy::ID = 0;
612
613FunctionPass *llvm::createSPIRVPostLegalizerLegacyPass() {
614 return new SPIRVPostLegalizerLegacy();
615}
616
617bool SPIRVPostLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
618 return runPostLegalizer(MF);
619}
620
621PreservedAnalyses
622SPIRVPostLegalizerPass::run(MachineFunction &MF,
623 MachineFunctionAnalysisManager &MFAM) {
624 MFPropsModifier _(*this, MF);
625 return runPostLegalizer(MF) ? getMachineFunctionPassPreservedAnalyses()
626 : PreservedAnalyses::all();
627}
628