1//===-- X86FastPreTileConfig.cpp - Fast Tile Register Configure------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to preconfig the shape of physical tile registers
10/// It inserts ldtilecfg ahead of each group of tile registers. The algorithm
11/// walk each instruction of basic block in reverse order. All the tile
12/// registers that live out the basic block would be spilled and reloaded
13/// before its user. It also check the depenedency of the shape to ensure
14/// the shape is defined before ldtilecfg.
15//
16//===----------------------------------------------------------------------===//
17
18#include "X86.h"
19#include "X86InstrBuilder.h"
20#include "X86MachineFunctionInfo.h"
21#include "X86RegisterInfo.h"
22#include "X86Subtarget.h"
23#include "llvm/ADT/PostOrderIterator.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachinePassManager.h"
30#include "llvm/CodeGen/MachineRegisterInfo.h"
31#include "llvm/CodeGen/Passes.h"
32#include "llvm/CodeGen/TargetInstrInfo.h"
33#include "llvm/CodeGen/TargetRegisterInfo.h"
34#include "llvm/IR/Analysis.h"
35#include "llvm/Support/Debug.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "x86-fast-pre-tile-config"
40
41STATISTIC(NumStores, "Number of stores added");
42STATISTIC(NumLoads, "Number of loads added");
43
44namespace {
45
46class X86FastPreTileConfigImpl {
47public:
48 X86FastPreTileConfigImpl() : StackSlotForVirtReg(-1) {}
49 bool runOnMachineFunction(MachineFunction &MF);
50
51private:
52 MachineFunction *MF = nullptr;
53 const X86Subtarget *ST = nullptr;
54 const TargetInstrInfo *TII = nullptr;
55 MachineRegisterInfo *MRI = nullptr;
56 X86MachineFunctionInfo *X86FI = nullptr;
57 MachineFrameInfo *MFI = nullptr;
58 const TargetRegisterInfo *TRI = nullptr;
59 MachineBasicBlock *MBB = nullptr;
60 int CfgSS = -1;
61 struct PHIInfo {
62 Register Row;
63 Register Col;
64 Register StackAddr;
65 };
66 DenseMap<MachineInstr *, struct PHIInfo> VisitedPHIs;
67
68 /// Maps virtual regs to the frame index where these values are spilled.
69 IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;
70
71 /// Has a bit set for tile virtual register for which it was determined
72 /// that it is alive across blocks.
73 BitVector MayLiveAcrossBlocks;
74
75 int getStackSpaceFor(Register VirtReg);
76 void InitializeTileConfigStackSpace();
77 bool mayLiveOut(Register VirtReg, MachineInstr *CfgMI);
78 void spill(MachineBasicBlock::iterator Before, Register VirtReg, bool Kill);
79 void reload(MachineBasicBlock::iterator UseMI, Register VirtReg,
80 MachineOperand *RowMO, MachineOperand *ColMO);
81 void canonicalizePHIs(MachineBasicBlock &MBB);
82 void convertPHI(MachineBasicBlock *MBB, MachineInstr &PHI);
83 void convertPHIs(MachineBasicBlock &MBB);
84 bool configBasicBlock(MachineBasicBlock &MBB);
85};
86
87class X86FastPreTileConfigLegacy : public MachineFunctionPass {
88public:
89 X86FastPreTileConfigLegacy() : MachineFunctionPass(ID) {}
90
91 /// Return the pass name.
92 StringRef getPassName() const override {
93 return "Fast Tile Register Preconfigure";
94 }
95
96 /// Perform tile register configure.
97 bool runOnMachineFunction(MachineFunction &MFunc) override;
98
99 static char ID;
100};
101
102} // end anonymous namespace
103
104char X86FastPreTileConfigLegacy::ID = 0;
105
106INITIALIZE_PASS_BEGIN(X86FastPreTileConfigLegacy, DEBUG_TYPE,
107 "Fast Tile Register Preconfigure", false, false)
108INITIALIZE_PASS_END(X86FastPreTileConfigLegacy, DEBUG_TYPE,
109 "Fast Tile Register Preconfigure", false, false)
110
111static bool dominates(MachineBasicBlock &MBB,
112 MachineBasicBlock::const_iterator A,
113 MachineBasicBlock::const_iterator B) {
114 auto MBBEnd = MBB.end();
115 if (B == MBBEnd)
116 return true;
117
118 MachineBasicBlock::const_iterator I = MBB.begin();
119 for (; &*I != A && &*I != B; ++I)
120 ;
121
122 return &*I == A;
123}
124
125/// This allocates space for the specified virtual register to be held on the
126/// stack.
127int X86FastPreTileConfigImpl::getStackSpaceFor(Register VirtReg) {
128 // Find the location Reg would belong...
129 int SS = StackSlotForVirtReg[VirtReg];
130 // Already has space allocated?
131 if (SS != -1)
132 return SS;
133
134 // Allocate a new stack object for this spill location...
135 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
136 unsigned Size = TRI->getSpillSize(RC);
137 Align Alignment = TRI->getSpillAlign(RC);
138 int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);
139
140 // Assign the slot.
141 StackSlotForVirtReg[VirtReg] = FrameIdx;
142 return FrameIdx;
143}
144
145/// Returns false if \p VirtReg is known to not live out of the current config.
146/// If \p VirtReg live out of the current MBB, it must live out of the current
147/// config
148bool X86FastPreTileConfigImpl::mayLiveOut(Register VirtReg,
149 MachineInstr *CfgMI) {
150 if (MayLiveAcrossBlocks.test(Idx: VirtReg.virtRegIndex()))
151 return true;
152
153 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: VirtReg)) {
154 if (UseInst.getParent() != MBB) {
155 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
156 return true;
157 }
158
159 // The use and def are in the same MBB. If the tile register is
160 // reconfigured, it is crobbered and we need to spill and reload
161 // tile register.
162 if (CfgMI) {
163 if (dominates(MBB&: *MBB, A: *CfgMI, B: UseInst)) {
164 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
165 return true;
166 }
167 }
168 }
169
170 return false;
171}
172
173void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
174 MachineBasicBlock &MBB = MF->front();
175 MachineInstr *MI = &*MBB.getFirstNonPHI();
176 DebugLoc DL;
177 if (ST->hasAVX512()) {
178 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128XRegClass);
179 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::AVX512_128_SET0), DestReg: Xmm);
180 Register Zmm = MRI->createVirtualRegister(RegClass: &X86::VR512RegClass);
181 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::SUBREG_TO_REG), DestReg: Zmm)
182 .addReg(RegNo: Xmm)
183 .addImm(Val: X86::sub_xmm);
184 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSZmr)), FI: CfgSS)
185 .addReg(RegNo: Zmm);
186 } else if (ST->hasAVX2()) {
187 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128RegClass);
188 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::V_SET0), DestReg: Xmm);
189 Register Ymm = MRI->createVirtualRegister(RegClass: &X86::VR256RegClass);
190 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::SUBREG_TO_REG), DestReg: Ymm)
191 .addReg(RegNo: Xmm)
192 .addImm(Val: X86::sub_xmm);
193 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSYmr)), FI: CfgSS)
194 .addReg(RegNo: Ymm);
195 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSYmr)), FI: CfgSS,
196 Offset: 32)
197 .addReg(RegNo: Ymm);
198 } else {
199 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled");
200 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;
201 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128RegClass);
202 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::V_SET0), DestReg: Xmm);
203 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS)
204 .addReg(RegNo: Xmm);
205 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 16)
206 .addReg(RegNo: Xmm);
207 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 32)
208 .addReg(RegNo: Xmm);
209 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 48)
210 .addReg(RegNo: Xmm);
211 }
212 // Fill in the palette first.
213 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV8mi)), FI: CfgSS)
214 .addImm(Val: 1);
215}
216
217/// Insert spill instruction for \p AssignedReg before \p Before.
218/// TODO: Update DBG_VALUEs with \p VirtReg operands with the stack slot.
219void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
220 Register VirtReg, bool Kill) {
221 LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " \n");
222 int FI = getStackSpaceFor(VirtReg);
223 LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');
224
225 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
226 // Don't need shape information for tile store, becasue it is adjacent to
227 // the tile def instruction.
228 TII->storeRegToStackSlot(MBB&: *MBB, MI: Before, SrcReg: VirtReg, isKill: Kill, FrameIndex: FI, RC: &RC, VReg: Register());
229 ++NumStores;
230
231 // TODO: update DBG_VALUEs
232}
233
234/// Insert reload instruction for \p PhysReg before \p Before.
235void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
236 Register OrigReg, MachineOperand *RowMO,
237 MachineOperand *ColMO) {
238 int FI = getStackSpaceFor(VirtReg: OrigReg);
239 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: OrigReg);
240 Register TileReg;
241 // Fold copy to tileload
242 // BB1:
243 // spill src to s
244 //
245 // BB2:
246 // t = copy src
247 // -->
248 // t = tileload (s)
249 if (UseMI->isCopy())
250 TileReg = UseMI->getOperand(i: 0).getReg();
251 else
252 TileReg = MRI->createVirtualRegister(RegClass: &RC);
253 // Can't use TII->loadRegFromStackSlot(), because we need the shape
254 // information for reload.
255 // tileloadd (%sp, %idx), %tmm
256 unsigned Opc = X86::PTILELOADDV;
257 Register StrideReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
258 // FIXME: MBB is not the parent of UseMI.
259 MachineInstr *NewMI = BuildMI(BB&: *UseMI->getParent(), I: UseMI, MIMD: DebugLoc(),
260 MCID: TII->get(Opcode: X86::MOV64ri), DestReg: StrideReg)
261 .addImm(Val: 64);
262 NewMI = addFrameReference(
263 MIB: BuildMI(BB&: *UseMI->getParent(), I: UseMI, MIMD: DebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: TileReg)
264 .addReg(RegNo: RowMO->getReg())
265 .addReg(RegNo: ColMO->getReg()),
266 FI);
267 MachineOperand &MO = NewMI->getOperand(i: 5);
268 MO.setReg(StrideReg);
269 RowMO->setIsKill(false);
270 ColMO->setIsKill(false);
271 // Erase copy instruction after it is folded.
272 if (UseMI->isCopy()) {
273 UseMI->eraseFromParent();
274 } else {
275 // Replace the register in the user MI.
276 for (auto &MO : UseMI->operands()) {
277 if (MO.isReg() && MO.getReg() == OrigReg)
278 MO.setReg(TileReg);
279 }
280 }
281
282 ++NumLoads;
283 LLVM_DEBUG(dbgs() << "Reloading " << printReg(OrigReg, TRI) << " into "
284 << printReg(TileReg, TRI) << '\n');
285}
286
287static bool isTileRegister(MachineRegisterInfo *MRI, Register Reg) {
288 if (Reg.isVirtual() &&
289 (MRI->getRegClass(Reg)->getID() == X86::TILERegClassID)) {
290 return true;
291 }
292
293 if (Reg >= X86::TMM0 && Reg <= X86::TMM7)
294 return true;
295
296 return false;
297}
298
299static bool isTileDef(MachineRegisterInfo *MRI, MachineInstr &MI) {
300 // The instruction must have 3 operands: tile def, row, col.
301 if (MI.isDebugInstr() || MI.getNumOperands() < 3 || !MI.isPseudo())
302 return false;
303 MachineOperand &MO = MI.getOperand(i: 0);
304
305 if (!MO.isReg())
306 return false;
307
308 return isTileRegister(MRI, Reg: MO.getReg());
309}
310
311static ShapeT getShape(MachineRegisterInfo *MRI, Register TileReg) {
312 MachineInstr *MI = MRI->getVRegDef(Reg: TileReg);
313 if (isTileDef(MRI, MI&: *MI)) {
314 MachineOperand *RowMO = &MI->getOperand(i: 1);
315 MachineOperand *ColMO = &MI->getOperand(i: 2);
316 return ShapeT(RowMO, ColMO, MRI);
317 } else if (MI->isCopy()) {
318 TileReg = MI->getOperand(i: 1).getReg();
319 return getShape(MRI, TileReg);
320 }
321
322 // The def should not be PHI node, because we walk the MBB in reverse post
323 // order.
324 assert(MI->isPHI() && "Unexpected PHI when get shape.");
325 llvm_unreachable("Unexpected MI when get shape.");
326}
327
328// BB0:
329// spill t0 to s0
330// BB1:
331// spill t1 to s1
332//
333// BB2:
334// t = phi [t0, bb0] [t1, bb1]
335// -->
336// row = phi [r0, bb0] [r1, bb1]
337// col = phi [c0, bb0] [c1, bb1]
338// s = phi [s0, bb0] [s1, bb1]
339// t = tileload row, col, s
340// The new instruction is inserted at the end of the phi node. The order
341// of the original phi node is not ensured.
342void X86FastPreTileConfigImpl::convertPHI(MachineBasicBlock *MBB,
343 MachineInstr &PHI) {
344 // 1. Create instruction to get stack slot address of each incoming block.
345 // 2. Create PHI node for the stack address.
346 // 3. Create PHI node for shape. If one of the incoming shape is immediate
347 // use the immediate and delete the PHI node.
348 // 4. Create tileload instruction from the stack address.
349 Register StackAddrReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
350 MachineInstrBuilder AddrPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
351 MCID: TII->get(Opcode: X86::PHI), DestReg: StackAddrReg);
352 Register RowReg = MRI->createVirtualRegister(RegClass: &X86::GR16RegClass);
353 MachineInstrBuilder RowPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
354 MCID: TII->get(Opcode: X86::PHI), DestReg: RowReg);
355 Register ColReg = MRI->createVirtualRegister(RegClass: &X86::GR16RegClass);
356 MachineInstrBuilder ColPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
357 MCID: TII->get(Opcode: X86::PHI), DestReg: ColReg);
358 // Record the mapping of phi node and its row/column information.
359 VisitedPHIs[&PHI] = {.Row: RowReg, .Col: ColReg, .StackAddr: StackAddrReg};
360
361 for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) {
362 // Get the 2 incoming value of tile register and MBB.
363 Register InTileReg = PHI.getOperand(i: I).getReg();
364 // Mark it as liveout, so that it will be spilled when visit
365 // the incoming MBB. Otherwise since phi will be deleted, it
366 // would miss spill when visit incoming MBB.
367 MayLiveAcrossBlocks.set(InTileReg.virtRegIndex());
368 MachineBasicBlock *InMBB = PHI.getOperand(i: I + 1).getMBB();
369
370 MachineInstr *TileDefMI = MRI->getVRegDef(Reg: InTileReg);
371 MachineBasicBlock::iterator InsertPos;
372 if (TileDefMI->isPHI()) {
373 InsertPos = TileDefMI->getParent()->getFirstNonPHI();
374 if (auto It = VisitedPHIs.find(Val: TileDefMI);
375 It != VisitedPHIs.end()) { // circular phi reference
376 // def t1
377 // / \
378 // def t2 t3 = phi(t1, t4) <--
379 // \ / |
380 // t4 = phi(t2, t3)-------------
381 //
382 // For each (row, column and stack address) append phi incoming value.
383 // Create r3 = phi(r1, r4)
384 // Create r4 = phi(r2, r3)
385 Register InRowReg = It->second.Row;
386 Register InColReg = It->second.Col;
387 Register InStackAddrReg = It->second.StackAddr;
388 RowPHI.addReg(RegNo: InRowReg).addMBB(MBB: InMBB);
389 ColPHI.addReg(RegNo: InColReg).addMBB(MBB: InMBB);
390 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
391 continue;
392 } else {
393 // Recursively convert PHI to tileload
394 convertPHI(MBB: TileDefMI->getParent(), PHI&: *TileDefMI);
395 // The PHI node is coverted to tileload instruction. Get the stack
396 // address from tileload operands.
397 MachineInstr *TileLoad = MRI->getVRegDef(Reg: InTileReg);
398 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV);
399 Register InRowReg = TileLoad->getOperand(i: 1).getReg();
400 Register InColReg = TileLoad->getOperand(i: 2).getReg();
401 Register InStackAddrReg = TileLoad->getOperand(i: 3).getReg();
402 RowPHI.addReg(RegNo: InRowReg).addMBB(MBB: InMBB);
403 ColPHI.addReg(RegNo: InColReg).addMBB(MBB: InMBB);
404 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
405 }
406 } else {
407 InsertPos = TileDefMI->getIterator();
408
409 // Fill the incoming operand of row/column phi instruction.
410 ShapeT Shape = getShape(MRI, TileReg: InTileReg);
411 Shape.getRow()->setIsKill(false);
412 Shape.getCol()->setIsKill(false);
413 RowPHI.addReg(RegNo: Shape.getRow()->getReg()).addMBB(MBB: InMBB);
414 ColPHI.addReg(RegNo: Shape.getCol()->getReg()).addMBB(MBB: InMBB);
415
416 // The incoming tile register live out of its def BB, it would be spilled.
417 // Create MI to get the spill stack slot address for the tile register
418 int FI = getStackSpaceFor(VirtReg: InTileReg);
419 Register InStackAddrReg =
420 MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
421 addOffset(MIB: BuildMI(BB&: *TileDefMI->getParent(), I: InsertPos, MIMD: DebugLoc(),
422 MCID: TII->get(Opcode: X86::LEA64r), DestReg: InStackAddrReg)
423 .addFrameIndex(Idx: FI),
424 Offset: 0);
425 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
426 }
427 }
428
429 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
430 Register StrideReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
431 BuildMI(BB&: *MBB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::MOV64ri), DestReg: StrideReg)
432 .addImm(Val: 64);
433 Register TileReg = PHI.getOperand(i: 0).getReg();
434 MachineInstr *NewMI = addDirectMem(
435 MIB: BuildMI(BB&: *MBB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PTILELOADDV), DestReg: TileReg)
436 .addReg(RegNo: RowReg)
437 .addReg(RegNo: ColReg),
438 Reg: StackAddrReg);
439 MachineOperand &MO = NewMI->getOperand(i: 5);
440 MO.setReg(StrideReg);
441 PHI.eraseFromParent();
442 VisitedPHIs.erase(Val: &PHI);
443}
444
445static bool isTileRegDef(MachineRegisterInfo *MRI, MachineInstr &MI) {
446 MachineOperand &MO = MI.getOperand(i: 0);
447 if (MO.isReg() && MO.getReg().isVirtual() && isTileRegister(MRI, Reg: MO.getReg()))
448 return true;
449 return false;
450}
451
452void X86FastPreTileConfigImpl::canonicalizePHIs(MachineBasicBlock &MBB) {
453 SmallVector<MachineInstr *, 8> PHIs;
454
455 for (MachineInstr &MI : MBB) {
456 if (!MI.isPHI())
457 break;
458 if (!isTileRegDef(MRI, MI))
459 continue;
460 PHIs.push_back(Elt: &MI);
461 }
462 // Canonicalize the phi node first. One tile phi may depeneds previous
463 // phi node. For below case, we need convert %t4.
464 //
465 // BB0:
466 // %t3 = phi (t1 BB1, t2 BB0)
467 // %t4 = phi (t5 BB1, t3 BB0)
468 // -->
469 // %t3 = phi (t1 BB1, t2 BB0)
470 // %t4 = phi (t5 BB1, t2 BB0)
471 //
472 while (!PHIs.empty()) {
473 MachineInstr *PHI = PHIs.pop_back_val();
474
475 // Find the operand that is incoming from the same MBB and the def
476 // is also phi node.
477 MachineOperand *InMO = nullptr;
478 MachineInstr *DefMI = nullptr;
479 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) {
480 Register InTileReg = PHI->getOperand(i: I).getReg();
481 MachineBasicBlock *InMBB = PHI->getOperand(i: I + 1).getMBB();
482 DefMI = MRI->getVRegDef(Reg: InTileReg);
483 if (InMBB != &MBB || !DefMI->isPHI())
484 continue;
485
486 InMO = &PHI->getOperand(i: I);
487 break;
488 }
489 // If can't find such operand, do nothing.
490 if (!InMO)
491 continue;
492
493 // Current phi node depends on previous phi node. Break the
494 // dependency.
495 Register DefTileReg;
496 for (unsigned I = 1, E = DefMI->getNumOperands(); I != E; I += 2) {
497 MachineBasicBlock *InMBB = PHI->getOperand(i: I + 1).getMBB();
498 if (InMBB != &MBB)
499 continue;
500 DefTileReg = DefMI->getOperand(i: I).getReg();
501 InMO->setReg(DefTileReg);
502 break;
503 }
504 }
505}
506
507void X86FastPreTileConfigImpl::convertPHIs(MachineBasicBlock &MBB) {
508 SmallVector<MachineInstr *, 8> PHIs;
509 for (MachineInstr &MI : MBB) {
510 if (!MI.isPHI())
511 break;
512 if (!isTileRegDef(MRI, MI))
513 continue;
514 PHIs.push_back(Elt: &MI);
515 }
516 while (!PHIs.empty()) {
517 MachineInstr *MI = PHIs.pop_back_val();
518 VisitedPHIs.clear();
519 convertPHI(MBB: &MBB, PHI&: *MI);
520 }
521}
522
523// PreTileConfig should configure the tile registers based on basic
524// block.
525bool X86FastPreTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
526 this->MBB = &MBB;
527 bool Change = false;
528 MachineInstr *LastShapeMI = nullptr;
529 MachineInstr *LastTileCfg = nullptr;
530 bool HasUnconfigTile = false;
531
532 auto Config = [&](MachineInstr &Before) {
533 if (CfgSS == -1)
534 CfgSS = MFI->CreateStackObject(Size: ST->getTileConfigSize(),
535 Alignment: ST->getTileConfigAlignment(), isSpillSlot: false);
536 LastTileCfg = addFrameReference(
537 MIB: BuildMI(BB&: MBB, I&: Before, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PLDTILECFGV)), FI: CfgSS);
538 LastTileCfg->setImplicitPhysRegDefsDead();
539 LastShapeMI = nullptr;
540 Change = true;
541 };
542 auto HasTileOperand = [](MachineRegisterInfo *MRI, MachineInstr &MI) {
543 for (const MachineOperand &MO : MI.operands()) {
544 if (!MO.isReg())
545 continue;
546 Register Reg = MO.getReg();
547 if (Reg.isVirtual() && isTileRegister(MRI, Reg))
548 return true;
549 }
550 return false;
551 };
552 for (MachineInstr &MI : reverse(C&: MBB)) {
553 // We have transformed phi node before configuring BB.
554 if (MI.isPHI())
555 break;
556 // Don't collect the shape of used tile, the tile should be defined
557 // before the tile use. Spill and reload would happen if there is only
558 // tile use after ldtilecfg, so the shape can be collected from reload.
559 // Take below code for example. %t would be reloaded before tilestore
560 // call
561 // ....
562 // tilestore %r, %c, %t
563 // -->
564 // call
565 // ldtilecfg
566 // %t = tileload %r, %c
567 // tilestore %r, %c, %t
568 if (HasTileOperand(MRI, MI))
569 HasUnconfigTile = true;
570 // According to AMX ABI, all the tile registers including config register
571 // are volatile. Caller need to save/restore config register.
572 if (MI.isCall() && HasUnconfigTile) {
573 MachineBasicBlock::iterator I;
574 if (LastShapeMI && dominates(MBB, A: MI, B: LastShapeMI))
575 I = ++LastShapeMI->getIterator();
576 else {
577 // Call can overwrite registers like rax, ensure the tile config
578 // instruction is sinked closer to first instruction that uses tile.
579 auto UseIt = MI.getIterator();
580 while (UseIt != MBB.end()) {
581 if (HasTileOperand(MRI, *UseIt))
582 break;
583 ++UseIt;
584 }
585 I = UseIt;
586 }
587 Config(*I);
588 HasUnconfigTile = false;
589 continue;
590 }
591 if (!isTileDef(MRI, MI))
592 continue;
593 //
594 //---------------------------------------------------------------------
595 // Don't handle COPY instruction. If the src and dst of the COPY can be
596 // in the same config in below case, we just check the shape of t0.
597 // def row0
598 // def col0
599 // ldtilecfg
600 // t0 = tielzero(row0, col0)
601 // t1 = copy t0
602 // ...
603 // If the src and dst of the COPY can NOT be in the same config in below
604 // case. Reload would be generated befor the copy instruction.
605 // def row0
606 // def col0
607 // t0 = tielzero(row0, col0)
608 // spill t0
609 // ...
610 // def row1
611 // def col1
612 // ldtilecfg
613 // t1 = tilezero(row1, col1)
614 // reload t0
615 // t1 = copy t0
616 //---------------------------------------------------------------------
617 //
618 // If MI dominate the last shape def instruction, we need insert
619 // ldtilecfg after LastShapeMI now. The config doesn't include
620 // current MI.
621 // def row0
622 // def col0
623 // tilezero(row0, col0) <- MI
624 // def row1
625 // def col1
626 // ldtilecfg <- insert
627 // tilezero(row1, col1)
628 if (LastShapeMI && dominates(MBB, A: MI, B: LastShapeMI))
629 Config(*(++LastShapeMI->getIterator()));
630 MachineOperand *RowMO = &MI.getOperand(i: 1);
631 MachineOperand *ColMO = &MI.getOperand(i: 2);
632 MachineInstr *RowMI = MRI->getVRegDef(Reg: RowMO->getReg());
633 MachineInstr *ColMI = MRI->getVRegDef(Reg: ColMO->getReg());
634 // If the shape is defined in current MBB, check the domination.
635 // FIXME how about loop?
636 if (RowMI->getParent() == &MBB) {
637 if (!LastShapeMI)
638 LastShapeMI = RowMI;
639 else if (dominates(MBB, A: LastShapeMI, B: RowMI))
640 LastShapeMI = RowMI;
641 }
642 if (ColMI->getParent() == &MBB) {
643 if (!LastShapeMI)
644 LastShapeMI = ColMI;
645 else if (dominates(MBB, A: LastShapeMI, B: ColMI))
646 LastShapeMI = ColMI;
647 }
648
649 // If there is user live out of the tilecfg, spill it and reload in
650 // before the user.
651 Register TileReg = MI.getOperand(i: 0).getReg();
652 if (mayLiveOut(VirtReg: TileReg, CfgMI: LastTileCfg))
653 spill(Before: ++MI.getIterator(), VirtReg: TileReg, Kill: false);
654 // reload() rewrites or erases UseMI.
655 for (MachineInstr &UseMI :
656 make_early_inc_range(Range: MRI->use_nodbg_instructions(Reg: TileReg))) {
657 if (UseMI.getParent() == &MBB) {
658 // check user should not across ldtilecfg
659 if (!LastTileCfg || !dominates(MBB, A: LastTileCfg, B: UseMI))
660 continue;
661 // reload befor UseMI
662 reload(UseMI: UseMI.getIterator(), OrigReg: TileReg, RowMO, ColMO);
663 } else {
664 // Don't reload for phi instruction, we handle phi reload separately.
665 // TODO: merge the reload for the same user MBB.
666 if (!UseMI.isPHI())
667 reload(UseMI: UseMI.getIterator(), OrigReg: TileReg, RowMO, ColMO);
668 }
669 }
670 }
671
672 // Configure tile registers at the head of the MBB
673 if (HasUnconfigTile) {
674 MachineInstr *Before;
675 if (LastShapeMI == nullptr || LastShapeMI->isPHI())
676 Before = &*MBB.getFirstNonPHI();
677 else
678 Before = &*(++LastShapeMI->getIterator());
679
680 Config(*Before);
681 }
682
683 return Change;
684}
685
686bool X86FastPreTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
687 X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
688 // Early exit in the common case of non-AMX code.
689 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
690 return false;
691
692 MF = &MFunc;
693 MRI = &MFunc.getRegInfo();
694 ST = &MFunc.getSubtarget<X86Subtarget>();
695 TII = ST->getInstrInfo();
696 MFI = &MFunc.getFrameInfo();
697 TRI = ST->getRegisterInfo();
698 CfgSS = -1;
699
700 unsigned NumVirtRegs = MRI->getNumVirtRegs();
701
702 StackSlotForVirtReg.resize(S: NumVirtRegs);
703 MayLiveAcrossBlocks.clear();
704 // We will create register during config. *3 is to make sure
705 // the virtual register number doesn't exceed the size of
706 // the bit vector.
707 MayLiveAcrossBlocks.resize(N: NumVirtRegs * 3);
708 bool Change = false;
709 assert(MRI->isSSA());
710
711 // Canonicalize the phi node first.
712 for (MachineBasicBlock &MBB : MFunc)
713 canonicalizePHIs(MBB);
714
715 // Loop over all of the basic blocks in reverse post order and insert
716 // ldtilecfg for tile registers. The reserse post order is to facilitate
717 // PHI node convert.
718 ReversePostOrderTraversal<MachineFunction *> RPOT(MF);
719 for (MachineBasicBlock *MBB : RPOT) {
720 convertPHIs(MBB&: *MBB);
721 Change |= configBasicBlock(MBB&: *MBB);
722 }
723
724 if (Change)
725 InitializeTileConfigStackSpace();
726
727 StackSlotForVirtReg.clear();
728 return Change;
729}
730
731FunctionPass *llvm::createX86FastPreTileConfigLegacyPass() {
732 return new X86FastPreTileConfigLegacy();
733}
734
735bool X86FastPreTileConfigLegacy::runOnMachineFunction(MachineFunction &MF) {
736 X86FastPreTileConfigImpl Impl;
737 return Impl.runOnMachineFunction(MFunc&: MF);
738}
739
740PreservedAnalyses
741X86FastPreTileConfigPass::run(MachineFunction &MF,
742 MachineFunctionAnalysisManager &MFAM) {
743 X86FastPreTileConfigImpl Impl;
744 bool Changed = Impl.runOnMachineFunction(MFunc&: MF);
745 return Changed ? getMachineFunctionPassPreservedAnalyses()
746 : PreservedAnalyses::all();
747}
748