1//===-- X86FastPreTileConfig.cpp - Fast Tile Register Configure------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to preconfig the shape of physical tile registers
10/// It inserts ldtilecfg ahead of each group of tile registers. The algorithm
11/// walk each instruction of basic block in reverse order. All the tile
12/// registers that live out the basic block would be spilled and reloaded
13/// before its user. It also check the depenedency of the shape to ensure
14/// the shape is defined before ldtilecfg.
15//
16//===----------------------------------------------------------------------===//
17
18#include "X86.h"
19#include "X86InstrBuilder.h"
20#include "X86MachineFunctionInfo.h"
21#include "X86RegisterInfo.h"
22#include "X86Subtarget.h"
23#include "llvm/ADT/PostOrderIterator.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachinePassManager.h"
30#include "llvm/CodeGen/MachineRegisterInfo.h"
31#include "llvm/CodeGen/Passes.h"
32#include "llvm/CodeGen/TargetInstrInfo.h"
33#include "llvm/CodeGen/TargetRegisterInfo.h"
34#include "llvm/IR/Analysis.h"
35#include "llvm/Support/Debug.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "x86-fast-pre-tile-config"
40
41STATISTIC(NumStores, "Number of stores added");
42STATISTIC(NumLoads, "Number of loads added");
43
44namespace {
45
46class X86FastPreTileConfigImpl {
47public:
48 X86FastPreTileConfigImpl() : StackSlotForVirtReg(-1) {}
49 bool runOnMachineFunction(MachineFunction &MF);
50
51private:
52 MachineFunction *MF = nullptr;
53 const X86Subtarget *ST = nullptr;
54 const TargetInstrInfo *TII = nullptr;
55 MachineRegisterInfo *MRI = nullptr;
56 X86MachineFunctionInfo *X86FI = nullptr;
57 MachineFrameInfo *MFI = nullptr;
58 const TargetRegisterInfo *TRI = nullptr;
59 MachineBasicBlock *MBB = nullptr;
60 int CfgSS = -1;
61 struct PHIInfo {
62 Register Row;
63 Register Col;
64 Register StackAddr;
65 };
66 DenseMap<MachineInstr *, struct PHIInfo> VisitedPHIs;
67
68 /// Maps virtual regs to the frame index where these values are spilled.
69 IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;
70
71 /// Has a bit set for tile virtual register for which it was determined
72 /// that it is alive across blocks.
73 BitVector MayLiveAcrossBlocks;
74
75 int getStackSpaceFor(Register VirtReg);
76 void InitializeTileConfigStackSpace();
77 bool mayLiveOut(Register VirtReg, MachineInstr *CfgMI);
78 void spill(MachineBasicBlock::iterator Before, Register VirtReg, bool Kill);
79 void reload(MachineBasicBlock::iterator UseMI, Register VirtReg,
80 MachineOperand *RowMO, MachineOperand *ColMO);
81 void canonicalizePHIs(MachineBasicBlock &MBB);
82 void convertPHI(MachineBasicBlock *MBB, MachineInstr &PHI);
83 void convertPHIs(MachineBasicBlock &MBB);
84 bool configBasicBlock(MachineBasicBlock &MBB);
85};
86
87class X86FastPreTileConfigLegacy : public MachineFunctionPass {
88public:
89 X86FastPreTileConfigLegacy() : MachineFunctionPass(ID) {}
90
91 /// Return the pass name.
92 StringRef getPassName() const override {
93 return "Fast Tile Register Preconfigure";
94 }
95
96 /// Perform tile register configure.
97 bool runOnMachineFunction(MachineFunction &MFunc) override;
98
99 static char ID;
100};
101
102} // end anonymous namespace
103
104char X86FastPreTileConfigLegacy::ID = 0;
105
106INITIALIZE_PASS_BEGIN(X86FastPreTileConfigLegacy, DEBUG_TYPE,
107 "Fast Tile Register Preconfigure", false, false)
108INITIALIZE_PASS_END(X86FastPreTileConfigLegacy, DEBUG_TYPE,
109 "Fast Tile Register Preconfigure", false, false)
110
111static bool dominates(MachineBasicBlock &MBB,
112 MachineBasicBlock::const_iterator A,
113 MachineBasicBlock::const_iterator B) {
114 auto MBBEnd = MBB.end();
115 if (B == MBBEnd)
116 return true;
117
118 MachineBasicBlock::const_iterator I = MBB.begin();
119 for (; &*I != A && &*I != B; ++I)
120 ;
121
122 return &*I == A;
123}
124
125/// This allocates space for the specified virtual register to be held on the
126/// stack.
127int X86FastPreTileConfigImpl::getStackSpaceFor(Register VirtReg) {
128 // Find the location Reg would belong...
129 int SS = StackSlotForVirtReg[VirtReg];
130 // Already has space allocated?
131 if (SS != -1)
132 return SS;
133
134 // Allocate a new stack object for this spill location...
135 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
136 unsigned Size = TRI->getSpillSize(RC);
137 Align Alignment = TRI->getSpillAlign(RC);
138 int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);
139
140 // Assign the slot.
141 StackSlotForVirtReg[VirtReg] = FrameIdx;
142 return FrameIdx;
143}
144
145/// Returns false if \p VirtReg is known to not live out of the current config.
146/// If \p VirtReg live out of the current MBB, it must live out of the current
147/// config
148bool X86FastPreTileConfigImpl::mayLiveOut(Register VirtReg,
149 MachineInstr *CfgMI) {
150 if (MayLiveAcrossBlocks.test(Idx: VirtReg.virtRegIndex()))
151 return true;
152
153 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: VirtReg)) {
154 if (UseInst.getParent() != MBB) {
155 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
156 return true;
157 }
158
159 // The use and def are in the same MBB. If the tile register is
160 // reconfigured, it is crobbered and we need to spill and reload
161 // tile register.
162 if (CfgMI) {
163 if (dominates(MBB&: *MBB, A: *CfgMI, B: UseInst)) {
164 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());
165 return true;
166 }
167 }
168 }
169
170 return false;
171}
172
173void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
174 MachineBasicBlock &MBB = MF->front();
175 MachineInstr *MI = &*MBB.getFirstNonPHI();
176 DebugLoc DL;
177 if (ST->hasAVX512()) {
178 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128XRegClass);
179 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::AVX512_128_SET0), DestReg: Xmm);
180 Register Zmm = MRI->createVirtualRegister(RegClass: &X86::VR512RegClass);
181 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::SUBREG_TO_REG), DestReg: Zmm)
182 .addReg(RegNo: Xmm)
183 .addImm(Val: X86::sub_xmm);
184 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSZmr)), FI: CfgSS)
185 .addReg(RegNo: Zmm);
186 } else if (ST->hasAVX2()) {
187 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128RegClass);
188 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::V_SET0), DestReg: Xmm);
189 Register Ymm = MRI->createVirtualRegister(RegClass: &X86::VR256RegClass);
190 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::SUBREG_TO_REG), DestReg: Ymm)
191 .addReg(RegNo: Xmm)
192 .addImm(Val: X86::sub_xmm);
193 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSYmr)), FI: CfgSS)
194 .addReg(RegNo: Ymm);
195 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::VMOVUPSYmr)), FI: CfgSS,
196 Offset: 32)
197 .addReg(RegNo: Ymm);
198 } else {
199 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled");
200 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;
201 Register Xmm = MRI->createVirtualRegister(RegClass: &X86::VR128RegClass);
202 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::V_SET0), DestReg: Xmm);
203 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS)
204 .addReg(RegNo: Xmm);
205 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 16)
206 .addReg(RegNo: Xmm);
207 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 32)
208 .addReg(RegNo: Xmm);
209 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: StoreOpc)), FI: CfgSS, Offset: 48)
210 .addReg(RegNo: Xmm);
211 }
212 // Fill in the palette first.
213 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV8mi)), FI: CfgSS)
214 .addImm(Val: 1);
215}
216
217/// Insert spill instruction for \p AssignedReg before \p Before.
218/// TODO: Update DBG_VALUEs with \p VirtReg operands with the stack slot.
219void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
220 Register VirtReg, bool Kill) {
221 LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " \n");
222 int FI = getStackSpaceFor(VirtReg);
223 LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');
224
225 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: VirtReg);
226 // Don't need shape information for tile store, becasue it is adjacent to
227 // the tile def instruction.
228 TII->storeRegToStackSlot(MBB&: *MBB, MI: Before, SrcReg: VirtReg, isKill: Kill, FrameIndex: FI, RC: &RC, VReg: Register());
229 ++NumStores;
230
231 // TODO: update DBG_VALUEs
232}
233
234/// Insert reload instruction for \p PhysReg before \p Before.
235void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
236 Register OrigReg, MachineOperand *RowMO,
237 MachineOperand *ColMO) {
238 int FI = getStackSpaceFor(VirtReg: OrigReg);
239 const TargetRegisterClass &RC = *MRI->getRegClass(Reg: OrigReg);
240 Register TileReg;
241 // Fold copy to tileload
242 // BB1:
243 // spill src to s
244 //
245 // BB2:
246 // t = copy src
247 // -->
248 // t = tileload (s)
249 if (UseMI->isCopy())
250 TileReg = UseMI->getOperand(i: 0).getReg();
251 else
252 TileReg = MRI->createVirtualRegister(RegClass: &RC);
253 // Can't use TII->loadRegFromStackSlot(), because we need the shape
254 // information for reload.
255 // tileloadd (%sp, %idx), %tmm
256 unsigned Opc = X86::PTILELOADDV;
257 Register StrideReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
258 // FIXME: MBB is not the parent of UseMI.
259 MachineInstr *NewMI = BuildMI(BB&: *UseMI->getParent(), I: UseMI, MIMD: DebugLoc(),
260 MCID: TII->get(Opcode: X86::MOV64ri), DestReg: StrideReg)
261 .addImm(Val: 64);
262 NewMI = addFrameReference(
263 MIB: BuildMI(BB&: *UseMI->getParent(), I: UseMI, MIMD: DebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: TileReg)
264 .addReg(RegNo: RowMO->getReg())
265 .addReg(RegNo: ColMO->getReg()),
266 FI);
267 MachineOperand &MO = NewMI->getOperand(i: 5);
268 MO.setReg(StrideReg);
269 MO.setIsKill(true);
270 RowMO->setIsKill(false);
271 ColMO->setIsKill(false);
272 // Erase copy instruction after it is folded.
273 if (UseMI->isCopy()) {
274 UseMI->eraseFromParent();
275 } else {
276 // Replace the register in the user MI.
277 for (auto &MO : UseMI->operands()) {
278 if (MO.isReg() && MO.getReg() == OrigReg)
279 MO.setReg(TileReg);
280 }
281 }
282
283 ++NumLoads;
284 LLVM_DEBUG(dbgs() << "Reloading " << printReg(OrigReg, TRI) << " into "
285 << printReg(TileReg, TRI) << '\n');
286}
287
288static bool isTileRegister(MachineRegisterInfo *MRI, Register Reg) {
289 if (Reg.isVirtual() &&
290 (MRI->getRegClass(Reg)->getID() == X86::TILERegClassID)) {
291 return true;
292 }
293
294 if (Reg >= X86::TMM0 && Reg <= X86::TMM7)
295 return true;
296
297 return false;
298}
299
300static bool isTileDef(MachineRegisterInfo *MRI, MachineInstr &MI) {
301 // The instruction must have 3 operands: tile def, row, col.
302 if (MI.isDebugInstr() || MI.getNumOperands() < 3 || !MI.isPseudo())
303 return false;
304 MachineOperand &MO = MI.getOperand(i: 0);
305
306 if (!MO.isReg())
307 return false;
308
309 return isTileRegister(MRI, Reg: MO.getReg());
310}
311
312static ShapeT getShape(MachineRegisterInfo *MRI, Register TileReg) {
313 MachineInstr *MI = MRI->getVRegDef(Reg: TileReg);
314 if (isTileDef(MRI, MI&: *MI)) {
315 MachineOperand *RowMO = &MI->getOperand(i: 1);
316 MachineOperand *ColMO = &MI->getOperand(i: 2);
317 return ShapeT(RowMO, ColMO, MRI);
318 } else if (MI->isCopy()) {
319 TileReg = MI->getOperand(i: 1).getReg();
320 return getShape(MRI, TileReg);
321 }
322
323 // The def should not be PHI node, because we walk the MBB in reverse post
324 // order.
325 assert(MI->isPHI() && "Unexpected PHI when get shape.");
326 llvm_unreachable("Unexpected MI when get shape.");
327}
328
329// BB0:
330// spill t0 to s0
331// BB1:
332// spill t1 to s1
333//
334// BB2:
335// t = phi [t0, bb0] [t1, bb1]
336// -->
337// row = phi [r0, bb0] [r1, bb1]
338// col = phi [c0, bb0] [c1, bb1]
339// s = phi [s0, bb0] [s1, bb1]
340// t = tileload row, col, s
341// The new instruction is inserted at the end of the phi node. The order
342// of the original phi node is not ensured.
343void X86FastPreTileConfigImpl::convertPHI(MachineBasicBlock *MBB,
344 MachineInstr &PHI) {
345 // 1. Create instruction to get stack slot address of each incoming block.
346 // 2. Create PHI node for the stack address.
347 // 3. Create PHI node for shape. If one of the incoming shape is immediate
348 // use the immediate and delete the PHI node.
349 // 4. Create tileload instruction from the stack address.
350 Register StackAddrReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
351 MachineInstrBuilder AddrPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
352 MCID: TII->get(Opcode: X86::PHI), DestReg: StackAddrReg);
353 Register RowReg = MRI->createVirtualRegister(RegClass: &X86::GR16RegClass);
354 MachineInstrBuilder RowPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
355 MCID: TII->get(Opcode: X86::PHI), DestReg: RowReg);
356 Register ColReg = MRI->createVirtualRegister(RegClass: &X86::GR16RegClass);
357 MachineInstrBuilder ColPHI = BuildMI(BB&: *MBB, I: ++PHI.getIterator(), MIMD: DebugLoc(),
358 MCID: TII->get(Opcode: X86::PHI), DestReg: ColReg);
359 // Record the mapping of phi node and its row/column information.
360 VisitedPHIs[&PHI] = {.Row: RowReg, .Col: ColReg, .StackAddr: StackAddrReg};
361
362 for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) {
363 // Get the 2 incoming value of tile register and MBB.
364 Register InTileReg = PHI.getOperand(i: I).getReg();
365 // Mark it as liveout, so that it will be spilled when visit
366 // the incoming MBB. Otherwise since phi will be deleted, it
367 // would miss spill when visit incoming MBB.
368 MayLiveAcrossBlocks.set(InTileReg.virtRegIndex());
369 MachineBasicBlock *InMBB = PHI.getOperand(i: I + 1).getMBB();
370
371 MachineInstr *TileDefMI = MRI->getVRegDef(Reg: InTileReg);
372 MachineBasicBlock::iterator InsertPos;
373 if (TileDefMI->isPHI()) {
374 InsertPos = TileDefMI->getParent()->getFirstNonPHI();
375 if (auto It = VisitedPHIs.find(Val: TileDefMI);
376 It != VisitedPHIs.end()) { // circular phi reference
377 // def t1
378 // / \
379 // def t2 t3 = phi(t1, t4) <--
380 // \ / |
381 // t4 = phi(t2, t3)-------------
382 //
383 // For each (row, column and stack address) append phi incoming value.
384 // Create r3 = phi(r1, r4)
385 // Create r4 = phi(r2, r3)
386 Register InRowReg = It->second.Row;
387 Register InColReg = It->second.Col;
388 Register InStackAddrReg = It->second.StackAddr;
389 RowPHI.addReg(RegNo: InRowReg).addMBB(MBB: InMBB);
390 ColPHI.addReg(RegNo: InColReg).addMBB(MBB: InMBB);
391 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
392 continue;
393 } else {
394 // Recursively convert PHI to tileload
395 convertPHI(MBB: TileDefMI->getParent(), PHI&: *TileDefMI);
396 // The PHI node is coverted to tileload instruction. Get the stack
397 // address from tileload operands.
398 MachineInstr *TileLoad = MRI->getVRegDef(Reg: InTileReg);
399 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV);
400 Register InRowReg = TileLoad->getOperand(i: 1).getReg();
401 Register InColReg = TileLoad->getOperand(i: 2).getReg();
402 Register InStackAddrReg = TileLoad->getOperand(i: 3).getReg();
403 RowPHI.addReg(RegNo: InRowReg).addMBB(MBB: InMBB);
404 ColPHI.addReg(RegNo: InColReg).addMBB(MBB: InMBB);
405 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
406 }
407 } else {
408 InsertPos = TileDefMI->getIterator();
409
410 // Fill the incoming operand of row/column phi instruction.
411 ShapeT Shape = getShape(MRI, TileReg: InTileReg);
412 Shape.getRow()->setIsKill(false);
413 Shape.getCol()->setIsKill(false);
414 RowPHI.addReg(RegNo: Shape.getRow()->getReg()).addMBB(MBB: InMBB);
415 ColPHI.addReg(RegNo: Shape.getCol()->getReg()).addMBB(MBB: InMBB);
416
417 // The incoming tile register live out of its def BB, it would be spilled.
418 // Create MI to get the spill stack slot address for the tile register
419 int FI = getStackSpaceFor(VirtReg: InTileReg);
420 Register InStackAddrReg =
421 MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
422 addOffset(MIB: BuildMI(BB&: *TileDefMI->getParent(), I: InsertPos, MIMD: DebugLoc(),
423 MCID: TII->get(Opcode: X86::LEA64r), DestReg: InStackAddrReg)
424 .addFrameIndex(Idx: FI),
425 Offset: 0);
426 AddrPHI.addReg(RegNo: InStackAddrReg).addMBB(MBB: InMBB);
427 }
428 }
429
430 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
431 Register StrideReg = MRI->createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
432 BuildMI(BB&: *MBB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::MOV64ri), DestReg: StrideReg)
433 .addImm(Val: 64);
434 Register TileReg = PHI.getOperand(i: 0).getReg();
435 MachineInstr *NewMI = addDirectMem(
436 MIB: BuildMI(BB&: *MBB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PTILELOADDV), DestReg: TileReg)
437 .addReg(RegNo: RowReg)
438 .addReg(RegNo: ColReg),
439 Reg: StackAddrReg);
440 MachineOperand &MO = NewMI->getOperand(i: 5);
441 MO.setReg(StrideReg);
442 MO.setIsKill(true);
443 PHI.eraseFromParent();
444 VisitedPHIs.erase(Val: &PHI);
445}
446
447static bool isTileRegDef(MachineRegisterInfo *MRI, MachineInstr &MI) {
448 MachineOperand &MO = MI.getOperand(i: 0);
449 if (MO.isReg() && MO.getReg().isVirtual() && isTileRegister(MRI, Reg: MO.getReg()))
450 return true;
451 return false;
452}
453
454void X86FastPreTileConfigImpl::canonicalizePHIs(MachineBasicBlock &MBB) {
455 SmallVector<MachineInstr *, 8> PHIs;
456
457 for (MachineInstr &MI : MBB) {
458 if (!MI.isPHI())
459 break;
460 if (!isTileRegDef(MRI, MI))
461 continue;
462 PHIs.push_back(Elt: &MI);
463 }
464 // Canonicalize the phi node first. One tile phi may depeneds previous
465 // phi node. For below case, we need convert %t4.
466 //
467 // BB0:
468 // %t3 = phi (t1 BB1, t2 BB0)
469 // %t4 = phi (t5 BB1, t3 BB0)
470 // -->
471 // %t3 = phi (t1 BB1, t2 BB0)
472 // %t4 = phi (t5 BB1, t2 BB0)
473 //
474 while (!PHIs.empty()) {
475 MachineInstr *PHI = PHIs.pop_back_val();
476
477 // Find the operand that is incoming from the same MBB and the def
478 // is also phi node.
479 MachineOperand *InMO = nullptr;
480 MachineInstr *DefMI = nullptr;
481 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) {
482 Register InTileReg = PHI->getOperand(i: I).getReg();
483 MachineBasicBlock *InMBB = PHI->getOperand(i: I + 1).getMBB();
484 DefMI = MRI->getVRegDef(Reg: InTileReg);
485 if (InMBB != &MBB || !DefMI->isPHI())
486 continue;
487
488 InMO = &PHI->getOperand(i: I);
489 break;
490 }
491 // If can't find such operand, do nothing.
492 if (!InMO)
493 continue;
494
495 // Current phi node depends on previous phi node. Break the
496 // dependency.
497 Register DefTileReg;
498 for (unsigned I = 1, E = DefMI->getNumOperands(); I != E; I += 2) {
499 MachineBasicBlock *InMBB = PHI->getOperand(i: I + 1).getMBB();
500 if (InMBB != &MBB)
501 continue;
502 DefTileReg = DefMI->getOperand(i: I).getReg();
503 InMO->setReg(DefTileReg);
504 break;
505 }
506 }
507}
508
509void X86FastPreTileConfigImpl::convertPHIs(MachineBasicBlock &MBB) {
510 SmallVector<MachineInstr *, 8> PHIs;
511 for (MachineInstr &MI : MBB) {
512 if (!MI.isPHI())
513 break;
514 if (!isTileRegDef(MRI, MI))
515 continue;
516 PHIs.push_back(Elt: &MI);
517 }
518 while (!PHIs.empty()) {
519 MachineInstr *MI = PHIs.pop_back_val();
520 VisitedPHIs.clear();
521 convertPHI(MBB: &MBB, PHI&: *MI);
522 }
523}
524
525// PreTileConfig should configure the tile registers based on basic
526// block.
527bool X86FastPreTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
528 this->MBB = &MBB;
529 bool Change = false;
530 MachineInstr *LastShapeMI = nullptr;
531 MachineInstr *LastTileCfg = nullptr;
532 bool HasUnconfigTile = false;
533
534 auto Config = [&](MachineInstr &Before) {
535 if (CfgSS == -1)
536 CfgSS = MFI->CreateStackObject(Size: ST->getTileConfigSize(),
537 Alignment: ST->getTileConfigAlignment(), isSpillSlot: false);
538 LastTileCfg = addFrameReference(
539 MIB: BuildMI(BB&: MBB, I&: Before, MIMD: DebugLoc(), MCID: TII->get(Opcode: X86::PLDTILECFGV)), FI: CfgSS);
540 LastShapeMI = nullptr;
541 Change = true;
542 };
543 auto HasTileOperand = [](MachineRegisterInfo *MRI, MachineInstr &MI) {
544 for (const MachineOperand &MO : MI.operands()) {
545 if (!MO.isReg())
546 continue;
547 Register Reg = MO.getReg();
548 if (Reg.isVirtual() && isTileRegister(MRI, Reg))
549 return true;
550 }
551 return false;
552 };
553 for (MachineInstr &MI : reverse(C&: MBB)) {
554 // We have transformed phi node before configuring BB.
555 if (MI.isPHI())
556 break;
557 // Don't collect the shape of used tile, the tile should be defined
558 // before the tile use. Spill and reload would happen if there is only
559 // tile use after ldtilecfg, so the shape can be collected from reload.
560 // Take below code for example. %t would be reloaded before tilestore
561 // call
562 // ....
563 // tilestore %r, %c, %t
564 // -->
565 // call
566 // ldtilecfg
567 // %t = tileload %r, %c
568 // tilestore %r, %c, %t
569 if (HasTileOperand(MRI, MI))
570 HasUnconfigTile = true;
571 // According to AMX ABI, all the tile registers including config register
572 // are volatile. Caller need to save/restore config register.
573 if (MI.isCall() && HasUnconfigTile) {
574 MachineBasicBlock::iterator I;
575 if (LastShapeMI && dominates(MBB, A: MI, B: LastShapeMI))
576 I = ++LastShapeMI->getIterator();
577 else {
578 // Call can overwrite registers like rax, ensure the tile config
579 // instruction is sinked closer to first instruction that uses tile.
580 auto UseIt = MI.getIterator();
581 while (UseIt != MBB.end()) {
582 if (HasTileOperand(MRI, *UseIt))
583 break;
584 ++UseIt;
585 }
586 I = UseIt;
587 }
588 Config(*I);
589 HasUnconfigTile = false;
590 continue;
591 }
592 if (!isTileDef(MRI, MI))
593 continue;
594 //
595 //---------------------------------------------------------------------
596 // Don't handle COPY instruction. If the src and dst of the COPY can be
597 // in the same config in below case, we just check the shape of t0.
598 // def row0
599 // def col0
600 // ldtilecfg
601 // t0 = tielzero(row0, col0)
602 // t1 = copy t0
603 // ...
604 // If the src and dst of the COPY can NOT be in the same config in below
605 // case. Reload would be generated befor the copy instruction.
606 // def row0
607 // def col0
608 // t0 = tielzero(row0, col0)
609 // spill t0
610 // ...
611 // def row1
612 // def col1
613 // ldtilecfg
614 // t1 = tilezero(row1, col1)
615 // reload t0
616 // t1 = copy t0
617 //---------------------------------------------------------------------
618 //
619 // If MI dominate the last shape def instruction, we need insert
620 // ldtilecfg after LastShapeMI now. The config doesn't include
621 // current MI.
622 // def row0
623 // def col0
624 // tilezero(row0, col0) <- MI
625 // def row1
626 // def col1
627 // ldtilecfg <- insert
628 // tilezero(row1, col1)
629 if (LastShapeMI && dominates(MBB, A: MI, B: LastShapeMI))
630 Config(*(++LastShapeMI->getIterator()));
631 MachineOperand *RowMO = &MI.getOperand(i: 1);
632 MachineOperand *ColMO = &MI.getOperand(i: 2);
633 MachineInstr *RowMI = MRI->getVRegDef(Reg: RowMO->getReg());
634 MachineInstr *ColMI = MRI->getVRegDef(Reg: ColMO->getReg());
635 // If the shape is defined in current MBB, check the domination.
636 // FIXME how about loop?
637 if (RowMI->getParent() == &MBB) {
638 if (!LastShapeMI)
639 LastShapeMI = RowMI;
640 else if (dominates(MBB, A: LastShapeMI, B: RowMI))
641 LastShapeMI = RowMI;
642 }
643 if (ColMI->getParent() == &MBB) {
644 if (!LastShapeMI)
645 LastShapeMI = ColMI;
646 else if (dominates(MBB, A: LastShapeMI, B: ColMI))
647 LastShapeMI = ColMI;
648 }
649
650 // If there is user live out of the tilecfg, spill it and reload in
651 // before the user.
652 Register TileReg = MI.getOperand(i: 0).getReg();
653 if (mayLiveOut(VirtReg: TileReg, CfgMI: LastTileCfg))
654 spill(Before: ++MI.getIterator(), VirtReg: TileReg, Kill: false);
655 for (MachineInstr &UseMI : MRI->use_instructions(Reg: TileReg)) {
656 if (UseMI.getParent() == &MBB) {
657 // check user should not across ldtilecfg
658 if (!LastTileCfg || !dominates(MBB, A: LastTileCfg, B: UseMI))
659 continue;
660 // reload befor UseMI
661 reload(UseMI: UseMI.getIterator(), OrigReg: TileReg, RowMO, ColMO);
662 } else {
663 // Don't reload for phi instruction, we handle phi reload separately.
664 // TODO: merge the reload for the same user MBB.
665 if (!UseMI.isPHI())
666 reload(UseMI: UseMI.getIterator(), OrigReg: TileReg, RowMO, ColMO);
667 }
668 }
669 }
670
671 // Configure tile registers at the head of the MBB
672 if (HasUnconfigTile) {
673 MachineInstr *Before;
674 if (LastShapeMI == nullptr || LastShapeMI->isPHI())
675 Before = &*MBB.getFirstNonPHI();
676 else
677 Before = &*(++LastShapeMI->getIterator());
678
679 Config(*Before);
680 }
681
682 return Change;
683}
684
685bool X86FastPreTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
686 X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
687 // Early exit in the common case of non-AMX code.
688 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
689 return false;
690
691 MF = &MFunc;
692 MRI = &MFunc.getRegInfo();
693 ST = &MFunc.getSubtarget<X86Subtarget>();
694 TII = ST->getInstrInfo();
695 MFI = &MFunc.getFrameInfo();
696 TRI = ST->getRegisterInfo();
697 CfgSS = -1;
698
699 unsigned NumVirtRegs = MRI->getNumVirtRegs();
700
701 StackSlotForVirtReg.resize(S: NumVirtRegs);
702 MayLiveAcrossBlocks.clear();
703 // We will create register during config. *3 is to make sure
704 // the virtual register number doesn't exceed the size of
705 // the bit vector.
706 MayLiveAcrossBlocks.resize(N: NumVirtRegs * 3);
707 bool Change = false;
708 assert(MRI->isSSA());
709
710 // Canonicalize the phi node first.
711 for (MachineBasicBlock &MBB : MFunc)
712 canonicalizePHIs(MBB);
713
714 // Loop over all of the basic blocks in reverse post order and insert
715 // ldtilecfg for tile registers. The reserse post order is to facilitate
716 // PHI node convert.
717 ReversePostOrderTraversal<MachineFunction *> RPOT(MF);
718 for (MachineBasicBlock *MBB : RPOT) {
719 convertPHIs(MBB&: *MBB);
720 Change |= configBasicBlock(MBB&: *MBB);
721 }
722
723 if (Change)
724 InitializeTileConfigStackSpace();
725
726 StackSlotForVirtReg.clear();
727 return Change;
728}
729
730FunctionPass *llvm::createX86FastPreTileConfigLegacyPass() {
731 return new X86FastPreTileConfigLegacy();
732}
733
734bool X86FastPreTileConfigLegacy::runOnMachineFunction(MachineFunction &MF) {
735 X86FastPreTileConfigImpl Impl;
736 return Impl.runOnMachineFunction(MFunc&: MF);
737}
738
739PreservedAnalyses
740X86FastPreTileConfigPass::run(MachineFunction &MF,
741 MachineFunctionAnalysisManager &MFAM) {
742 X86FastPreTileConfigImpl Impl;
743 bool Changed = Impl.runOnMachineFunction(MFunc&: MF);
744 return Changed ? getMachineFunctionPassPreservedAnalyses()
745 : PreservedAnalyses::all();
746}
747