1//===-- X86LowerTileCopy.cpp - Expand Tile Copy Instructions---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the pass which lower AMX tile copy instructions. Since
10// there is no tile copy instruction, we need store tile register to stack
11// and load from stack to another tile register. We need extra GR to hold
12// the stride, and we need stack slot to hold the tile data register.
13// We would run this pass after copy propagation, so that we don't miss copy
14// optimization. And we would run this pass before prolog/epilog insertion,
15// so that we can allocate stack slot.
16//
17//===----------------------------------------------------------------------===//
18
19#include "X86.h"
20#include "X86InstrBuilder.h"
21#include "X86InstrInfo.h"
22#include "X86MachineFunctionInfo.h"
23#include "X86Subtarget.h"
24#include "llvm/CodeGen/LiveRegUnits.h"
25#include "llvm/CodeGen/MachineBasicBlock.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineOperand.h"
32#include "llvm/CodeGen/Passes.h"
33#include "llvm/IR/DebugLoc.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "x86-lower-tile-copy"
38
39namespace {
40
41class X86LowerTileCopyLegacy : public MachineFunctionPass {
42public:
43 static char ID;
44
45 X86LowerTileCopyLegacy() : MachineFunctionPass(ID) {}
46
47 void getAnalysisUsage(AnalysisUsage &AU) const override;
48
49 bool runOnMachineFunction(MachineFunction &MF) override;
50
51 StringRef getPassName() const override { return "X86 Lower Tile Copy"; }
52};
53
54} // namespace
55
56char X86LowerTileCopyLegacy::ID = 0;
57
58INITIALIZE_PASS_BEGIN(X86LowerTileCopyLegacy, DEBUG_TYPE, "Tile Copy Lowering",
59 false, false)
60INITIALIZE_PASS_END(X86LowerTileCopyLegacy, DEBUG_TYPE, "Tile Copy Lowering",
61 false, false)
62
63void X86LowerTileCopyLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
64 AU.setPreservesAll();
65 MachineFunctionPass::getAnalysisUsage(AU);
66}
67
68FunctionPass *llvm::createX86LowerTileCopyLegacyPass() {
69 return new X86LowerTileCopyLegacy();
70}
71
72static bool lowerTileCopy(MachineFunction &MF) {
73 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
74 if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
75 return false;
76
77 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
78 assert(ST.hasAMXTILE() && "Only supported on AMXTILE targets");
79
80 const X86InstrInfo *TII = ST.getInstrInfo();
81 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
82 BitVector GR64Regs =
83 TRI->getAllocatableSet(MF, RC: TRI->getRegClass(i: X86::GR64RegClassID));
84 bool Changed = false;
85
86 for (MachineBasicBlock &MBB : MF) {
87 LiveRegUnits UsedRegs(*TRI);
88 UsedRegs.addLiveOuts(MBB);
89 for (MachineInstr &MI : llvm::make_early_inc_range(Range: reverse(C&: MBB))) {
90 if (MI.isDebugInstr())
91 continue;
92 UsedRegs.stepBackward(MI);
93 if (!MI.isCopy())
94 continue;
95 MachineOperand &DstMO = MI.getOperand(i: 0);
96 MachineOperand &SrcMO = MI.getOperand(i: 1);
97 Register SrcReg = SrcMO.getReg();
98 Register DstReg = DstMO.getReg();
99 if (!X86::TILERegClass.contains(Reg1: DstReg, Reg2: SrcReg))
100 continue;
101
102 // Allocate stack slot for tile register
103 unsigned Size = TRI->getSpillSize(RC: X86::TILERegClass);
104 Align Alignment = TRI->getSpillAlign(RC: X86::TILERegClass);
105 int TileSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
106
107 int StrideSS = 0;
108
109 // Pick a killed register to avoid a save/reload.
110 Register GR64Cand = X86::NoRegister;
111 for (auto RegT : GR64Regs.set_bits()) {
112 if (UsedRegs.available(Reg: RegT)) {
113 GR64Cand = RegT;
114 break;
115 }
116 }
117
118 const DebugLoc &DL = MI.getDebugLoc();
119 if (GR64Cand) {
120 // mov 64 %reg
121 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64ri), DestReg: GR64Cand).addImm(Val: 64);
122 } else {
123 // No available register? Save RAX and reload it after use.
124
125 // Allocate stack slot for stride register
126 Size = TRI->getSpillSize(RC: X86::GR64RegClass);
127 Alignment = TRI->getSpillAlign(RC: X86::GR64RegClass);
128 StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
129
130 // mov %reg (%sp)
131 addFrameReference(MIB: BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64mr)),
132 FI: StrideSS)
133 .addReg(RegNo: X86::RAX);
134 // mov 64 %reg
135 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64ri), DestReg: X86::RAX).addImm(Val: 64);
136 }
137 // tilestored %tmm, (%sp, %idx)
138#define GET_EGPR_IF_ENABLED(OPC) (ST.hasEGPR() ? OPC##_EVEX : OPC)
139 unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
140 MachineInstr *NewMI =
141 addFrameReference(MIB: BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc)), FI: TileSS)
142 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: SrcMO.isKill()));
143 MachineOperand *MO = &NewMI->getOperand(i: X86::AddrIndexReg);
144 MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
145 // tileloadd (%sp, %idx), %tmm
146 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
147#undef GET_EGPR_IF_ENABLED
148 NewMI = addFrameReference(MIB: BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg),
149 FI: TileSS);
150 MO = &NewMI->getOperand(i: 1 + X86::AddrIndexReg);
151 MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
152 MO->setIsKill(true);
153 if (!GR64Cand) {
154 // restore %rax
155 // mov (%sp) %rax
156 addFrameReference(
157 MIB: BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: X86::MOV64rm), DestReg: X86::RAX), FI: StrideSS);
158 }
159 MI.eraseFromParent();
160 Changed = true;
161 }
162 }
163 return Changed;
164}
165
166bool X86LowerTileCopyLegacy::runOnMachineFunction(MachineFunction &MF) {
167 return lowerTileCopy(MF);
168}
169
170PreservedAnalyses
171X86LowerTileCopyPass::run(MachineFunction &MF,
172 MachineFunctionAnalysisManager &MFAM) {
173 return lowerTileCopy(MF) ? getMachineFunctionPassPreservedAnalyses()
174 .preserveSet<CFGAnalyses>()
175 : PreservedAnalyses::all();
176}
177