1//===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass identifies floating point stores that should not be combined into
10// store pairs. Later we may do the same for floating point loads.
11// ===---------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64Subtarget.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineFunctionPass.h"
17#include "llvm/CodeGen/MachineInstr.h"
18#include "llvm/CodeGen/MachineTraceMetrics.h"
19#include "llvm/CodeGen/RegisterClassInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/CodeGen/TargetSchedule.h"
22#include "llvm/Support/Debug.h"
23#include "llvm/Support/raw_ostream.h"
24
25using namespace llvm;
26
27#define DEBUG_TYPE "aarch64-stp-suppress"
28
29#define STPSUPPRESS_PASS_NAME "AArch64 Store Pair Suppression"
30
31namespace {
32class AArch64StorePairSuppress : public MachineFunctionPass {
33 const AArch64InstrInfo *TII;
34 const TargetRegisterInfo *TRI;
35 const MachineRegisterInfo *MRI;
36 TargetSchedModel SchedModel;
37 MachineTraceMetrics *Traces;
38 MachineTraceMetrics::Ensemble *MinInstr;
39
40public:
41 static char ID;
42 AArch64StorePairSuppress() : MachineFunctionPass(ID) {}
43
44 StringRef getPassName() const override { return STPSUPPRESS_PASS_NAME; }
45
46 bool runOnMachineFunction(MachineFunction &F) override;
47
48private:
49 bool shouldAddSTPToBlock(const MachineBasicBlock *BB);
50
51 bool isNarrowFPStore(const MachineInstr &MI);
52
53 void getAnalysisUsage(AnalysisUsage &AU) const override {
54 AU.setPreservesCFG();
55 AU.addRequired<MachineTraceMetricsWrapperPass>();
56 AU.addPreserved<MachineTraceMetricsWrapperPass>();
57 MachineFunctionPass::getAnalysisUsage(AU);
58 }
59};
60char AArch64StorePairSuppress::ID = 0;
61} // anonymous
62
63INITIALIZE_PASS(AArch64StorePairSuppress, "aarch64-stp-suppress",
64 STPSUPPRESS_PASS_NAME, false, false)
65
66FunctionPass *llvm::createAArch64StorePairSuppressPass() {
67 return new AArch64StorePairSuppress();
68}
69
70/// Return true if an STP can be added to this block without increasing the
71/// critical resource height. STP is good to form in Ld/St limited blocks and
72/// bad to form in float-point limited blocks. This is true independent of the
73/// critical path. If the critical path is longer than the resource height, the
74/// extra vector ops can limit physreg renaming. Otherwise, it could simply
75/// oversaturate the vector units.
76bool AArch64StorePairSuppress::shouldAddSTPToBlock(const MachineBasicBlock *BB) {
77 if (!MinInstr)
78 MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
79
80 MachineTraceMetrics::Trace BBTrace = MinInstr->getTrace(MBB: BB);
81 unsigned ResLength = BBTrace.getResourceLength();
82
83 // Get the machine model's scheduling class for STPDi and STRDui.
84 // Bypass TargetSchedule's SchedClass resolution since we only have an opcode.
85 unsigned SCIdx = TII->get(Opcode: AArch64::STPDi).getSchedClass();
86 const MCSchedClassDesc *PairSCDesc =
87 SchedModel.getMCSchedModel()->getSchedClassDesc(SchedClassIdx: SCIdx);
88
89 unsigned SCIdx2 = TII->get(Opcode: AArch64::STRDui).getSchedClass();
90 const MCSchedClassDesc *SingleSCDesc =
91 SchedModel.getMCSchedModel()->getSchedClassDesc(SchedClassIdx: SCIdx2);
92
93 // If a subtarget does not define resources for STPDi, bail here.
94 if (PairSCDesc->isValid() && !PairSCDesc->isVariant() &&
95 SingleSCDesc->isValid() && !SingleSCDesc->isVariant()) {
96 // Compute the new critical resource length after replacing 2 separate
97 // STRDui with one STPDi.
98 unsigned ResLenWithSTP =
99 BBTrace.getResourceLength(Extrablocks: {}, ExtraInstrs: PairSCDesc, RemoveInstrs: {SingleSCDesc, SingleSCDesc});
100 if (ResLenWithSTP > ResLength) {
101 LLVM_DEBUG(dbgs() << " Suppress STP in BB: " << BB->getNumber()
102 << " resources " << ResLength << " -> " << ResLenWithSTP
103 << "\n");
104 return false;
105 }
106 }
107 return true;
108}
109
110/// Return true if this is a floating-point store smaller than the V reg. On
111/// cyclone, these require a vector shuffle before storing a pair.
112/// Ideally we would call getMatchingPairOpcode() and have the machine model
113/// tell us if it's profitable with no cpu knowledge here.
114///
115/// FIXME: We plan to develop a decent Target abstraction for simple loads and
116/// stores. Until then use a nasty switch similar to AArch64LoadStoreOptimizer.
117bool AArch64StorePairSuppress::isNarrowFPStore(const MachineInstr &MI) {
118 switch (MI.getOpcode()) {
119 default:
120 return false;
121 case AArch64::STRSui:
122 case AArch64::STRDui:
123 case AArch64::STURSi:
124 case AArch64::STURDi:
125 return true;
126 }
127}
128
129bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
130 if (skipFunction(F: MF.getFunction()) || MF.getFunction().hasOptSize())
131 return false;
132
133 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
134 if (!ST.enableStorePairSuppress())
135 return false;
136
137 TII = ST.getInstrInfo();
138 TRI = ST.getRegisterInfo();
139 MRI = &MF.getRegInfo();
140 SchedModel.init(TSInfo: &ST);
141 Traces = &getAnalysis<MachineTraceMetricsWrapperPass>().getMTM();
142 MinInstr = nullptr;
143
144 LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << MF.getName() << '\n');
145
146 if (!SchedModel.hasInstrSchedModel()) {
147 LLVM_DEBUG(dbgs() << " Skipping pass: no machine model present.\n");
148 return false;
149 }
150
151 // Check for a sequence of stores to the same base address. We don't need to
152 // precisely determine whether a store pair can be formed. But we do want to
153 // filter out most situations where we can't form store pairs to avoid
154 // computing trace metrics in those cases.
155 for (auto &MBB : MF) {
156 bool SuppressSTP = false;
157 unsigned PrevBaseReg = 0;
158 for (auto &MI : MBB) {
159 if (!isNarrowFPStore(MI))
160 continue;
161 const MachineOperand *BaseOp;
162 int64_t Offset;
163 bool OffsetIsScalable;
164 if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
165 TRI) &&
166 BaseOp->isReg()) {
167 Register BaseReg = BaseOp->getReg();
168 if (PrevBaseReg == BaseReg) {
169 // If this block can take STPs, skip ahead to the next block.
170 if (!SuppressSTP && shouldAddSTPToBlock(BB: MI.getParent()))
171 break;
172 // Otherwise, continue unpairing the stores in this block.
173 LLVM_DEBUG(dbgs() << "Unpairing store " << MI << "\n");
174 SuppressSTP = true;
175 TII->suppressLdStPair(MI);
176 }
177 PrevBaseReg = BaseReg;
178 } else
179 PrevBaseReg = 0;
180 }
181 }
182 // This pass just sets some internal MachineMemOperand flags. It can't really
183 // invalidate anything.
184 return false;
185}
186