1//===-- AArch64CodeLayoutOpt.cpp - Code Layout Optimizations --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass runs after instruction scheduling and employs code layout
10// optimizations for certain patterns.
11//
12// Option -aarch64-code-layout-opt-enable selects instruction pairs to optimize:
13// cmp-csel: Enable CMP/CMN-CSEL code layout optimization
14// fcmp-fcsel: Enable FCMP-FCSEL code layout optimization
15//
16// The initial implementation induces function alignment when a supported
17// pattern is detected, and possibly instruction-alignment when a pair would
18// straddle cache-lines.
19//===----------------------------------------------------------------------===//
20
21#include "AArch64.h"
22#include "AArch64InstrInfo.h"
23#include "AArch64Subtarget.h"
24#include "llvm/ADT/BitmaskEnum.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/Support/Debug.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/MathExtras.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "aarch64-code-layout-opt"
37#define DBG(...) LLVM_DEBUG(dbgs() << DEBUG_TYPE ": " << __VA_ARGS__)
38#define AARCH64_CODE_LAYOUT_OPT_NAME "AArch64 Code Layout Optimization"
39
40enum CodeLayoutOpt {
41 None = 0,
42 CmpCsel = 1 << 0, // Align CMP/CMN-CSEL pairs
43 FcmpFcsel = 1 << 1, // Align FCMP-FCSEL pairs
44 LLVM_MARK_AS_BITMASK_ENUM(FcmpFcsel)
45};
46
47static cl::bits<CodeLayoutOpt> EnableCodeAlignment(
48 "aarch64-code-layout-opt-enable", cl::Hidden, cl::CommaSeparated,
49 cl::desc("Enable code alignment optimization for instruction pairs"),
50 cl::values(
51 clEnumValN(None, "none", "Disable the code alignment pass"),
52 clEnumValN(CmpCsel, "cmp-csel", "CMP/CMN-CSEL pair alignment (32-bit)"),
53 clEnumValN(FcmpFcsel, "fcmp-fcsel", "FCMP-FCSEL pair alignment")));
54
55static cl::opt<unsigned> FunctionAlignBytes(
56 "aarch64-code-layout-opt-align-functions", cl::Hidden,
57 cl::desc("Function alignment in bytes for code layout optimization "
58 "(must be a power of 2)"),
59 cl::init(Val: 64), cl::callback(CB: [](const unsigned &Val) {
60 if (!isPowerOf2_32(Value: Val))
61 report_fatal_error(
62 reason: "aarch64-code-layout-opt-align must be a power of 2");
63 }));
64
65STATISTIC(NumFunctionsAligned,
66 "Number of functions with aligned (to 64-bytes by default)");
67STATISTIC(NumCmpCselPairsDetected,
68 "Number of CMP/CMN-CSEL pairs detected for alignment");
69STATISTIC(NumFcmpFcselPairsDetected,
70 "Number of FCMP-FCSEL pairs detected for alignment");
71
72namespace {
73
74class AArch64CodeLayoutOpt : public MachineFunctionPass {
75public:
76 static char ID;
77 AArch64CodeLayoutOpt() : MachineFunctionPass(ID) {}
78 void getAnalysisUsage(AnalysisUsage &AU) const override;
79 bool runOnMachineFunction(MachineFunction &MF) override;
80 StringRef getPassName() const override {
81 return AARCH64_CODE_LAYOUT_OPT_NAME;
82 }
83
84private:
85 const AArch64InstrInfo *TII = nullptr;
86
87 /// Align each fusible CMP/CMN-CSEL or FCMP-FCSEL pair in \p MBB by emitting
88 /// .p2align before the lead instruction (splitting the block if needed).
89 /// \returns true iff at least one pair was found and aligned.
90 bool alignLayoutSensitivePatterns(MachineBasicBlock *MBB, CodeLayoutOpt CLO);
91
92 /// Emit .p2align before MI. Splits the block if MI is not at its start.
93 void emitP2Align(MachineInstr &MI, Align DesiredAlign,
94 unsigned MaxSkipBytes = 4);
95
96 bool optimizeForCodeLayout(MachineFunction &MF, CodeLayoutOpt CLO);
97};
98
99} // end anonymous namespace
100
101char AArch64CodeLayoutOpt::ID = 0;
102
103INITIALIZE_PASS(AArch64CodeLayoutOpt, "aarch64-code-layout-opt",
104 AARCH64_CODE_LAYOUT_OPT_NAME, false, false)
105
106void AArch64CodeLayoutOpt::getAnalysisUsage(AnalysisUsage &AU) const {
107 AU.setPreservesAll();
108 MachineFunctionPass::getAnalysisUsage(AU);
109}
110
111FunctionPass *llvm::createAArch64CodeLayoutOptPass() {
112 return new AArch64CodeLayoutOpt();
113}
114
115/// \returns true iff Opc is a floating-point comparison (FCMP/FCMPE).
116static bool isFloatingPointCompare(unsigned Opc) {
117 switch (Opc) {
118 case AArch64::FCMPSrr:
119 case AArch64::FCMPDrr:
120 case AArch64::FCMPESrr:
121 case AArch64::FCMPEDrr:
122 case AArch64::FCMPHrr:
123 case AArch64::FCMPEHrr:
124 return true;
125 default:
126 return false;
127 }
128}
129
130/// \returns true iff Opc is a floating-point conditional select (FCSEL).
131static bool isFloatingPointConditionalSelect(unsigned Opc) {
132 switch (Opc) {
133 case AArch64::FCSELSrrr:
134 case AArch64::FCSELDrrr:
135 case AArch64::FCSELHrrr:
136 return true;
137 default:
138 return false;
139 }
140}
141
142/// \returns true if MI is a qualifying 32-bit CMP or CMN instruction.
143/// CMP is encoded as SUBS with WZR destination, CMN as ADDS with WZR.
144/// Only simple variants (no shifted/extended reg) qualify, and immediate
145/// variants require no LSL shift and small immediates (<=15).
146static bool isQualifyingIntCompare(const MachineInstr &MI) {
147 switch (MI.getOpcode()) {
148 case AArch64::SUBSWrr:
149 case AArch64::ADDSWrr:
150 return MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr);
151 case AArch64::SUBSWri:
152 case AArch64::ADDSWri:
153 return MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) &&
154 MI.getOperand(i: 3).getImm() == 0 && MI.getOperand(i: 2).getImm() <= 15;
155 case AArch64::SUBSWrs:
156 case AArch64::ADDSWrs:
157 return MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) &&
158 !AArch64InstrInfo::hasShiftedReg(MI);
159 case AArch64::SUBSWrx:
160 return MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) &&
161 !AArch64InstrInfo::hasExtendedReg(MI);
162 default:
163 return false;
164 }
165}
166
167bool AArch64CodeLayoutOpt::runOnMachineFunction(MachineFunction &MF) {
168 const Function &F = MF.getFunction();
169 // hasOptSize() returns true for both -Os and -Oz.
170 if (F.hasOptSize())
171 return false;
172
173 const auto *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
174 TII = Subtarget->getInstrInfo();
175
176 CodeLayoutOpt CLO = None;
177 if (EnableCodeAlignment.getNumOccurrences()) {
178 if (EnableCodeAlignment.isSet(V: CodeLayoutOpt::CmpCsel))
179 CLO |= CodeLayoutOpt::CmpCsel;
180 if (EnableCodeAlignment.isSet(V: CodeLayoutOpt::FcmpFcsel))
181 CLO |= CodeLayoutOpt::FcmpFcsel;
182 } else {
183 // Default: enable when the subtarget opts in via FeatureAlignCmpCSelPairs.
184 if (Subtarget->hasAlignCmpCSelPairs()) {
185 if (Subtarget->hasFuseCmpCSel())
186 CLO |= CodeLayoutOpt::CmpCsel;
187 if (Subtarget->hasFuseFCmpFCSel())
188 CLO |= CodeLayoutOpt::FcmpFcsel;
189 }
190 }
191
192 if (CLO == None)
193 return false;
194
195 return optimizeForCodeLayout(MF, CLO);
196}
197
198void AArch64CodeLayoutOpt::emitP2Align(MachineInstr &MI, Align DesiredAlign,
199 unsigned MaxSkipBytes) {
200 MachineBasicBlock *MBB = MI.getParent();
201
202 auto FirstReal =
203 skipDebugInstructionsForward(It: MBB->instr_begin(), End: MBB->instr_end());
204 if (&*FirstReal != &MI) {
205 auto PrevIt = prev_nodbg(It: MI.getIterator(), Begin: MBB->instr_begin());
206 MBB = MBB->splitAt(SplitInst&: *PrevIt, /*UpdateLiveIns=*/true);
207 }
208
209 MBB->setAlignment(DesiredAlign);
210 MBB->setMaxBytesForAlignment(MaxSkipBytes);
211}
212
213// Align each fusible CMP/CMN-CSEL or FCMP-FCSEL pair in MBB by emitting
214// .p2align before the lead instruction (splitting the block if needed).
215// A pair is: a qualifying lead instruction immediately followed by its
216// consumer (CMP/CMN→CSEL or FCMP→FCSEL), with no intervening instructions.
217// Returns true iff at least one pair was found and aligned.
218bool AArch64CodeLayoutOpt::alignLayoutSensitivePatterns(MachineBasicBlock *MBB,
219 CodeLayoutOpt CLO) {
220 auto End = MBB->instr_end();
221 SmallVector<std::pair<MachineInstr *, bool>, 4> Pairs;
222
223 for (auto &MI : instructionsWithoutDebug(It: MBB->begin(), End: MBB->end())) {
224 auto NextIt =
225 skipDebugInstructionsForward(It: std::next(x: MI.getIterator()), End);
226 if (NextIt == End)
227 break;
228
229 // --- CMP/CMN-CSEL detection ---
230 if ((CLO & CodeLayoutOpt::CmpCsel) && isQualifyingIntCompare(MI) &&
231 NextIt->getOpcode() == AArch64::CSELWr) {
232 Pairs.push_back(Elt: {&MI, true});
233 continue;
234 }
235
236 // --- FCMP-FCSEL detection ---
237 if ((CLO & CodeLayoutOpt::FcmpFcsel) &&
238 isFloatingPointCompare(Opc: MI.getOpcode()) &&
239 isFloatingPointConditionalSelect(Opc: NextIt->getOpcode())) {
240 Pairs.push_back(Elt: {&MI, false});
241 continue;
242 }
243 }
244
245 for (auto &[MI, IsCmpCsel] : Pairs) {
246 emitP2Align(MI&: *MI, DesiredAlign: Align(64));
247 DBG(".p2align 6, , 4 before " << *MI);
248 ++(IsCmpCsel ? NumCmpCselPairsDetected : NumFcmpFcselPairsDetected);
249 }
250
251 return !Pairs.empty();
252}
253
254bool AArch64CodeLayoutOpt::optimizeForCodeLayout(MachineFunction &MF,
255 CodeLayoutOpt CLO) {
256 DBG("optimizeForCodeLayout: " << MF.getName() << "\n");
257
258 bool Changed = false;
259 for (auto &MBB : MF)
260 Changed |= alignLayoutSensitivePatterns(MBB: &MBB, CLO);
261
262 if (!Changed)
263 return false;
264
265 if (MF.getAlignment() < Align(FunctionAlignBytes)) {
266 MF.setAlignment(Align(FunctionAlignBytes));
267 ++NumFunctionsAligned;
268 DBG("Set " << FunctionAlignBytes << "-byte alignment for function "
269 << MF.getName() << "\n");
270 } else {
271 DBG("Function " << MF.getName() << " already has sufficient alignment\n");
272 }
273 return true;
274}
275