1//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This file contains a pass that performs optimization on SIMD instructions
9// with high latency by splitting them into more efficient series of
10// instructions.
11//
12// 1. Rewrite certain SIMD instructions with vector element due to their
13// inefficiency on some targets.
14//
15// For example:
16// fmla v0.4s, v1.4s, v2.s[1]
17//
18// Is rewritten into:
19// dup v3.4s, v2.s[1]
20// fmla v0.4s, v1.4s, v3.4s
21//
22// 2. Rewrite interleaved memory access instructions due to their
23// inefficiency on some targets.
24//
25// For example:
26// st2 {v0.4s, v1.4s}, addr
27//
28// Is rewritten into:
29// zip1 v2.4s, v0.4s, v1.4s
30// zip2 v3.4s, v0.4s, v1.4s
31// stp q2, q3, addr
32//
33//===----------------------------------------------------------------------===//
34
35#include "AArch64InstrInfo.h"
36#include "AArch64Subtarget.h"
37#include "llvm/ADT/SmallVector.h"
38#include "llvm/ADT/Statistic.h"
39#include "llvm/ADT/StringMap.h"
40#include "llvm/ADT/StringRef.h"
41#include "llvm/CodeGen/MachineBasicBlock.h"
42#include "llvm/CodeGen/MachineFunction.h"
43#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
44#include "llvm/CodeGen/MachineFunctionPass.h"
45#include "llvm/CodeGen/MachineInstr.h"
46#include "llvm/CodeGen/MachineInstrBuilder.h"
47#include "llvm/CodeGen/MachineOperand.h"
48#include "llvm/CodeGen/MachineRegisterInfo.h"
49#include "llvm/CodeGen/RegisterClassInfo.h"
50#include "llvm/CodeGen/TargetInstrInfo.h"
51#include "llvm/CodeGen/TargetSchedule.h"
52#include "llvm/CodeGen/TargetSubtargetInfo.h"
53#include "llvm/MC/MCInstrDesc.h"
54#include "llvm/MC/MCSchedule.h"
55#include "llvm/Pass.h"
56#include <map>
57
58using namespace llvm;
59
60#define DEBUG_TYPE "aarch64-simd-instr-opt"
61
62STATISTIC(NumModifiedInstr,
63 "Number of SIMD instructions modified");
64
65#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \
66 "AArch64 SIMD instructions optimization pass"
67
68namespace {
69
70// A costly instruction is replaced in this work by N efficient instructions
71// The maximum of N is currently 10 and it is for ST4 case.
72constexpr unsigned MaxNumRepl = 10;
73
74class AArch64SIMDInstrOptImpl {
75public:
76 const AArch64InstrInfo *TII;
77 MachineRegisterInfo *MRI;
78 TargetSchedModel SchedModel;
79
80 using SIMDInstrTableMap = std::map<std::pair<unsigned, std::string>, bool>;
81
82 using InterlEarlyExitMap = StringMap<bool>;
83
84 // The two maps below are used to cache decisions instead of recomputing. Note
85 // that we're only storing references, the data is scoped at the Pass level to
86 // enable the caching.
87 //
88 // This is used to cache instruction replacement decisions within function
89 // units and across function units.
90 SIMDInstrTableMap &SIMDInstrTable;
91
92 // This is used to cache the decision of whether to leave the interleaved
93 // store instructions replacement pass early or not for a particular target.
94 InterlEarlyExitMap &InterlEarlyExit;
95
96 typedef enum {
97 VectorElem,
98 Interleave
99 } Subpass;
100
101 // Instruction represented by OrigOpc is replaced by instructions in ReplOpc.
102 struct InstReplInfo {
103 unsigned OrigOpc;
104 unsigned ReplOpc[MaxNumRepl];
105 unsigned NumRepl;
106 const TargetRegisterClass *RC;
107 };
108
109#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \
110 {OpcOrg, {OpcR0, OpcR1, OpcR2}, 3, &RC}
111#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \
112 OpcR7, OpcR8, OpcR9, RC) \
113 {OpcOrg, \
114 {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, \
115 10, \
116 &RC}
117
118 AArch64SIMDInstrOptImpl(SIMDInstrTableMap &SIMDInstrTable,
119 InterlEarlyExitMap &InterlEarlyExit)
120 : SIMDInstrTable(SIMDInstrTable), InterlEarlyExit(InterlEarlyExit) {}
121
122 /// Based only on latency of instructions, determine if it is cost efficient
123 /// to replace the instruction InstDesc by the instructions stored in the
124 /// array InstDescRepl.
125 /// Return true if replacement is expected to be faster.
126 bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
127 SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID);
128
129 /// Determine if we need to exit the instruction replacement optimization
130 /// passes early. This makes sure that no compile time is spent in this pass
131 /// for targets with no need for any of these optimizations.
132 /// Return true if early exit of the pass is recommended.
133 bool shouldExitEarly(MachineFunction *MF, Subpass SP);
134
135 /// Check whether an equivalent DUP instruction has already been
136 /// created or not.
137 /// Return true when the DUP instruction already exists. In this case,
138 /// DestReg will point to the destination of the already created DUP.
139 bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg,
140 unsigned LaneNumber, unsigned *DestReg) const;
141
142 /// Certain SIMD instructions with vector element operand are not efficient.
143 /// Rewrite them into SIMD instructions with vector operands. This rewrite
144 /// is driven by the latency of the instructions.
145 /// Return true if the SIMD instruction is modified.
146 bool optimizeVectElement(MachineInstr &MI);
147
148 /// Process The REG_SEQUENCE instruction, and extract the source
149 /// operands of the ST2/4 instruction from it.
150 /// Example of such instructions.
151 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
152 /// Return true when the instruction is processed successfully.
153 bool processSeqRegInst(MachineInstr *DefiningMI, unsigned *StReg,
154 RegState *StRegKill, unsigned NumArg) const;
155
156 /// Load/Store Interleaving instructions are not always beneficial.
157 /// Replace them by ZIP instructionand classical load/store.
158 /// Return true if the SIMD instruction is modified.
159 bool optimizeLdStInterleave(MachineInstr &MI);
160
161 /// Return the number of useful source registers for this
162 /// instruction (2 for ST2 and 4 for ST4).
163 unsigned determineSrcReg(MachineInstr &MI) const;
164
165 bool run(MachineFunction &MF);
166};
167
168struct AArch64SIMDInstrOptLegacy : public MachineFunctionPass {
169 static char ID;
170
171 AArch64SIMDInstrOptImpl::SIMDInstrTableMap SIMDInstrTable;
172 AArch64SIMDInstrOptImpl::InterlEarlyExitMap InterlEarlyExit;
173
174 AArch64SIMDInstrOptLegacy() : MachineFunctionPass(ID) {}
175
176 bool runOnMachineFunction(MachineFunction &Fn) override;
177
178 StringRef getPassName() const override {
179 return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME;
180 }
181
182 void getAnalysisUsage(AnalysisUsage &AU) const override {
183 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
184 MachineFunctionPass::getAnalysisUsage(AU);
185 }
186};
187
188char AArch64SIMDInstrOptLegacy::ID = 0;
189
190// The Instruction Replacement Table.
191constexpr AArch64SIMDInstrOptImpl::InstReplInfo IRT[] = {
192 // ST2 instructions
193 RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
194 AArch64::STPQi, AArch64::FPR128RegClass),
195 RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
196 AArch64::STPQi, AArch64::FPR128RegClass),
197 RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
198 AArch64::STPDi, AArch64::FPR64RegClass),
199 RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
200 AArch64::STPQi, AArch64::FPR128RegClass),
201 RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
202 AArch64::STPDi, AArch64::FPR64RegClass),
203 RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
204 AArch64::STPQi, AArch64::FPR128RegClass),
205 RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
206 AArch64::STPDi, AArch64::FPR64RegClass),
207 // ST4 instructions
208 RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
209 AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64,
210 AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
211 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
212 RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
213 AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32,
214 AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
215 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
216 RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
217 AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32,
218 AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
219 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
220 RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
221 AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16,
222 AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
223 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
224 RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
225 AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16,
226 AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
227 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
228 RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
229 AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8,
230 AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
231 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
232 RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
233 AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8,
234 AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
235 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass)};
236
237} // end anonymous namespace
238
239INITIALIZE_PASS(AArch64SIMDInstrOptLegacy, "aarch64-simd-instr-opt",
240 AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false)
241
242/// Based only on latency of instructions, determine if it is cost efficient
243/// to replace the instruction InstDesc by the instructions stored in the
244/// array InstDescRepl.
245/// Return true if replacement is expected to be faster.
246bool AArch64SIMDInstrOptImpl::shouldReplaceInst(
247 MachineFunction *MF, const MCInstrDesc *InstDesc,
248 SmallVectorImpl<const MCInstrDesc *> &InstDescRepl) {
249 // Check if replacement decision is already available in the cached table.
250 // if so, return it.
251 std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU());
252 auto InstID = std::make_pair(x: InstDesc->getOpcode(), y&: Subtarget);
253 auto It = SIMDInstrTable.find(x: InstID);
254 if (It != SIMDInstrTable.end())
255 return It->second;
256
257 unsigned SCIdx = InstDesc->getSchedClass();
258 const MCSchedClassDesc *SCDesc =
259 SchedModel.getMCSchedModel()->getSchedClassDesc(SchedClassIdx: SCIdx);
260
261 // If a target does not define resources for the instructions
262 // of interest, then return false for no replacement.
263 const MCSchedClassDesc *SCDescRepl;
264 if (!SCDesc->isValid() || SCDesc->isVariant())
265 {
266 SIMDInstrTable[InstID] = false;
267 return false;
268 }
269 for (const auto *IDesc : InstDescRepl)
270 {
271 SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc(
272 SchedClassIdx: IDesc->getSchedClass());
273 if (!SCDescRepl->isValid() || SCDescRepl->isVariant())
274 {
275 SIMDInstrTable[InstID] = false;
276 return false;
277 }
278 }
279
280 // Replacement cost.
281 unsigned ReplCost = 0;
282 for (const auto *IDesc :InstDescRepl)
283 ReplCost += SchedModel.computeInstrLatency(Opcode: IDesc->getOpcode());
284
285 if (SchedModel.computeInstrLatency(Opcode: InstDesc->getOpcode()) > ReplCost)
286 {
287 SIMDInstrTable[InstID] = true;
288 return true;
289 }
290 else
291 {
292 SIMDInstrTable[InstID] = false;
293 return false;
294 }
295}
296
297/// Determine if we need to exit this pass for a kind of instruction replacement
298/// early. This makes sure that no compile time is spent in this pass for
299/// targets with no need for any of these optimizations beyond performing this
300/// check.
301/// Return true if early exit of this pass for a kind of instruction
302/// replacement is recommended for a target.
303bool AArch64SIMDInstrOptImpl::shouldExitEarly(MachineFunction *MF, Subpass SP) {
304 const MCInstrDesc *OriginalMCID;
305 SmallVector<const MCInstrDesc *, MaxNumRepl> ReplInstrMCID;
306
307 switch (SP) {
308 // For this optimization, check by comparing the latency of a representative
309 // instruction to that of the replacement instructions.
310 // TODO: check for all concerned instructions.
311 case VectorElem:
312 OriginalMCID = &TII->get(Opcode: AArch64::FMLAv4i32_indexed);
313 ReplInstrMCID.push_back(Elt: &TII->get(Opcode: AArch64::DUPv4i32lane));
314 ReplInstrMCID.push_back(Elt: &TII->get(Opcode: AArch64::FMLAv4f32));
315 if (shouldReplaceInst(MF, InstDesc: OriginalMCID, InstDescRepl&: ReplInstrMCID))
316 return false;
317 break;
318
319 // For this optimization, check for all concerned instructions.
320 case Interleave:
321 std::string Subtarget =
322 std::string(SchedModel.getSubtargetInfo()->getCPU());
323 auto It = InterlEarlyExit.find(Key: Subtarget);
324 if (It != InterlEarlyExit.end())
325 return It->second;
326
327 for (const auto &I : IRT) {
328 OriginalMCID = &TII->get(Opcode: I.OrigOpc);
329 for (unsigned J = 0; J < I.NumRepl; ++J)
330 ReplInstrMCID.push_back(Elt: &TII->get(Opcode: I.ReplOpc[J]));
331 if (shouldReplaceInst(MF, InstDesc: OriginalMCID, InstDescRepl&: ReplInstrMCID)) {
332 InterlEarlyExit[Subtarget] = false;
333 return false;
334 }
335 ReplInstrMCID.clear();
336 }
337 InterlEarlyExit[Subtarget] = true;
338 break;
339 }
340
341 return true;
342}
343
344/// Check whether an equivalent DUP instruction has already been
345/// created or not.
346/// Return true when the DUP instruction already exists. In this case,
347/// DestReg will point to the destination of the already created DUP.
348bool AArch64SIMDInstrOptImpl::reuseDUP(MachineInstr &MI, unsigned DupOpcode,
349 unsigned SrcReg, unsigned LaneNumber,
350 unsigned *DestReg) const {
351 for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin();
352 MII != MIE;) {
353 MII--;
354 MachineInstr *CurrentMI = &*MII;
355
356 if (CurrentMI->getOpcode() == DupOpcode &&
357 CurrentMI->getNumOperands() == 3 &&
358 CurrentMI->getOperand(i: 1).getReg() == SrcReg &&
359 CurrentMI->getOperand(i: 2).getImm() == LaneNumber) {
360 *DestReg = CurrentMI->getOperand(i: 0).getReg();
361 return true;
362 }
363 }
364
365 return false;
366}
367
368/// Certain SIMD instructions with vector element operand are not efficient.
369/// Rewrite them into SIMD instructions with vector operands. This rewrite
370/// is driven by the latency of the instructions.
371/// The instruction of concerns are for the time being FMLA, FMLS, FMUL,
372/// and FMULX and hence they are hardcoded.
373///
374/// For example:
375/// fmla v0.4s, v1.4s, v2.s[1]
376///
377/// Is rewritten into
378/// dup v3.4s, v2.s[1] // DUP not necessary if redundant
379/// fmla v0.4s, v1.4s, v3.4s
380///
381/// Return true if the SIMD instruction is modified.
382bool AArch64SIMDInstrOptImpl::optimizeVectElement(MachineInstr &MI) {
383 const MCInstrDesc *MulMCID, *DupMCID;
384 const TargetRegisterClass *RC = &AArch64::FPR128RegClass;
385
386 switch (MI.getOpcode()) {
387 default:
388 return false;
389
390 // 4X32 instructions
391 case AArch64::FMLAv4i32_indexed:
392 DupMCID = &TII->get(Opcode: AArch64::DUPv4i32lane);
393 MulMCID = &TII->get(Opcode: AArch64::FMLAv4f32);
394 break;
395 case AArch64::FMLSv4i32_indexed:
396 DupMCID = &TII->get(Opcode: AArch64::DUPv4i32lane);
397 MulMCID = &TII->get(Opcode: AArch64::FMLSv4f32);
398 break;
399 case AArch64::FMULXv4i32_indexed:
400 DupMCID = &TII->get(Opcode: AArch64::DUPv4i32lane);
401 MulMCID = &TII->get(Opcode: AArch64::FMULXv4f32);
402 break;
403 case AArch64::FMULv4i32_indexed:
404 DupMCID = &TII->get(Opcode: AArch64::DUPv4i32lane);
405 MulMCID = &TII->get(Opcode: AArch64::FMULv4f32);
406 break;
407
408 // 2X64 instructions
409 case AArch64::FMLAv2i64_indexed:
410 DupMCID = &TII->get(Opcode: AArch64::DUPv2i64lane);
411 MulMCID = &TII->get(Opcode: AArch64::FMLAv2f64);
412 break;
413 case AArch64::FMLSv2i64_indexed:
414 DupMCID = &TII->get(Opcode: AArch64::DUPv2i64lane);
415 MulMCID = &TII->get(Opcode: AArch64::FMLSv2f64);
416 break;
417 case AArch64::FMULXv2i64_indexed:
418 DupMCID = &TII->get(Opcode: AArch64::DUPv2i64lane);
419 MulMCID = &TII->get(Opcode: AArch64::FMULXv2f64);
420 break;
421 case AArch64::FMULv2i64_indexed:
422 DupMCID = &TII->get(Opcode: AArch64::DUPv2i64lane);
423 MulMCID = &TII->get(Opcode: AArch64::FMULv2f64);
424 break;
425
426 // 2X32 instructions
427 case AArch64::FMLAv2i32_indexed:
428 RC = &AArch64::FPR64RegClass;
429 DupMCID = &TII->get(Opcode: AArch64::DUPv2i32lane);
430 MulMCID = &TII->get(Opcode: AArch64::FMLAv2f32);
431 break;
432 case AArch64::FMLSv2i32_indexed:
433 RC = &AArch64::FPR64RegClass;
434 DupMCID = &TII->get(Opcode: AArch64::DUPv2i32lane);
435 MulMCID = &TII->get(Opcode: AArch64::FMLSv2f32);
436 break;
437 case AArch64::FMULXv2i32_indexed:
438 RC = &AArch64::FPR64RegClass;
439 DupMCID = &TII->get(Opcode: AArch64::DUPv2i32lane);
440 MulMCID = &TII->get(Opcode: AArch64::FMULXv2f32);
441 break;
442 case AArch64::FMULv2i32_indexed:
443 RC = &AArch64::FPR64RegClass;
444 DupMCID = &TII->get(Opcode: AArch64::DUPv2i32lane);
445 MulMCID = &TII->get(Opcode: AArch64::FMULv2f32);
446 break;
447 }
448
449 SmallVector<const MCInstrDesc*, 2> ReplInstrMCID;
450 ReplInstrMCID.push_back(Elt: DupMCID);
451 ReplInstrMCID.push_back(Elt: MulMCID);
452 if (!shouldReplaceInst(MF: MI.getParent()->getParent(), InstDesc: &TII->get(Opcode: MI.getOpcode()),
453 InstDescRepl&: ReplInstrMCID))
454 return false;
455
456 const DebugLoc &DL = MI.getDebugLoc();
457 MachineBasicBlock &MBB = *MI.getParent();
458 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
459
460 // Get the operands of the current SIMD arithmetic instruction.
461 Register MulDest = MI.getOperand(i: 0).getReg();
462 Register SrcReg0 = MI.getOperand(i: 1).getReg();
463 RegState Src0IsKill = getKillRegState(B: MI.getOperand(i: 1).isKill());
464 Register SrcReg1 = MI.getOperand(i: 2).getReg();
465 RegState Src1IsKill = getKillRegState(B: MI.getOperand(i: 2).isKill());
466 unsigned DupDest;
467
468 // Instructions of interest have either 4 or 5 operands.
469 if (MI.getNumOperands() == 5) {
470 Register SrcReg2 = MI.getOperand(i: 3).getReg();
471 RegState Src2IsKill = getKillRegState(B: MI.getOperand(i: 3).isKill());
472 unsigned LaneNumber = MI.getOperand(i: 4).getImm();
473 // Create a new DUP instruction. Note that if an equivalent DUP instruction
474 // has already been created before, then use that one instead of creating
475 // a new one.
476 if (!reuseDUP(MI, DupOpcode: DupMCID->getOpcode(), SrcReg: SrcReg2, LaneNumber, DestReg: &DupDest)) {
477 DupDest = MRI.createVirtualRegister(RegClass: RC);
478 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *DupMCID, DestReg: DupDest)
479 .addReg(RegNo: SrcReg2, Flags: Src2IsKill)
480 .addImm(Val: LaneNumber);
481 }
482 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *MulMCID, DestReg: MulDest)
483 .addReg(RegNo: SrcReg0, Flags: Src0IsKill)
484 .addReg(RegNo: SrcReg1, Flags: Src1IsKill)
485 .addReg(RegNo: DupDest, Flags: Src2IsKill);
486 } else if (MI.getNumOperands() == 4) {
487 unsigned LaneNumber = MI.getOperand(i: 3).getImm();
488 if (!reuseDUP(MI, DupOpcode: DupMCID->getOpcode(), SrcReg: SrcReg1, LaneNumber, DestReg: &DupDest)) {
489 DupDest = MRI.createVirtualRegister(RegClass: RC);
490 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *DupMCID, DestReg: DupDest)
491 .addReg(RegNo: SrcReg1, Flags: Src1IsKill)
492 .addImm(Val: LaneNumber);
493 }
494 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *MulMCID, DestReg: MulDest)
495 .addReg(RegNo: SrcReg0, Flags: Src0IsKill)
496 .addReg(RegNo: DupDest, Flags: Src1IsKill);
497 } else {
498 return false;
499 }
500
501 ++NumModifiedInstr;
502 return true;
503}
504
505/// Load/Store Interleaving instructions are not always beneficial.
506/// Replace them by ZIP instructions and classical load/store.
507///
508/// For example:
509/// st2 {v0.4s, v1.4s}, addr
510///
511/// Is rewritten into:
512/// zip1 v2.4s, v0.4s, v1.4s
513/// zip2 v3.4s, v0.4s, v1.4s
514/// stp q2, q3, addr
515//
516/// For example:
517/// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr
518///
519/// Is rewritten into:
520/// zip1 v4.4s, v0.4s, v2.4s
521/// zip2 v5.4s, v0.4s, v2.4s
522/// zip1 v6.4s, v1.4s, v3.4s
523/// zip2 v7.4s, v1.4s, v3.4s
524/// zip1 v8.4s, v4.4s, v6.4s
525/// zip2 v9.4s, v4.4s, v6.4s
526/// zip1 v10.4s, v5.4s, v7.4s
527/// zip2 v11.4s, v5.4s, v7.4s
528/// stp q8, q9, addr
529/// stp q10, q11, addr+32
530///
531/// Currently only instructions related to ST2 and ST4 are considered.
532/// Other may be added later.
533/// Return true if the SIMD instruction is modified.
534bool AArch64SIMDInstrOptImpl::optimizeLdStInterleave(MachineInstr &MI) {
535
536 unsigned SeqReg, AddrReg;
537 unsigned StReg[4];
538 RegState StRegKill[4];
539 MachineInstr *DefiningMI;
540 const DebugLoc &DL = MI.getDebugLoc();
541 MachineBasicBlock &MBB = *MI.getParent();
542 SmallVector<unsigned, MaxNumRepl> ZipDest;
543 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;
544
545 // If current instruction matches any of the rewriting rules, then
546 // gather information about parameters of the new instructions.
547 bool Match = false;
548 for (const auto &I : IRT) {
549 if (MI.getOpcode() == I.OrigOpc) {
550 SeqReg = MI.getOperand(i: 0).getReg();
551 AddrReg = MI.getOperand(i: 1).getReg();
552 DefiningMI = MRI->getUniqueVRegDef(Reg: SeqReg);
553 unsigned NumReg = determineSrcReg(MI);
554 if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumArg: NumReg))
555 return false;
556
557 for (unsigned J = 0; J < I.NumRepl; ++J) {
558 unsigned Repl = I.ReplOpc[J];
559 ReplInstrMCID.push_back(Elt: &TII->get(Opcode: Repl));
560 // Generate destination registers but only for non-store instruction.
561 if (Repl != AArch64::STPQi && Repl != AArch64::STPDi)
562 ZipDest.push_back(Elt: MRI->createVirtualRegister(RegClass: I.RC));
563 }
564 Match = true;
565 break;
566 }
567 }
568
569 if (!Match)
570 return false;
571
572 // Determine if it is profitable to replace MI by the series of instructions
573 // represented in ReplInstrMCID.
574 if (!shouldReplaceInst(MF: MI.getParent()->getParent(), InstDesc: &TII->get(Opcode: MI.getOpcode()),
575 InstDescRepl&: ReplInstrMCID))
576 return false;
577
578 // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at
579 // this point, the code generation is hardcoded and does not rely on the IRT
580 // table used above given that code generation for ST2 replacement is somewhat
581 // different than for ST4 replacement. We could have added more info into the
582 // table related to how we build new instructions but we may be adding more
583 // complexity with that).
584 switch (MI.getOpcode()) {
585 default:
586 return false;
587
588 case AArch64::ST2Twov16b:
589 case AArch64::ST2Twov8b:
590 case AArch64::ST2Twov8h:
591 case AArch64::ST2Twov4h:
592 case AArch64::ST2Twov4s:
593 case AArch64::ST2Twov2s:
594 case AArch64::ST2Twov2d:
595 // ZIP instructions
596 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[0], DestReg: ZipDest[0])
597 .addReg(RegNo: StReg[0])
598 .addReg(RegNo: StReg[1]);
599 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[1], DestReg: ZipDest[1])
600 .addReg(RegNo: StReg[0], Flags: StRegKill[0])
601 .addReg(RegNo: StReg[1], Flags: StRegKill[1]);
602 // STP instructions
603 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[2])
604 .addReg(RegNo: ZipDest[0])
605 .addReg(RegNo: ZipDest[1])
606 .addReg(RegNo: AddrReg)
607 .addImm(Val: 0);
608 break;
609
610 case AArch64::ST4Fourv16b:
611 case AArch64::ST4Fourv8b:
612 case AArch64::ST4Fourv8h:
613 case AArch64::ST4Fourv4h:
614 case AArch64::ST4Fourv4s:
615 case AArch64::ST4Fourv2s:
616 case AArch64::ST4Fourv2d:
617 // ZIP instructions
618 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[0], DestReg: ZipDest[0])
619 .addReg(RegNo: StReg[0])
620 .addReg(RegNo: StReg[2]);
621 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[1], DestReg: ZipDest[1])
622 .addReg(RegNo: StReg[0], Flags: StRegKill[0])
623 .addReg(RegNo: StReg[2], Flags: StRegKill[2]);
624 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[2], DestReg: ZipDest[2])
625 .addReg(RegNo: StReg[1])
626 .addReg(RegNo: StReg[3]);
627 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[3], DestReg: ZipDest[3])
628 .addReg(RegNo: StReg[1], Flags: StRegKill[1])
629 .addReg(RegNo: StReg[3], Flags: StRegKill[3]);
630 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[4], DestReg: ZipDest[4])
631 .addReg(RegNo: ZipDest[0])
632 .addReg(RegNo: ZipDest[2]);
633 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[5], DestReg: ZipDest[5])
634 .addReg(RegNo: ZipDest[0])
635 .addReg(RegNo: ZipDest[2]);
636 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[6], DestReg: ZipDest[6])
637 .addReg(RegNo: ZipDest[1])
638 .addReg(RegNo: ZipDest[3]);
639 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[7], DestReg: ZipDest[7])
640 .addReg(RegNo: ZipDest[1])
641 .addReg(RegNo: ZipDest[3]);
642 // stp instructions
643 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[8])
644 .addReg(RegNo: ZipDest[4])
645 .addReg(RegNo: ZipDest[5])
646 .addReg(RegNo: AddrReg)
647 .addImm(Val: 0);
648 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: *ReplInstrMCID[9])
649 .addReg(RegNo: ZipDest[6])
650 .addReg(RegNo: ZipDest[7])
651 .addReg(RegNo: AddrReg)
652 .addImm(Val: 2);
653 break;
654 }
655
656 ++NumModifiedInstr;
657 return true;
658}
659
660/// Process The REG_SEQUENCE instruction, and extract the source
661/// operands of the ST2/4 instruction from it.
662/// Example of such instruction.
663/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
664/// Return true when the instruction is processed successfully.
665bool AArch64SIMDInstrOptImpl::processSeqRegInst(MachineInstr *DefiningMI,
666 unsigned *StReg,
667 RegState *StRegKill,
668 unsigned NumArg) const {
669 assert(DefiningMI != nullptr);
670 if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE)
671 return false;
672
673 for (unsigned i=0; i<NumArg; i++) {
674 StReg[i] = DefiningMI->getOperand(i: 2*i+1).getReg();
675 StRegKill[i] = getKillRegState(B: DefiningMI->getOperand(i: 2*i+1).isKill());
676
677 // Validation check for the other arguments.
678 if (DefiningMI->getOperand(i: 2*i+2).isImm()) {
679 switch (DefiningMI->getOperand(i: 2*i+2).getImm()) {
680 default:
681 return false;
682
683 case AArch64::dsub0:
684 case AArch64::dsub1:
685 case AArch64::dsub2:
686 case AArch64::dsub3:
687 case AArch64::qsub0:
688 case AArch64::qsub1:
689 case AArch64::qsub2:
690 case AArch64::qsub3:
691 break;
692 }
693 }
694 else
695 return false;
696 }
697 return true;
698}
699
700/// Return the number of useful source registers for this instruction
701/// (2 for ST2 and 4 for ST4).
702unsigned AArch64SIMDInstrOptImpl::determineSrcReg(MachineInstr &MI) const {
703 switch (MI.getOpcode()) {
704 default:
705 llvm_unreachable("Unsupported instruction for this pass");
706
707 case AArch64::ST2Twov16b:
708 case AArch64::ST2Twov8b:
709 case AArch64::ST2Twov8h:
710 case AArch64::ST2Twov4h:
711 case AArch64::ST2Twov4s:
712 case AArch64::ST2Twov2s:
713 case AArch64::ST2Twov2d:
714 return 2;
715
716 case AArch64::ST4Fourv16b:
717 case AArch64::ST4Fourv8b:
718 case AArch64::ST4Fourv8h:
719 case AArch64::ST4Fourv4h:
720 case AArch64::ST4Fourv4s:
721 case AArch64::ST4Fourv2s:
722 case AArch64::ST4Fourv2d:
723 return 4;
724 }
725}
726
727bool AArch64SIMDInstrOptImpl::run(MachineFunction &MF) {
728 MRI = &MF.getRegInfo();
729 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
730 TII = ST.getInstrInfo();
731 SchedModel.init(TSInfo: &ST);
732 if (!SchedModel.hasInstrSchedModel())
733 return false;
734
735 bool Changed = false;
736 for (auto OptimizationKind : {VectorElem, Interleave}) {
737 if (!shouldExitEarly(MF: &MF, SP: OptimizationKind)) {
738 SmallVector<MachineInstr *, 8> RemoveMIs;
739 for (MachineBasicBlock &MBB : MF) {
740 for (MachineInstr &MI : MBB) {
741 bool InstRewrite;
742 if (OptimizationKind == VectorElem)
743 InstRewrite = optimizeVectElement(MI) ;
744 else
745 InstRewrite = optimizeLdStInterleave(MI);
746 if (InstRewrite) {
747 // Add MI to the list of instructions to be removed given that it
748 // has been replaced.
749 RemoveMIs.push_back(Elt: &MI);
750 Changed = true;
751 }
752 }
753 }
754 for (MachineInstr *MI : RemoveMIs)
755 MI->eraseFromParent();
756 }
757 }
758
759 return Changed;
760}
761
762bool AArch64SIMDInstrOptLegacy::runOnMachineFunction(MachineFunction &MF) {
763 if (skipFunction(F: MF.getFunction()))
764 return false;
765
766 return AArch64SIMDInstrOptImpl(SIMDInstrTable, InterlEarlyExit).run(MF);
767}
768
769PreservedAnalyses
770AArch64SIMDInstrOptPass::run(MachineFunction &MF,
771 MachineFunctionAnalysisManager &MFAM) {
772 const bool Changed =
773 AArch64SIMDInstrOptImpl(SIMDInstrTable, InterlEarlyExit).run(MF);
774 if (!Changed)
775 return PreservedAnalyses::all();
776
777 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
778 PA.preserveSet<CFGAnalyses>();
779 return PA;
780}
781
782/// Returns an instance of the high cost ASIMD instruction replacement
783/// optimization pass.
784FunctionPass *llvm::createAArch64SIMDInstrOptPass() {
785 return new AArch64SIMDInstrOptLegacy();
786}
787