1//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AArch64 implementation of the DAG scheduling
10/// mutation to pair instructions back to back.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MacroFusion.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/CodeGen/MacroFusion.h"
18#include "llvm/CodeGen/TargetInstrInfo.h"
19
20#define DEBUG_TYPE "aarch64-macro-fusion"
21
22using namespace llvm;
23
24STATISTIC(NumFusedArithmeticBcc, "Number of arithmetic-Bcc fusions");
25STATISTIC(NumFusedArithmeticCbz, "Number of arithmetic-Cbz fusions");
26STATISTIC(NumFusedAES, "Number of AES fusions");
27STATISTIC(NumFusedCryptoEOR, "Number of crypto-EOR fusions");
28STATISTIC(NumFusedAdrpAdd, "Number of ADRP-ADD fusions");
29STATISTIC(NumFusedLiterals, "Number of literal-generation fusions");
30STATISTIC(NumFusedAddress, "Number of address-generation load/store fusions");
31STATISTIC(NumFusedCmpCSel, "Number of compare-CSEL fusions");
32STATISTIC(NumFusedFCmpFCSel, "Number of FP-compare-FCSEL fusions");
33STATISTIC(NumFusedCmpCSet, "Number of compare-CSET fusions");
34STATISTIC(NumFusedArithmeticLogic, "Number of arithmetic-logic fusions");
35STATISTIC(NumFusedAddSub2RegAndConstOne,
36 "Number of add/sub-two-register-and-constant-one fusions");
37STATISTIC(NumFusedAppleSMECompute, "Number of Apple SME compute fusions");
38STATISTIC(NumFusedFMinFMax, "Number of FMIN-FMAX fusions");
39
40/// CMN, CMP, TST followed by Bcc
41static bool isArithmeticBccPair(const MachineInstr *FirstMI,
42 const MachineInstr &SecondMI, bool CmpOnly) {
43 if (SecondMI.getOpcode() != AArch64::Bcc)
44 return false;
45
46 // Assume the 1st instr to be a wildcard if it is unspecified.
47 if (FirstMI == nullptr)
48 return true;
49
50 // If we're in CmpOnly mode, we only fuse arithmetic instructions that
51 // discard their result.
52 if (CmpOnly && FirstMI->getOperand(i: 0).isReg() &&
53 !(FirstMI->getOperand(i: 0).getReg() == AArch64::XZR ||
54 FirstMI->getOperand(i: 0).getReg() == AArch64::WZR)) {
55 return false;
56 }
57
58 switch (FirstMI->getOpcode()) {
59 case AArch64::ADDSWri:
60 case AArch64::ADDSWrr:
61 case AArch64::ADDSXri:
62 case AArch64::ADDSXrr:
63 case AArch64::ANDSWri:
64 case AArch64::ANDSWrr:
65 case AArch64::ANDSXri:
66 case AArch64::ANDSXrr:
67 case AArch64::SUBSWri:
68 case AArch64::SUBSWrr:
69 case AArch64::SUBSXri:
70 case AArch64::SUBSXrr:
71 case AArch64::BICSWrr:
72 case AArch64::BICSXrr:
73 return true;
74 case AArch64::ADDSWrs:
75 case AArch64::ADDSXrs:
76 case AArch64::ANDSWrs:
77 case AArch64::ANDSXrs:
78 case AArch64::SUBSWrs:
79 case AArch64::SUBSXrs:
80 case AArch64::BICSWrs:
81 case AArch64::BICSXrs:
82 // Shift value can be 0 making these behave like the "rr" variant...
83 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
84 }
85
86 return false;
87}
88
89/// ALU operations followed by CBZ/CBNZ.
90static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
91 const MachineInstr &SecondMI) {
92 if (SecondMI.getOpcode() != AArch64::CBZW &&
93 SecondMI.getOpcode() != AArch64::CBZX &&
94 SecondMI.getOpcode() != AArch64::CBNZW &&
95 SecondMI.getOpcode() != AArch64::CBNZX &&
96 SecondMI.getOpcode() != AArch64::TBZW &&
97 SecondMI.getOpcode() != AArch64::TBZX &&
98 SecondMI.getOpcode() != AArch64::TBNZW &&
99 SecondMI.getOpcode() != AArch64::TBNZX)
100 return false;
101
102 // Assume the 1st instr to be a wildcard if it is unspecified.
103 if (FirstMI == nullptr)
104 return true;
105
106 switch (FirstMI->getOpcode()) {
107 case AArch64::ADDWri:
108 case AArch64::ADDWrr:
109 case AArch64::ADDXri:
110 case AArch64::ADDXrr:
111 case AArch64::ANDWri:
112 case AArch64::ANDWrr:
113 case AArch64::ANDXri:
114 case AArch64::ANDXrr:
115 case AArch64::EORWri:
116 case AArch64::EORWrr:
117 case AArch64::EORXri:
118 case AArch64::EORXrr:
119 case AArch64::ORRWri:
120 case AArch64::ORRWrr:
121 case AArch64::ORRXri:
122 case AArch64::ORRXrr:
123 case AArch64::ORNWrr:
124 case AArch64::ORNXrr:
125 case AArch64::SUBWri:
126 case AArch64::SUBWrr:
127 case AArch64::SUBXri:
128 case AArch64::SUBXrr:
129 case AArch64::BICWrr:
130 case AArch64::BICXrr:
131 return true;
132 case AArch64::ADDWrs:
133 case AArch64::ADDXrs:
134 case AArch64::ANDWrs:
135 case AArch64::ANDXrs:
136 case AArch64::EORWrs:
137 case AArch64::EORXrs:
138 case AArch64::ORNWrs:
139 case AArch64::ORNXrs:
140 case AArch64::ORRWrs:
141 case AArch64::ORRXrs:
142 case AArch64::SUBWrs:
143 case AArch64::SUBXrs:
144 case AArch64::BICWrs:
145 case AArch64::BICXrs:
146 // Shift value can be 0 making these behave like the "rr" variant...
147 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
148 }
149
150 return false;
151}
152
153// True unless the pair provably defines different physical registers. This is
154// an exact match, so it doesn't account for subreg overlap. Pre-RA the dests
155// are still virtual, so it returns true.
156static bool mayDefSameReg(const MachineInstr &FirstMI,
157 const MachineInstr &SecondMI) {
158 Register DestFirst = FirstMI.getOperand(i: 0).getReg();
159 Register DestSecond = SecondMI.getOperand(i: 0).getReg();
160 if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
161 return true;
162 return DestFirst == DestSecond;
163}
164
165/// AES crypto encoding or decoding.
166static bool isAESPair(const MachineInstr *FirstMI,
167 const MachineInstr &SecondMI) {
168 // Assume the 1st instr to be a wildcard if it is unspecified.
169 unsigned SecondOpcode = SecondMI.getOpcode();
170 switch (SecondOpcode) {
171 // AES encode.
172 case AArch64::AESMCrr:
173 case AArch64::AESMCrrTied:
174 if (FirstMI == nullptr)
175 return true;
176 if (FirstMI->getOpcode() != AArch64::AESErr)
177 return false;
178 return SecondOpcode == AArch64::AESMCrrTied ||
179 mayDefSameReg(FirstMI: *FirstMI, SecondMI);
180 // AES decode.
181 case AArch64::AESIMCrr:
182 case AArch64::AESIMCrrTied:
183 if (FirstMI == nullptr)
184 return true;
185 if (FirstMI->getOpcode() != AArch64::AESDrr)
186 return false;
187 return SecondOpcode == AArch64::AESIMCrrTied ||
188 mayDefSameReg(FirstMI: *FirstMI, SecondMI);
189 }
190
191 return false;
192}
193
194/// AESE/AESD/PMULL + EOR.
195static bool isCryptoEORPair(const MachineInstr *FirstMI,
196 const MachineInstr &SecondMI) {
197 if (SecondMI.getOpcode() != AArch64::EORv16i8)
198 return false;
199
200 // Assume the 1st instr to be a wildcard if it is unspecified.
201 if (FirstMI == nullptr)
202 return true;
203
204 switch (FirstMI->getOpcode()) {
205 case AArch64::AESErr:
206 case AArch64::AESDrr:
207 case AArch64::PMULLv16i8:
208 case AArch64::PMULLv8i8:
209 case AArch64::PMULLv1i64:
210 case AArch64::PMULLv2i64:
211 return true;
212 }
213
214 return false;
215}
216
217static bool isAdrpAddPair(const MachineInstr *FirstMI,
218 const MachineInstr &SecondMI) {
219 // Assume the 1st instr to be a wildcard if it is unspecified.
220 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
221 SecondMI.getOpcode() == AArch64::ADDXri)
222 return true;
223 return false;
224}
225
226/// Fuse address generation and loads or stores.
227static bool isAddressLdStPair(const MachineInstr *FirstMI,
228 const MachineInstr &SecondMI) {
229 switch (SecondMI.getOpcode()) {
230 case AArch64::STRBBui:
231 case AArch64::STRBui:
232 case AArch64::STRDui:
233 case AArch64::STRHHui:
234 case AArch64::STRHui:
235 case AArch64::STRQui:
236 case AArch64::STRSui:
237 case AArch64::STRWui:
238 case AArch64::STRXui:
239 case AArch64::LDRBBui:
240 case AArch64::LDRBui:
241 case AArch64::LDRDui:
242 case AArch64::LDRHHui:
243 case AArch64::LDRHui:
244 case AArch64::LDRQui:
245 case AArch64::LDRSui:
246 case AArch64::LDRWui:
247 case AArch64::LDRXui:
248 case AArch64::LDRSBWui:
249 case AArch64::LDRSBXui:
250 case AArch64::LDRSHWui:
251 case AArch64::LDRSHXui:
252 case AArch64::LDRSWui:
253 // Assume the 1st instr to be a wildcard if it is unspecified.
254 if (FirstMI == nullptr)
255 return true;
256
257 switch (FirstMI->getOpcode()) {
258 case AArch64::ADR:
259 return SecondMI.getOperand(i: 2).getImm() == 0;
260 case AArch64::ADRP:
261 return true;
262 }
263 }
264
265 return false;
266}
267
268/// Compare and conditional select.
269static bool isCmpCSelPair(const MachineInstr *FirstMI,
270 const MachineInstr &SecondMI) {
271 // 32 bits
272 if (SecondMI.getOpcode() == AArch64::CSELWr) {
273 // Assume the 1st instr to be a wildcard if it is unspecified.
274 if (FirstMI == nullptr)
275 return true;
276
277 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr))
278 switch (FirstMI->getOpcode()) {
279 case AArch64::SUBSWrs:
280 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
281 case AArch64::SUBSWrx:
282 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
283 case AArch64::SUBSWrr:
284 case AArch64::SUBSWri:
285 return true;
286 }
287 }
288
289 // 64 bits
290 if (SecondMI.getOpcode() == AArch64::CSELXr) {
291 // Assume the 1st instr to be a wildcard if it is unspecified.
292 if (FirstMI == nullptr)
293 return true;
294
295 if (FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
296 switch (FirstMI->getOpcode()) {
297 case AArch64::SUBSXrs:
298 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
299 case AArch64::SUBSXrx:
300 case AArch64::SUBSXrx64:
301 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
302 case AArch64::SUBSXrr:
303 case AArch64::SUBSXri:
304 return true;
305 }
306 }
307
308 return false;
309}
310
311/// Floating-point compare and floating-point conditional select.
312static bool isFCmpFCSelPair(const MachineInstr *FirstMI,
313 const MachineInstr &SecondMI) {
314 switch (SecondMI.getOpcode()) {
315 case AArch64::FCSELSrrr:
316 case AArch64::FCSELDrrr:
317 case AArch64::FCSELHrrr:
318 break;
319 default:
320 return false;
321 }
322
323 // Assume the 1st instr to be a wildcard if it is unspecified.
324 if (FirstMI == nullptr)
325 return true;
326
327 switch (FirstMI->getOpcode()) {
328 case AArch64::FCMPSrr:
329 case AArch64::FCMPDrr:
330 case AArch64::FCMPESrr:
331 case AArch64::FCMPEDrr:
332 case AArch64::FCMPHrr:
333 case AArch64::FCMPEHrr:
334 return true;
335 default:
336 return false;
337 }
338}
339
340/// Compare and cset.
341static bool isCmpCSetPair(const MachineInstr *FirstMI,
342 const MachineInstr &SecondMI) {
343 if ((SecondMI.getOpcode() == AArch64::CSINCWr &&
344 SecondMI.getOperand(i: 1).getReg() == AArch64::WZR &&
345 SecondMI.getOperand(i: 2).getReg() == AArch64::WZR) ||
346 (SecondMI.getOpcode() == AArch64::CSINCXr &&
347 SecondMI.getOperand(i: 1).getReg() == AArch64::XZR &&
348 SecondMI.getOperand(i: 2).getReg() == AArch64::XZR)) {
349 // Assume the 1st instr to be a wildcard if it is unspecified.
350 if (FirstMI == nullptr)
351 return true;
352
353 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
354 FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
355 switch (FirstMI->getOpcode()) {
356 case AArch64::SUBSWrs:
357 case AArch64::SUBSXrs:
358 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
359 case AArch64::SUBSWrx:
360 case AArch64::SUBSXrx:
361 case AArch64::SUBSXrx64:
362 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
363 case AArch64::SUBSWri:
364 case AArch64::SUBSWrr:
365 case AArch64::SUBSXri:
366 case AArch64::SUBSXrr:
367 return true;
368 }
369 }
370
371 return false;
372}
373
374// Arithmetic and logic.
375static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
376 const MachineInstr &SecondMI) {
377 if (AArch64InstrInfo::hasShiftedReg(MI: SecondMI))
378 return false;
379
380 switch (SecondMI.getOpcode()) {
381 // Arithmetic
382 case AArch64::ADDWrr:
383 case AArch64::ADDXrr:
384 case AArch64::SUBWrr:
385 case AArch64::SUBXrr:
386 case AArch64::ADDWrs:
387 case AArch64::ADDXrs:
388 case AArch64::SUBWrs:
389 case AArch64::SUBXrs:
390 // Logic
391 case AArch64::ANDWrr:
392 case AArch64::ANDXrr:
393 case AArch64::BICWrr:
394 case AArch64::BICXrr:
395 case AArch64::EONWrr:
396 case AArch64::EONXrr:
397 case AArch64::EORWrr:
398 case AArch64::EORXrr:
399 case AArch64::ORNWrr:
400 case AArch64::ORNXrr:
401 case AArch64::ORRWrr:
402 case AArch64::ORRXrr:
403 case AArch64::ANDWrs:
404 case AArch64::ANDXrs:
405 case AArch64::BICWrs:
406 case AArch64::BICXrs:
407 case AArch64::EONWrs:
408 case AArch64::EONXrs:
409 case AArch64::EORWrs:
410 case AArch64::EORXrs:
411 case AArch64::ORNWrs:
412 case AArch64::ORNXrs:
413 case AArch64::ORRWrs:
414 case AArch64::ORRXrs:
415 // Assume the 1st instr to be a wildcard if it is unspecified.
416 if (FirstMI == nullptr)
417 return true;
418
419 // Arithmetic
420 switch (FirstMI->getOpcode()) {
421 case AArch64::ADDWrr:
422 case AArch64::ADDXrr:
423 case AArch64::ADDSWrr:
424 case AArch64::ADDSXrr:
425 case AArch64::SUBWrr:
426 case AArch64::SUBXrr:
427 case AArch64::SUBSWrr:
428 case AArch64::SUBSXrr:
429 return true;
430 case AArch64::ADDWrs:
431 case AArch64::ADDXrs:
432 case AArch64::ADDSWrs:
433 case AArch64::ADDSXrs:
434 case AArch64::SUBWrs:
435 case AArch64::SUBXrs:
436 case AArch64::SUBSWrs:
437 case AArch64::SUBSXrs:
438 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
439 }
440 break;
441
442 // Arithmetic, setting flags.
443 case AArch64::ADDSWrr:
444 case AArch64::ADDSXrr:
445 case AArch64::SUBSWrr:
446 case AArch64::SUBSXrr:
447 case AArch64::ADDSWrs:
448 case AArch64::ADDSXrs:
449 case AArch64::SUBSWrs:
450 case AArch64::SUBSXrs:
451 // Assume the 1st instr to be a wildcard if it is unspecified.
452 if (FirstMI == nullptr)
453 return true;
454
455 // Arithmetic, not setting flags.
456 switch (FirstMI->getOpcode()) {
457 case AArch64::ADDWrr:
458 case AArch64::ADDXrr:
459 case AArch64::SUBWrr:
460 case AArch64::SUBXrr:
461 return true;
462 case AArch64::ADDWrs:
463 case AArch64::ADDXrs:
464 case AArch64::SUBWrs:
465 case AArch64::SUBXrs:
466 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
467 }
468 break;
469 }
470
471 return false;
472}
473
474// "(A + B) + 1" or "(A - B) - 1"
475static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
476 const MachineInstr &SecondMI) {
477 bool NeedsSubtract = false;
478
479 // The 2nd instr must be an add-immediate or subtract-immediate.
480 switch (SecondMI.getOpcode()) {
481 case AArch64::SUBWri:
482 case AArch64::SUBXri:
483 NeedsSubtract = true;
484 [[fallthrough]];
485 case AArch64::ADDWri:
486 case AArch64::ADDXri:
487 break;
488
489 default:
490 return false;
491 }
492
493 // The immediate in the 2nd instr must be "1".
494 if (!SecondMI.getOperand(i: 2).isImm() || SecondMI.getOperand(i: 2).getImm() != 1) {
495 return false;
496 }
497
498 // Assume the 1st instr to be a wildcard if it is unspecified.
499 if (FirstMI == nullptr) {
500 return true;
501 }
502
503 switch (FirstMI->getOpcode()) {
504 case AArch64::SUBWrs:
505 case AArch64::SUBXrs:
506 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
507 return false;
508 [[fallthrough]];
509 case AArch64::SUBWrr:
510 case AArch64::SUBXrr:
511 if (NeedsSubtract) {
512 return true;
513 }
514 break;
515
516 case AArch64::ADDWrs:
517 case AArch64::ADDXrs:
518 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
519 return false;
520 [[fallthrough]];
521 case AArch64::ADDWrr:
522 case AArch64::ADDXrr:
523 if (!NeedsSubtract) {
524 return true;
525 }
526 break;
527 }
528
529 return false;
530}
531
532// Checks whether RegMO's class is Class, or a subclass of it. Supports both
533// physical and virtual registers.
534static bool isRegInClass(const MachineOperand &RegMO,
535 const TargetRegisterClass &Class,
536 const MachineRegisterInfo &MRI) {
537 Register Reg = RegMO.getReg();
538 if (Reg.isVirtual())
539 return Class.hasSubClassEq(RC: MRI.getRegClass(Reg));
540 return Class.contains(Reg: Reg.asMCReg());
541}
542
543// Checks whether MI defines any register whose class is any of Classes, or a
544// subclass of them. Supports both physical and virtual registers.
545static bool
546definesRegInClass(const MachineInstr &MI,
547 const ArrayRef<const TargetRegisterClass *> Classes) {
548 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
549 return llvm::any_of(Range: MI.all_defs(), P: [&](const MachineOperand &MO) {
550 return llvm::any_of(Range: Classes, P: [&](const TargetRegisterClass *Class) {
551 return isRegInClass(RegMO: MO, Class: *Class, MRI);
552 });
553 });
554}
555
556// Checks whether MI reads/uses any register whose class is any of Classes, or a
557// subclass of them. Supports both physical and virtual registers.
558static bool
559readsRegInClass(const MachineInstr &MI,
560 const ArrayRef<const TargetRegisterClass *> Classes) {
561 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
562 return llvm::any_of(Range: MI.all_uses(), P: [&](const MachineOperand &MO) {
563 return llvm::any_of(Range: Classes, P: [&](const TargetRegisterClass *Class) {
564 return isRegInClass(RegMO: MO, Class: *Class, MRI);
565 });
566 });
567}
568
569static bool isFusableAppleSMEComputeOp(const MachineInstr &MI,
570 const TargetInstrInfo &TII,
571 const TargetRegisterInfo *TRI) {
572 const bool ReadOrWriteZA = MI.readsRegister(Reg: AArch64::ZA, TRI) ||
573 MI.definesRegister(Reg: AArch64::ZA, TRI);
574
575 // (read/write ZA or read/write Z)
576 if (!ReadOrWriteZA &&
577 !definesRegInClass(MI, Classes: {&AArch64::ZPRRegClass, &AArch64::ZPR2RegClass,
578 &AArch64::ZPR3RegClass, &AArch64::ZPR4RegClass}))
579 return false;
580
581 // (NOT load/store)
582 if (MI.mayLoad() || MI.mayStore())
583 return false;
584
585 // (NOT write P)
586 if (definesRegInClass(MI, Classes: {&AArch64::PPRRegClass, &AArch64::PPR2RegClass,
587 &AArch64::PNRRegClass}))
588 return false;
589
590 // (NOT write GPR)
591 const bool WriteGPR =
592 definesRegInClass(MI, Classes: {&AArch64::GPR32RegClass, &AArch64::GPR64RegClass});
593 if (WriteGPR)
594 return false;
595
596 // (NOT read/write NZCV)
597 if (MI.readsRegister(Reg: AArch64::NZCV, TRI) ||
598 MI.definesRegister(Reg: AArch64::NZCV, TRI))
599 return false;
600
601 const bool ReadGPR =
602 readsRegInClass(MI, Classes: {&AArch64::GPR32RegClass, &AArch64::GPR64RegClass});
603
604 // ( (NOT read GPR) or read/write ZA )
605 if (ReadGPR && !ReadOrWriteZA)
606 return false;
607
608 return true;
609}
610
611static bool isAppleSMEComputePair(const MachineInstr *FirstMI,
612 const MachineInstr &SecondMI,
613 const TargetInstrInfo &TII,
614 const TargetRegisterInfo *TRI) {
615 if (!isFusableAppleSMEComputeOp(MI: SecondMI, TII, TRI))
616 return false;
617 // Assume the 1st instr to be a wildcard if it is unspecified.
618 if (FirstMI == nullptr)
619 return true;
620 if (isFusableAppleSMEComputeOp(MI: *FirstMI, TII, TRI))
621 return true;
622 return false;
623}
624
625namespace {
626enum class FMinFMaxKind { None, Scalar, Vector, ScalarNM, VectorNM };
627} // end anonymous namespace
628
629// Floating-point minimum or maximum, scalar (H/S/D) or vector (Vd), and their
630// number (NM) variants.
631static FMinFMaxKind getFMinFMaxKind(unsigned Opcode) {
632 switch (Opcode) {
633 case AArch64::FMAXHrr:
634 case AArch64::FMAXSrr:
635 case AArch64::FMAXDrr:
636 case AArch64::FMINHrr:
637 case AArch64::FMINSrr:
638 case AArch64::FMINDrr:
639 return FMinFMaxKind::Scalar;
640 case AArch64::FMAXv4f16:
641 case AArch64::FMAXv8f16:
642 case AArch64::FMAXv2f32:
643 case AArch64::FMAXv4f32:
644 case AArch64::FMAXv2f64:
645 case AArch64::FMINv4f16:
646 case AArch64::FMINv8f16:
647 case AArch64::FMINv2f32:
648 case AArch64::FMINv4f32:
649 case AArch64::FMINv2f64:
650 return FMinFMaxKind::Vector;
651 case AArch64::FMAXNMHrr:
652 case AArch64::FMAXNMSrr:
653 case AArch64::FMAXNMDrr:
654 case AArch64::FMINNMHrr:
655 case AArch64::FMINNMSrr:
656 case AArch64::FMINNMDrr:
657 return FMinFMaxKind::ScalarNM;
658 case AArch64::FMAXNMv4f16:
659 case AArch64::FMAXNMv8f16:
660 case AArch64::FMAXNMv2f32:
661 case AArch64::FMAXNMv4f32:
662 case AArch64::FMAXNMv2f64:
663 case AArch64::FMINNMv4f16:
664 case AArch64::FMINNMv8f16:
665 case AArch64::FMINNMv2f32:
666 case AArch64::FMINNMv4f32:
667 case AArch64::FMINNMv2f64:
668 return FMinFMaxKind::VectorNM;
669 }
670 return FMinFMaxKind::None;
671}
672
673// FMIN + FMAX, or FMINNM + FMAXNM, both scalar or both vector.
674static bool isFMinFMaxPair(const MachineInstr *FirstMI,
675 const MachineInstr &SecondMI) {
676 FMinFMaxKind SecondKind = getFMinFMaxKind(Opcode: SecondMI.getOpcode());
677 if (SecondKind == FMinFMaxKind::None)
678 return false;
679
680 // Assume the 1st instr to be a wildcard if it is unspecified.
681 if (FirstMI == nullptr)
682 return true;
683
684 if (getFMinFMaxKind(Opcode: FirstMI->getOpcode()) != SecondKind)
685 return false;
686
687 return mayDefSameReg(FirstMI: *FirstMI, SecondMI);
688}
689
690/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
691/// together. Given SecondMI, when FirstMI is unspecified, then check if
692/// SecondMI may be part of a fused pair at all.
693static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
694 const TargetSubtargetInfo &TSI,
695 const MachineInstr *FirstMI,
696 const MachineInstr &SecondMI,
697 const SDep *Dep) {
698 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
699 const TargetRegisterInfo *TRI = TSI.getRegisterInfo();
700
701 // All checking functions assume that the 1st instr is a wildcard if it is
702 // unspecified.
703
704 // FuseAppleSMECompute does not require a specific dependency kind
705 if (ST.hasFuseAppleSMECompute() &&
706 isAppleSMEComputePair(FirstMI, SecondMI, TII, TRI)) {
707 ++NumFusedAppleSMECompute;
708 return true;
709 }
710
711 // All the other fusions require RAW dependency
712 if (isNonDataDep(Dep))
713 return false;
714
715 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
716 bool CmpOnly = !ST.hasArithmeticBccFusion();
717 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly)) {
718 ++NumFusedArithmeticBcc;
719 return true;
720 }
721 }
722 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI)) {
723 ++NumFusedArithmeticCbz;
724 return true;
725 }
726 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI)) {
727 ++NumFusedAES;
728 return true;
729 }
730 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI)) {
731 ++NumFusedCryptoEOR;
732 return true;
733 }
734 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI)) {
735 ++NumFusedAdrpAdd;
736 return true;
737 }
738 if (ST.hasFuseLiterals() && ST.fusesMOVImmPair(FirstMI, SecondMI)) {
739 ++NumFusedLiterals;
740 return true;
741 }
742 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) {
743 ++NumFusedAddress;
744 return true;
745 }
746 if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI)) {
747 ++NumFusedCmpCSel;
748 return true;
749 }
750 if (ST.hasFuseFCmpFCSel() && isFCmpFCSelPair(FirstMI, SecondMI)) {
751 ++NumFusedFCmpFCSel;
752 return true;
753 }
754 if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI)) {
755 ++NumFusedCmpCSet;
756 return true;
757 }
758 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI)) {
759 ++NumFusedArithmeticLogic;
760 return true;
761 }
762 if (ST.hasFuseAddSub2RegAndConstOne() &&
763 isAddSub2RegAndConstOnePair(FirstMI, SecondMI)) {
764 ++NumFusedAddSub2RegAndConstOne;
765 return true;
766 }
767 if (ST.hasFuseFMinFMax() && isFMinFMaxPair(FirstMI, SecondMI)) {
768 ++NumFusedFMinFMax;
769 return true;
770 }
771
772 return false;
773}
774
775std::unique_ptr<ScheduleDAGMutation>
776llvm::createAArch64MacroFusionDAGMutation() {
777 return createMacroFusionDAGMutation(Predicates: shouldScheduleAdjacent);
778}
779