1//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AArch64 implementation of the DAG scheduling
10/// mutation to pair instructions back to back.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MacroFusion.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/CodeGen/MacroFusion.h"
18#include "llvm/CodeGen/TargetInstrInfo.h"
19
20#define DEBUG_TYPE "aarch64-macro-fusion"
21
22using namespace llvm;
23
24STATISTIC(NumFusedArithmeticBcc, "Number of arithmetic-Bcc fusions");
25STATISTIC(NumFusedArithmeticCbz, "Number of arithmetic-Cbz fusions");
26STATISTIC(NumFusedAES, "Number of AES fusions");
27STATISTIC(NumFusedCryptoEOR, "Number of crypto-EOR fusions");
28STATISTIC(NumFusedAdrpAdd, "Number of ADRP-ADD fusions");
29STATISTIC(NumFusedLiterals, "Number of literal-generation fusions");
30STATISTIC(NumFusedAddress, "Number of address-generation load/store fusions");
31STATISTIC(NumFusedCmpCSel, "Number of compare-CSEL fusions");
32STATISTIC(NumFusedFCmpFCSel, "Number of FP-compare-FCSEL fusions");
33STATISTIC(NumFusedCmpCSet, "Number of compare-CSET fusions");
34STATISTIC(NumFusedArithmeticLogic, "Number of arithmetic-logic fusions");
35STATISTIC(NumFusedAddSub2RegAndConstOne,
36 "Number of add/sub-two-register-and-constant-one fusions");
37STATISTIC(NumFusedAppleSMECompute, "Number of Apple SME compute fusions");
38STATISTIC(NumFusedFMinFMax, "Number of FMIN-FMAX fusions");
39
40/// CMN, CMP, TST followed by Bcc
41static bool isArithmeticBccPair(const MachineInstr *FirstMI,
42 const MachineInstr &SecondMI, bool CmpOnly) {
43 if (SecondMI.getOpcode() != AArch64::Bcc)
44 return false;
45
46 // Assume the 1st instr to be a wildcard if it is unspecified.
47 if (FirstMI == nullptr)
48 return true;
49
50 // If we're in CmpOnly mode, we only fuse arithmetic instructions that
51 // discard their result.
52 if (CmpOnly && FirstMI->getOperand(i: 0).isReg() &&
53 !(FirstMI->getOperand(i: 0).getReg() == AArch64::XZR ||
54 FirstMI->getOperand(i: 0).getReg() == AArch64::WZR)) {
55 return false;
56 }
57
58 switch (FirstMI->getOpcode()) {
59 case AArch64::ADDSWri:
60 case AArch64::ADDSWrr:
61 case AArch64::ADDSXri:
62 case AArch64::ADDSXrr:
63 case AArch64::ANDSWri:
64 case AArch64::ANDSWrr:
65 case AArch64::ANDSXri:
66 case AArch64::ANDSXrr:
67 case AArch64::SUBSWri:
68 case AArch64::SUBSWrr:
69 case AArch64::SUBSXri:
70 case AArch64::SUBSXrr:
71 case AArch64::BICSWrr:
72 case AArch64::BICSXrr:
73 return true;
74 case AArch64::ADDSWrs:
75 case AArch64::ADDSXrs:
76 case AArch64::ANDSWrs:
77 case AArch64::ANDSXrs:
78 case AArch64::SUBSWrs:
79 case AArch64::SUBSXrs:
80 case AArch64::BICSWrs:
81 case AArch64::BICSXrs:
82 // Shift value can be 0 making these behave like the "rr" variant...
83 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
84 }
85
86 return false;
87}
88
89/// ALU operations followed by CBZ/CBNZ.
90static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
91 const MachineInstr &SecondMI) {
92 if (SecondMI.getOpcode() != AArch64::CBZW &&
93 SecondMI.getOpcode() != AArch64::CBZX &&
94 SecondMI.getOpcode() != AArch64::CBNZW &&
95 SecondMI.getOpcode() != AArch64::CBNZX &&
96 SecondMI.getOpcode() != AArch64::TBZW &&
97 SecondMI.getOpcode() != AArch64::TBZX &&
98 SecondMI.getOpcode() != AArch64::TBNZW &&
99 SecondMI.getOpcode() != AArch64::TBNZX)
100 return false;
101
102 // Assume the 1st instr to be a wildcard if it is unspecified.
103 if (FirstMI == nullptr)
104 return true;
105
106 switch (FirstMI->getOpcode()) {
107 case AArch64::ADDWri:
108 case AArch64::ADDWrr:
109 case AArch64::ADDXri:
110 case AArch64::ADDXrr:
111 case AArch64::ANDWri:
112 case AArch64::ANDWrr:
113 case AArch64::ANDXri:
114 case AArch64::ANDXrr:
115 case AArch64::EORWri:
116 case AArch64::EORWrr:
117 case AArch64::EORXri:
118 case AArch64::EORXrr:
119 case AArch64::ORRWri:
120 case AArch64::ORRWrr:
121 case AArch64::ORRXri:
122 case AArch64::ORRXrr:
123 case AArch64::ORNWrr:
124 case AArch64::ORNXrr:
125 case AArch64::SUBWri:
126 case AArch64::SUBWrr:
127 case AArch64::SUBXri:
128 case AArch64::SUBXrr:
129 case AArch64::BICWrr:
130 case AArch64::BICXrr:
131 return true;
132 case AArch64::ADDWrs:
133 case AArch64::ADDXrs:
134 case AArch64::ANDWrs:
135 case AArch64::ANDXrs:
136 case AArch64::EORWrs:
137 case AArch64::EORXrs:
138 case AArch64::ORNWrs:
139 case AArch64::ORNXrs:
140 case AArch64::ORRWrs:
141 case AArch64::ORRXrs:
142 case AArch64::SUBWrs:
143 case AArch64::SUBXrs:
144 case AArch64::BICWrs:
145 case AArch64::BICXrs:
146 // Shift value can be 0 making these behave like the "rr" variant...
147 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
148 }
149
150 return false;
151}
152
153// True unless the pair provably writes non overlapping physical registers.
154// Pre-RA the dests are still virtual, and post-RA it requires a genuine WAW,
155// that is overlapping dest regs. Overlapping includes sub and super register
156// relations, e.g. W0 and X0, which matches the register unit based dependency
157// model of the scheduling DAG.
158static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
159 const MachineInstr &SecondMI,
160 const TargetRegisterInfo *TRI) {
161 Register DestFirst = FirstMI.getOperand(i: 0).getReg();
162 Register DestSecond = SecondMI.getOperand(i: 0).getReg();
163 if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
164 return true;
165 return TRI->regsOverlap(RegA: DestFirst, RegB: DestSecond);
166}
167
168/// AES crypto encoding or decoding.
169static bool isAESPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
170 const TargetRegisterInfo *TRI) {
171 // Assume the 1st instr to be a wildcard if it is unspecified.
172 unsigned SecondOpcode = SecondMI.getOpcode();
173 switch (SecondOpcode) {
174 // AES encode.
175 case AArch64::AESMCrr:
176 case AArch64::AESMCrrTied:
177 if (FirstMI == nullptr)
178 return true;
179 if (FirstMI->getOpcode() != AArch64::AESErr)
180 return false;
181 return SecondOpcode == AArch64::AESMCrrTied ||
182 mayHaveWAWDependency(FirstMI: *FirstMI, SecondMI, TRI);
183 // AES decode.
184 case AArch64::AESIMCrr:
185 case AArch64::AESIMCrrTied:
186 if (FirstMI == nullptr)
187 return true;
188 if (FirstMI->getOpcode() != AArch64::AESDrr)
189 return false;
190 return SecondOpcode == AArch64::AESIMCrrTied ||
191 mayHaveWAWDependency(FirstMI: *FirstMI, SecondMI, TRI);
192 }
193
194 return false;
195}
196
197/// AESE/AESD/PMULL + EOR.
198static bool isCryptoEORPair(const MachineInstr *FirstMI,
199 const MachineInstr &SecondMI) {
200 if (SecondMI.getOpcode() != AArch64::EORv16i8)
201 return false;
202
203 // Assume the 1st instr to be a wildcard if it is unspecified.
204 if (FirstMI == nullptr)
205 return true;
206
207 switch (FirstMI->getOpcode()) {
208 case AArch64::AESErr:
209 case AArch64::AESDrr:
210 case AArch64::PMULLv16i8:
211 case AArch64::PMULLv8i8:
212 case AArch64::PMULLv1i64:
213 case AArch64::PMULLv2i64:
214 return true;
215 }
216
217 return false;
218}
219
220static bool isAdrpAddPair(const MachineInstr *FirstMI,
221 const MachineInstr &SecondMI) {
222 // Assume the 1st instr to be a wildcard if it is unspecified.
223 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
224 SecondMI.getOpcode() == AArch64::ADDXri)
225 return true;
226 return false;
227}
228
229/// Fuse address generation and loads or stores.
230static bool isAddressLdStPair(const MachineInstr *FirstMI,
231 const MachineInstr &SecondMI) {
232 switch (SecondMI.getOpcode()) {
233 case AArch64::STRBBui:
234 case AArch64::STRBui:
235 case AArch64::STRDui:
236 case AArch64::STRHHui:
237 case AArch64::STRHui:
238 case AArch64::STRQui:
239 case AArch64::STRSui:
240 case AArch64::STRWui:
241 case AArch64::STRXui:
242 case AArch64::LDRBBui:
243 case AArch64::LDRBui:
244 case AArch64::LDRDui:
245 case AArch64::LDRHHui:
246 case AArch64::LDRHui:
247 case AArch64::LDRQui:
248 case AArch64::LDRSui:
249 case AArch64::LDRWui:
250 case AArch64::LDRXui:
251 case AArch64::LDRSBWui:
252 case AArch64::LDRSBXui:
253 case AArch64::LDRSHWui:
254 case AArch64::LDRSHXui:
255 case AArch64::LDRSWui:
256 // Assume the 1st instr to be a wildcard if it is unspecified.
257 if (FirstMI == nullptr)
258 return true;
259
260 switch (FirstMI->getOpcode()) {
261 case AArch64::ADR:
262 return SecondMI.getOperand(i: 2).getImm() == 0;
263 case AArch64::ADRP:
264 return true;
265 }
266 }
267
268 return false;
269}
270
271/// Compare and conditional select.
272static bool isCmpCSelPair(const MachineInstr *FirstMI,
273 const MachineInstr &SecondMI) {
274 // 32 bits
275 if (SecondMI.getOpcode() == AArch64::CSELWr) {
276 // Assume the 1st instr to be a wildcard if it is unspecified.
277 if (FirstMI == nullptr)
278 return true;
279
280 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr))
281 switch (FirstMI->getOpcode()) {
282 case AArch64::SUBSWrs:
283 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
284 case AArch64::SUBSWrx:
285 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
286 case AArch64::SUBSWrr:
287 case AArch64::SUBSWri:
288 return true;
289 }
290 }
291
292 // 64 bits
293 if (SecondMI.getOpcode() == AArch64::CSELXr) {
294 // Assume the 1st instr to be a wildcard if it is unspecified.
295 if (FirstMI == nullptr)
296 return true;
297
298 if (FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
299 switch (FirstMI->getOpcode()) {
300 case AArch64::SUBSXrs:
301 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
302 case AArch64::SUBSXrx:
303 case AArch64::SUBSXrx64:
304 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
305 case AArch64::SUBSXrr:
306 case AArch64::SUBSXri:
307 return true;
308 }
309 }
310
311 return false;
312}
313
314/// Floating-point compare and floating-point conditional select.
315static bool isFCmpFCSelPair(const MachineInstr *FirstMI,
316 const MachineInstr &SecondMI) {
317 switch (SecondMI.getOpcode()) {
318 case AArch64::FCSELSrrr:
319 case AArch64::FCSELDrrr:
320 case AArch64::FCSELHrrr:
321 break;
322 default:
323 return false;
324 }
325
326 // Assume the 1st instr to be a wildcard if it is unspecified.
327 if (FirstMI == nullptr)
328 return true;
329
330 switch (FirstMI->getOpcode()) {
331 case AArch64::FCMPSrr:
332 case AArch64::FCMPDrr:
333 case AArch64::FCMPESrr:
334 case AArch64::FCMPEDrr:
335 case AArch64::FCMPHrr:
336 case AArch64::FCMPEHrr:
337 return true;
338 default:
339 return false;
340 }
341}
342
343/// Compare and cset.
344static bool isCmpCSetPair(const MachineInstr *FirstMI,
345 const MachineInstr &SecondMI) {
346 if ((SecondMI.getOpcode() == AArch64::CSINCWr &&
347 SecondMI.getOperand(i: 1).getReg() == AArch64::WZR &&
348 SecondMI.getOperand(i: 2).getReg() == AArch64::WZR) ||
349 (SecondMI.getOpcode() == AArch64::CSINCXr &&
350 SecondMI.getOperand(i: 1).getReg() == AArch64::XZR &&
351 SecondMI.getOperand(i: 2).getReg() == AArch64::XZR)) {
352 // Assume the 1st instr to be a wildcard if it is unspecified.
353 if (FirstMI == nullptr)
354 return true;
355
356 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
357 FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
358 switch (FirstMI->getOpcode()) {
359 case AArch64::SUBSWrs:
360 case AArch64::SUBSXrs:
361 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
362 case AArch64::SUBSWrx:
363 case AArch64::SUBSXrx:
364 case AArch64::SUBSXrx64:
365 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
366 case AArch64::SUBSWri:
367 case AArch64::SUBSWrr:
368 case AArch64::SUBSXri:
369 case AArch64::SUBSXrr:
370 return true;
371 }
372 }
373
374 return false;
375}
376
377// Arithmetic and logic.
378static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
379 const MachineInstr &SecondMI) {
380 if (AArch64InstrInfo::hasShiftedReg(MI: SecondMI))
381 return false;
382
383 switch (SecondMI.getOpcode()) {
384 // Arithmetic
385 case AArch64::ADDWrr:
386 case AArch64::ADDXrr:
387 case AArch64::SUBWrr:
388 case AArch64::SUBXrr:
389 case AArch64::ADDWrs:
390 case AArch64::ADDXrs:
391 case AArch64::SUBWrs:
392 case AArch64::SUBXrs:
393 // Logic
394 case AArch64::ANDWrr:
395 case AArch64::ANDXrr:
396 case AArch64::BICWrr:
397 case AArch64::BICXrr:
398 case AArch64::EONWrr:
399 case AArch64::EONXrr:
400 case AArch64::EORWrr:
401 case AArch64::EORXrr:
402 case AArch64::ORNWrr:
403 case AArch64::ORNXrr:
404 case AArch64::ORRWrr:
405 case AArch64::ORRXrr:
406 case AArch64::ANDWrs:
407 case AArch64::ANDXrs:
408 case AArch64::BICWrs:
409 case AArch64::BICXrs:
410 case AArch64::EONWrs:
411 case AArch64::EONXrs:
412 case AArch64::EORWrs:
413 case AArch64::EORXrs:
414 case AArch64::ORNWrs:
415 case AArch64::ORNXrs:
416 case AArch64::ORRWrs:
417 case AArch64::ORRXrs:
418 // Assume the 1st instr to be a wildcard if it is unspecified.
419 if (FirstMI == nullptr)
420 return true;
421
422 // Arithmetic
423 switch (FirstMI->getOpcode()) {
424 case AArch64::ADDWrr:
425 case AArch64::ADDXrr:
426 case AArch64::ADDSWrr:
427 case AArch64::ADDSXrr:
428 case AArch64::SUBWrr:
429 case AArch64::SUBXrr:
430 case AArch64::SUBSWrr:
431 case AArch64::SUBSXrr:
432 return true;
433 case AArch64::ADDWrs:
434 case AArch64::ADDXrs:
435 case AArch64::ADDSWrs:
436 case AArch64::ADDSXrs:
437 case AArch64::SUBWrs:
438 case AArch64::SUBXrs:
439 case AArch64::SUBSWrs:
440 case AArch64::SUBSXrs:
441 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
442 }
443 break;
444
445 // Arithmetic, setting flags.
446 case AArch64::ADDSWrr:
447 case AArch64::ADDSXrr:
448 case AArch64::SUBSWrr:
449 case AArch64::SUBSXrr:
450 case AArch64::ADDSWrs:
451 case AArch64::ADDSXrs:
452 case AArch64::SUBSWrs:
453 case AArch64::SUBSXrs:
454 // Assume the 1st instr to be a wildcard if it is unspecified.
455 if (FirstMI == nullptr)
456 return true;
457
458 // Arithmetic, not setting flags.
459 switch (FirstMI->getOpcode()) {
460 case AArch64::ADDWrr:
461 case AArch64::ADDXrr:
462 case AArch64::SUBWrr:
463 case AArch64::SUBXrr:
464 return true;
465 case AArch64::ADDWrs:
466 case AArch64::ADDXrs:
467 case AArch64::SUBWrs:
468 case AArch64::SUBXrs:
469 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
470 }
471 break;
472 }
473
474 return false;
475}
476
477// "(A + B) + 1" or "(A - B) - 1"
478static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
479 const MachineInstr &SecondMI) {
480 bool NeedsSubtract = false;
481
482 // The 2nd instr must be an add-immediate or subtract-immediate.
483 switch (SecondMI.getOpcode()) {
484 case AArch64::SUBWri:
485 case AArch64::SUBXri:
486 NeedsSubtract = true;
487 [[fallthrough]];
488 case AArch64::ADDWri:
489 case AArch64::ADDXri:
490 break;
491
492 default:
493 return false;
494 }
495
496 // The immediate in the 2nd instr must be "1".
497 if (!SecondMI.getOperand(i: 2).isImm() || SecondMI.getOperand(i: 2).getImm() != 1) {
498 return false;
499 }
500
501 // Assume the 1st instr to be a wildcard if it is unspecified.
502 if (FirstMI == nullptr) {
503 return true;
504 }
505
506 switch (FirstMI->getOpcode()) {
507 case AArch64::SUBWrs:
508 case AArch64::SUBXrs:
509 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
510 return false;
511 [[fallthrough]];
512 case AArch64::SUBWrr:
513 case AArch64::SUBXrr:
514 if (NeedsSubtract) {
515 return true;
516 }
517 break;
518
519 case AArch64::ADDWrs:
520 case AArch64::ADDXrs:
521 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
522 return false;
523 [[fallthrough]];
524 case AArch64::ADDWrr:
525 case AArch64::ADDXrr:
526 if (!NeedsSubtract) {
527 return true;
528 }
529 break;
530 }
531
532 return false;
533}
534
535static bool definesRegInClass(const MachineInstr &MI,
536 const TargetRegisterInfo *TRI,
537 const TargetRegisterClass &Class) {
538 return llvm::any_of(Range: Class, P: [&MI, TRI](MCPhysReg Reg) {
539 return MI.definesRegister(Reg, TRI);
540 });
541}
542
543static bool readsRegInClass(const MachineInstr &MI,
544 const TargetRegisterInfo *TRI,
545 const TargetRegisterClass &Class) {
546 return llvm::any_of(
547 Range: Class, P: [&MI, TRI](MCPhysReg Reg) { return MI.readsRegister(Reg, TRI); });
548}
549
550static bool isFusableAppleSMEComputeOp(const MachineInstr &MI,
551 const TargetInstrInfo &TII,
552 const TargetRegisterInfo *TRI) {
553 const bool ReadOrWriteZA = MI.readsRegister(Reg: AArch64::ZA, TRI) ||
554 MI.definesRegister(Reg: AArch64::ZA, TRI);
555
556 // (read/write ZA or read/write Z)
557 if (!ReadOrWriteZA && !definesRegInClass(MI, TRI, Class: AArch64::ZPRRegClass))
558 return false;
559
560 // (NOT load/store)
561 if (MI.mayLoad() || MI.mayStore())
562 return false;
563
564 // (NOT write P)
565 if (definesRegInClass(MI, TRI, Class: AArch64::PPRRegClass))
566 return false;
567
568 // (NOT write GPR)
569 const bool WriteGPR = definesRegInClass(MI, TRI, Class: AArch64::GPR32RegClass) ||
570 definesRegInClass(MI, TRI, Class: AArch64::GPR64RegClass);
571 if (WriteGPR)
572 return false;
573
574 // (NOT read/write NZCV)
575 if (MI.readsRegister(Reg: AArch64::NZCV, TRI) ||
576 MI.definesRegister(Reg: AArch64::NZCV, TRI))
577 return false;
578
579 const bool ReadGPR = readsRegInClass(MI, TRI, Class: AArch64::GPR32RegClass) &&
580 readsRegInClass(MI, TRI, Class: AArch64::GPR64RegClass);
581
582 // ( (NOT read GPR) or read/write ZA )
583 if (ReadGPR && !ReadOrWriteZA)
584 return false;
585
586 return true;
587}
588
589static bool isAppleSMEComputePair(const MachineInstr *FirstMI,
590 const MachineInstr &SecondMI,
591 const TargetInstrInfo &TII,
592 const TargetRegisterInfo *TRI) {
593 if (!isFusableAppleSMEComputeOp(MI: SecondMI, TII, TRI))
594 return false;
595 // Assume the 1st instr to be a wildcard if it is unspecified.
596 if (FirstMI == nullptr)
597 return true;
598 if (isFusableAppleSMEComputeOp(MI: *FirstMI, TII, TRI))
599 return true;
600 return false;
601}
602
603// Floating-point minimum or maximum, scalar (H/S/D) or vector (Vd).
604static bool isFMinFMax(unsigned Opcode) {
605 switch (Opcode) {
606 // Scalar.
607 case AArch64::FMAXHrr:
608 case AArch64::FMAXSrr:
609 case AArch64::FMAXDrr:
610 case AArch64::FMINHrr:
611 case AArch64::FMINSrr:
612 case AArch64::FMINDrr:
613 // Vector.
614 case AArch64::FMAXv4f16:
615 case AArch64::FMAXv8f16:
616 case AArch64::FMAXv2f32:
617 case AArch64::FMAXv4f32:
618 case AArch64::FMAXv2f64:
619 case AArch64::FMINv4f16:
620 case AArch64::FMINv8f16:
621 case AArch64::FMINv2f32:
622 case AArch64::FMINv4f32:
623 case AArch64::FMINv2f64:
624 return true;
625 }
626 return false;
627}
628
629// FMIN + FMAX.
630static bool isFMinFMaxPair(const MachineInstr *FirstMI,
631 const MachineInstr &SecondMI,
632 const TargetRegisterInfo *TRI) {
633 if (!isFMinFMax(Opcode: SecondMI.getOpcode()))
634 return false;
635
636 // Assume the 1st instr to be a wildcard if it is unspecified.
637 if (FirstMI == nullptr)
638 return true;
639
640 if (!isFMinFMax(Opcode: FirstMI->getOpcode()))
641 return false;
642
643 return mayHaveWAWDependency(FirstMI: *FirstMI, SecondMI, TRI);
644}
645
646/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
647/// together. Given SecondMI, when FirstMI is unspecified, then check if
648/// SecondMI may be part of a fused pair at all.
649static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
650 const TargetSubtargetInfo &TSI,
651 const MachineInstr *FirstMI,
652 const MachineInstr &SecondMI,
653 const SDep *Dep) {
654 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
655 const TargetRegisterInfo *TRI = TSI.getRegisterInfo();
656
657 // All checking functions assume that the 1st instr is a wildcard if it is
658 // unspecified.
659
660 // FuseAppleSMECompute does not require a specific dependency kind
661 if (ST.hasFuseAppleSMECompute() &&
662 isAppleSMEComputePair(FirstMI, SecondMI, TII, TRI)) {
663 ++NumFusedAppleSMECompute;
664 return true;
665 }
666
667 // All the other fusions require RAW dependency
668 if (isNonDataDep(Dep))
669 return false;
670
671 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
672 bool CmpOnly = !ST.hasArithmeticBccFusion();
673 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly)) {
674 ++NumFusedArithmeticBcc;
675 return true;
676 }
677 }
678 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI)) {
679 ++NumFusedArithmeticCbz;
680 return true;
681 }
682 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI, TRI)) {
683 ++NumFusedAES;
684 return true;
685 }
686 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI)) {
687 ++NumFusedCryptoEOR;
688 return true;
689 }
690 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI)) {
691 ++NumFusedAdrpAdd;
692 return true;
693 }
694 if (ST.hasFuseLiterals() && ST.fusesMOVImmPair(FirstMI, SecondMI)) {
695 ++NumFusedLiterals;
696 return true;
697 }
698 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) {
699 ++NumFusedAddress;
700 return true;
701 }
702 if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI)) {
703 ++NumFusedCmpCSel;
704 return true;
705 }
706 if (ST.hasFuseFCmpFCSel() && isFCmpFCSelPair(FirstMI, SecondMI)) {
707 ++NumFusedFCmpFCSel;
708 return true;
709 }
710 if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI)) {
711 ++NumFusedCmpCSet;
712 return true;
713 }
714 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI)) {
715 ++NumFusedArithmeticLogic;
716 return true;
717 }
718 if (ST.hasFuseAddSub2RegAndConstOne() &&
719 isAddSub2RegAndConstOnePair(FirstMI, SecondMI)) {
720 ++NumFusedAddSub2RegAndConstOne;
721 return true;
722 }
723 if (ST.hasFuseFMinFMax() && isFMinFMaxPair(FirstMI, SecondMI, TRI)) {
724 ++NumFusedFMinFMax;
725 return true;
726 }
727
728 return false;
729}
730
731std::unique_ptr<ScheduleDAGMutation>
732llvm::createAArch64MacroFusionDAGMutation() {
733 return createMacroFusionDAGMutation(Predicates: shouldScheduleAdjacent);
734}
735