1//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AArch64 implementation of the DAG scheduling
10/// mutation to pair instructions back to back.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64MacroFusion.h"
15#include "AArch64Subtarget.h"
16#include "llvm/CodeGen/MacroFusion.h"
17#include "llvm/CodeGen/TargetInstrInfo.h"
18
19using namespace llvm;
20
21/// CMN, CMP, TST followed by Bcc
22static bool isArithmeticBccPair(const MachineInstr *FirstMI,
23 const MachineInstr &SecondMI, bool CmpOnly) {
24 if (SecondMI.getOpcode() != AArch64::Bcc)
25 return false;
26
27 // Assume the 1st instr to be a wildcard if it is unspecified.
28 if (FirstMI == nullptr)
29 return true;
30
31 // If we're in CmpOnly mode, we only fuse arithmetic instructions that
32 // discard their result.
33 if (CmpOnly && FirstMI->getOperand(i: 0).isReg() &&
34 !(FirstMI->getOperand(i: 0).getReg() == AArch64::XZR ||
35 FirstMI->getOperand(i: 0).getReg() == AArch64::WZR)) {
36 return false;
37 }
38
39 switch (FirstMI->getOpcode()) {
40 case AArch64::ADDSWri:
41 case AArch64::ADDSWrr:
42 case AArch64::ADDSXri:
43 case AArch64::ADDSXrr:
44 case AArch64::ANDSWri:
45 case AArch64::ANDSWrr:
46 case AArch64::ANDSXri:
47 case AArch64::ANDSXrr:
48 case AArch64::SUBSWri:
49 case AArch64::SUBSWrr:
50 case AArch64::SUBSXri:
51 case AArch64::SUBSXrr:
52 case AArch64::BICSWrr:
53 case AArch64::BICSXrr:
54 return true;
55 case AArch64::ADDSWrs:
56 case AArch64::ADDSXrs:
57 case AArch64::ANDSWrs:
58 case AArch64::ANDSXrs:
59 case AArch64::SUBSWrs:
60 case AArch64::SUBSXrs:
61 case AArch64::BICSWrs:
62 case AArch64::BICSXrs:
63 // Shift value can be 0 making these behave like the "rr" variant...
64 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
65 }
66
67 return false;
68}
69
70/// ALU operations followed by CBZ/CBNZ.
71static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
72 const MachineInstr &SecondMI) {
73 if (SecondMI.getOpcode() != AArch64::CBZW &&
74 SecondMI.getOpcode() != AArch64::CBZX &&
75 SecondMI.getOpcode() != AArch64::CBNZW &&
76 SecondMI.getOpcode() != AArch64::CBNZX &&
77 SecondMI.getOpcode() != AArch64::TBZW &&
78 SecondMI.getOpcode() != AArch64::TBZX &&
79 SecondMI.getOpcode() != AArch64::TBNZW &&
80 SecondMI.getOpcode() != AArch64::TBNZX)
81 return false;
82
83 // Assume the 1st instr to be a wildcard if it is unspecified.
84 if (FirstMI == nullptr)
85 return true;
86
87 switch (FirstMI->getOpcode()) {
88 case AArch64::ADDWri:
89 case AArch64::ADDWrr:
90 case AArch64::ADDXri:
91 case AArch64::ADDXrr:
92 case AArch64::ANDWri:
93 case AArch64::ANDWrr:
94 case AArch64::ANDXri:
95 case AArch64::ANDXrr:
96 case AArch64::EORWri:
97 case AArch64::EORWrr:
98 case AArch64::EORXri:
99 case AArch64::EORXrr:
100 case AArch64::ORRWri:
101 case AArch64::ORRWrr:
102 case AArch64::ORRXri:
103 case AArch64::ORRXrr:
104 case AArch64::ORNWrr:
105 case AArch64::ORNXrr:
106 case AArch64::SUBWri:
107 case AArch64::SUBWrr:
108 case AArch64::SUBXri:
109 case AArch64::SUBXrr:
110 case AArch64::BICWrr:
111 case AArch64::BICXrr:
112 return true;
113 case AArch64::ADDWrs:
114 case AArch64::ADDXrs:
115 case AArch64::ANDWrs:
116 case AArch64::ANDXrs:
117 case AArch64::EORWrs:
118 case AArch64::EORXrs:
119 case AArch64::ORNWrs:
120 case AArch64::ORNXrs:
121 case AArch64::ORRWrs:
122 case AArch64::ORRXrs:
123 case AArch64::SUBWrs:
124 case AArch64::SUBXrs:
125 case AArch64::BICWrs:
126 case AArch64::BICXrs:
127 // Shift value can be 0 making these behave like the "rr" variant...
128 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
129 }
130
131 return false;
132}
133
134// True unless the pair provably writes different physical registers. Pre-RA
135// the dests are still virtual, and post-RA it requires a genuine WAW (same dest
136// reg).
137static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
138 const MachineInstr &SecondMI) {
139 Register DestFirst = FirstMI.getOperand(i: 0).getReg();
140 Register DestSecond = SecondMI.getOperand(i: 0).getReg();
141 if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
142 return true;
143 return DestFirst == DestSecond;
144}
145
146/// AES crypto encoding or decoding.
147static bool isAESPair(const MachineInstr *FirstMI,
148 const MachineInstr &SecondMI) {
149 // Assume the 1st instr to be a wildcard if it is unspecified.
150 unsigned SecondOpcode = SecondMI.getOpcode();
151 switch (SecondOpcode) {
152 // AES encode.
153 case AArch64::AESMCrr:
154 case AArch64::AESMCrrTied:
155 if (FirstMI == nullptr)
156 return true;
157 if (FirstMI->getOpcode() != AArch64::AESErr)
158 return false;
159 return SecondOpcode == AArch64::AESMCrrTied ||
160 mayHaveWAWDependency(FirstMI: *FirstMI, SecondMI);
161 // AES decode.
162 case AArch64::AESIMCrr:
163 case AArch64::AESIMCrrTied:
164 if (FirstMI == nullptr)
165 return true;
166 if (FirstMI->getOpcode() != AArch64::AESDrr)
167 return false;
168 return SecondOpcode == AArch64::AESIMCrrTied ||
169 mayHaveWAWDependency(FirstMI: *FirstMI, SecondMI);
170 }
171
172 return false;
173}
174
175/// AESE/AESD/PMULL + EOR.
176static bool isCryptoEORPair(const MachineInstr *FirstMI,
177 const MachineInstr &SecondMI) {
178 if (SecondMI.getOpcode() != AArch64::EORv16i8)
179 return false;
180
181 // Assume the 1st instr to be a wildcard if it is unspecified.
182 if (FirstMI == nullptr)
183 return true;
184
185 switch (FirstMI->getOpcode()) {
186 case AArch64::AESErr:
187 case AArch64::AESDrr:
188 case AArch64::PMULLv16i8:
189 case AArch64::PMULLv8i8:
190 case AArch64::PMULLv1i64:
191 case AArch64::PMULLv2i64:
192 return true;
193 }
194
195 return false;
196}
197
198static bool isAdrpAddPair(const MachineInstr *FirstMI,
199 const MachineInstr &SecondMI) {
200 // Assume the 1st instr to be a wildcard if it is unspecified.
201 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
202 SecondMI.getOpcode() == AArch64::ADDXri)
203 return true;
204 return false;
205}
206
207/// Literal generation.
208static bool isLiteralsPair(const MachineInstr *FirstMI,
209 const MachineInstr &SecondMI) {
210 // Assume the 1st instr to be a wildcard if it is unspecified.
211 // 32 bit immediate.
212 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
213 (SecondMI.getOpcode() == AArch64::MOVKWi &&
214 SecondMI.getOperand(i: 3).getImm() == 16))
215 return true;
216
217 // Lower half of 64 bit immediate.
218 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
219 (SecondMI.getOpcode() == AArch64::MOVKXi &&
220 SecondMI.getOperand(i: 3).getImm() == 16))
221 return true;
222
223 // Upper half of 64 bit immediate.
224 if ((FirstMI == nullptr ||
225 (FirstMI->getOpcode() == AArch64::MOVKXi &&
226 FirstMI->getOperand(i: 3).getImm() == 32)) &&
227 (SecondMI.getOpcode() == AArch64::MOVKXi &&
228 SecondMI.getOperand(i: 3).getImm() == 48))
229 return true;
230
231 return false;
232}
233
234/// Fuse address generation and loads or stores.
235static bool isAddressLdStPair(const MachineInstr *FirstMI,
236 const MachineInstr &SecondMI) {
237 switch (SecondMI.getOpcode()) {
238 case AArch64::STRBBui:
239 case AArch64::STRBui:
240 case AArch64::STRDui:
241 case AArch64::STRHHui:
242 case AArch64::STRHui:
243 case AArch64::STRQui:
244 case AArch64::STRSui:
245 case AArch64::STRWui:
246 case AArch64::STRXui:
247 case AArch64::LDRBBui:
248 case AArch64::LDRBui:
249 case AArch64::LDRDui:
250 case AArch64::LDRHHui:
251 case AArch64::LDRHui:
252 case AArch64::LDRQui:
253 case AArch64::LDRSui:
254 case AArch64::LDRWui:
255 case AArch64::LDRXui:
256 case AArch64::LDRSBWui:
257 case AArch64::LDRSBXui:
258 case AArch64::LDRSHWui:
259 case AArch64::LDRSHXui:
260 case AArch64::LDRSWui:
261 // Assume the 1st instr to be a wildcard if it is unspecified.
262 if (FirstMI == nullptr)
263 return true;
264
265 switch (FirstMI->getOpcode()) {
266 case AArch64::ADR:
267 return SecondMI.getOperand(i: 2).getImm() == 0;
268 case AArch64::ADRP:
269 return true;
270 }
271 }
272
273 return false;
274}
275
276/// Compare and conditional select.
277static bool isCmpCSelPair(const MachineInstr *FirstMI,
278 const MachineInstr &SecondMI) {
279 // 32 bits
280 if (SecondMI.getOpcode() == AArch64::CSELWr) {
281 // Assume the 1st instr to be a wildcard if it is unspecified.
282 if (FirstMI == nullptr)
283 return true;
284
285 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr))
286 switch (FirstMI->getOpcode()) {
287 case AArch64::SUBSWrs:
288 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
289 case AArch64::SUBSWrx:
290 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
291 case AArch64::SUBSWrr:
292 case AArch64::SUBSWri:
293 return true;
294 }
295 }
296
297 // 64 bits
298 if (SecondMI.getOpcode() == AArch64::CSELXr) {
299 // Assume the 1st instr to be a wildcard if it is unspecified.
300 if (FirstMI == nullptr)
301 return true;
302
303 if (FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
304 switch (FirstMI->getOpcode()) {
305 case AArch64::SUBSXrs:
306 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
307 case AArch64::SUBSXrx:
308 case AArch64::SUBSXrx64:
309 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
310 case AArch64::SUBSXrr:
311 case AArch64::SUBSXri:
312 return true;
313 }
314 }
315
316 return false;
317}
318
319/// Floating-point compare and floating-point conditional select.
320static bool isFCmpFCSelPair(const MachineInstr *FirstMI,
321 const MachineInstr &SecondMI) {
322 switch (SecondMI.getOpcode()) {
323 case AArch64::FCSELSrrr:
324 case AArch64::FCSELDrrr:
325 case AArch64::FCSELHrrr:
326 break;
327 default:
328 return false;
329 }
330
331 // Assume the 1st instr to be a wildcard if it is unspecified.
332 if (FirstMI == nullptr)
333 return true;
334
335 switch (FirstMI->getOpcode()) {
336 case AArch64::FCMPSrr:
337 case AArch64::FCMPDrr:
338 case AArch64::FCMPESrr:
339 case AArch64::FCMPEDrr:
340 case AArch64::FCMPHrr:
341 case AArch64::FCMPEHrr:
342 return true;
343 default:
344 return false;
345 }
346}
347
348/// Compare and cset.
349static bool isCmpCSetPair(const MachineInstr *FirstMI,
350 const MachineInstr &SecondMI) {
351 if ((SecondMI.getOpcode() == AArch64::CSINCWr &&
352 SecondMI.getOperand(i: 1).getReg() == AArch64::WZR &&
353 SecondMI.getOperand(i: 2).getReg() == AArch64::WZR) ||
354 (SecondMI.getOpcode() == AArch64::CSINCXr &&
355 SecondMI.getOperand(i: 1).getReg() == AArch64::XZR &&
356 SecondMI.getOperand(i: 2).getReg() == AArch64::XZR)) {
357 // Assume the 1st instr to be a wildcard if it is unspecified.
358 if (FirstMI == nullptr)
359 return true;
360
361 if (FirstMI->definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
362 FirstMI->definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
363 switch (FirstMI->getOpcode()) {
364 case AArch64::SUBSWrs:
365 case AArch64::SUBSXrs:
366 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
367 case AArch64::SUBSWrx:
368 case AArch64::SUBSXrx:
369 case AArch64::SUBSXrx64:
370 return !AArch64InstrInfo::hasExtendedReg(MI: *FirstMI);
371 case AArch64::SUBSWri:
372 case AArch64::SUBSWrr:
373 case AArch64::SUBSXri:
374 case AArch64::SUBSXrr:
375 return true;
376 }
377 }
378
379 return false;
380}
381
382// Arithmetic and logic.
383static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
384 const MachineInstr &SecondMI) {
385 if (AArch64InstrInfo::hasShiftedReg(MI: SecondMI))
386 return false;
387
388 switch (SecondMI.getOpcode()) {
389 // Arithmetic
390 case AArch64::ADDWrr:
391 case AArch64::ADDXrr:
392 case AArch64::SUBWrr:
393 case AArch64::SUBXrr:
394 case AArch64::ADDWrs:
395 case AArch64::ADDXrs:
396 case AArch64::SUBWrs:
397 case AArch64::SUBXrs:
398 // Logic
399 case AArch64::ANDWrr:
400 case AArch64::ANDXrr:
401 case AArch64::BICWrr:
402 case AArch64::BICXrr:
403 case AArch64::EONWrr:
404 case AArch64::EONXrr:
405 case AArch64::EORWrr:
406 case AArch64::EORXrr:
407 case AArch64::ORNWrr:
408 case AArch64::ORNXrr:
409 case AArch64::ORRWrr:
410 case AArch64::ORRXrr:
411 case AArch64::ANDWrs:
412 case AArch64::ANDXrs:
413 case AArch64::BICWrs:
414 case AArch64::BICXrs:
415 case AArch64::EONWrs:
416 case AArch64::EONXrs:
417 case AArch64::EORWrs:
418 case AArch64::EORXrs:
419 case AArch64::ORNWrs:
420 case AArch64::ORNXrs:
421 case AArch64::ORRWrs:
422 case AArch64::ORRXrs:
423 // Assume the 1st instr to be a wildcard if it is unspecified.
424 if (FirstMI == nullptr)
425 return true;
426
427 // Arithmetic
428 switch (FirstMI->getOpcode()) {
429 case AArch64::ADDWrr:
430 case AArch64::ADDXrr:
431 case AArch64::ADDSWrr:
432 case AArch64::ADDSXrr:
433 case AArch64::SUBWrr:
434 case AArch64::SUBXrr:
435 case AArch64::SUBSWrr:
436 case AArch64::SUBSXrr:
437 return true;
438 case AArch64::ADDWrs:
439 case AArch64::ADDXrs:
440 case AArch64::ADDSWrs:
441 case AArch64::ADDSXrs:
442 case AArch64::SUBWrs:
443 case AArch64::SUBXrs:
444 case AArch64::SUBSWrs:
445 case AArch64::SUBSXrs:
446 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
447 }
448 break;
449
450 // Arithmetic, setting flags.
451 case AArch64::ADDSWrr:
452 case AArch64::ADDSXrr:
453 case AArch64::SUBSWrr:
454 case AArch64::SUBSXrr:
455 case AArch64::ADDSWrs:
456 case AArch64::ADDSXrs:
457 case AArch64::SUBSWrs:
458 case AArch64::SUBSXrs:
459 // Assume the 1st instr to be a wildcard if it is unspecified.
460 if (FirstMI == nullptr)
461 return true;
462
463 // Arithmetic, not setting flags.
464 switch (FirstMI->getOpcode()) {
465 case AArch64::ADDWrr:
466 case AArch64::ADDXrr:
467 case AArch64::SUBWrr:
468 case AArch64::SUBXrr:
469 return true;
470 case AArch64::ADDWrs:
471 case AArch64::ADDXrs:
472 case AArch64::SUBWrs:
473 case AArch64::SUBXrs:
474 return !AArch64InstrInfo::hasShiftedReg(MI: *FirstMI);
475 }
476 break;
477 }
478
479 return false;
480}
481
482// "(A + B) + 1" or "(A - B) - 1"
483static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
484 const MachineInstr &SecondMI) {
485 bool NeedsSubtract = false;
486
487 // The 2nd instr must be an add-immediate or subtract-immediate.
488 switch (SecondMI.getOpcode()) {
489 case AArch64::SUBWri:
490 case AArch64::SUBXri:
491 NeedsSubtract = true;
492 [[fallthrough]];
493 case AArch64::ADDWri:
494 case AArch64::ADDXri:
495 break;
496
497 default:
498 return false;
499 }
500
501 // The immediate in the 2nd instr must be "1".
502 if (!SecondMI.getOperand(i: 2).isImm() || SecondMI.getOperand(i: 2).getImm() != 1) {
503 return false;
504 }
505
506 // Assume the 1st instr to be a wildcard if it is unspecified.
507 if (FirstMI == nullptr) {
508 return true;
509 }
510
511 switch (FirstMI->getOpcode()) {
512 case AArch64::SUBWrs:
513 case AArch64::SUBXrs:
514 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
515 return false;
516 [[fallthrough]];
517 case AArch64::SUBWrr:
518 case AArch64::SUBXrr:
519 if (NeedsSubtract) {
520 return true;
521 }
522 break;
523
524 case AArch64::ADDWrs:
525 case AArch64::ADDXrs:
526 if (AArch64InstrInfo::hasShiftedReg(MI: *FirstMI))
527 return false;
528 [[fallthrough]];
529 case AArch64::ADDWrr:
530 case AArch64::ADDXrr:
531 if (!NeedsSubtract) {
532 return true;
533 }
534 break;
535 }
536
537 return false;
538}
539
540/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
541/// together. Given SecondMI, when FirstMI is unspecified, then check if
542/// SecondMI may be part of a fused pair at all.
543static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
544 const TargetSubtargetInfo &TSI,
545 const MachineInstr *FirstMI,
546 const MachineInstr &SecondMI) {
547 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
548
549 // All checking functions assume that the 1st instr is a wildcard if it is
550 // unspecified.
551 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
552 bool CmpOnly = !ST.hasArithmeticBccFusion();
553 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
554 return true;
555 }
556 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
557 return true;
558 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
559 return true;
560 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
561 return true;
562 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))
563 return true;
564 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
565 return true;
566 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
567 return true;
568 if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI))
569 return true;
570 if (ST.hasFuseFCmpFCSel() && isFCmpFCSelPair(FirstMI, SecondMI))
571 return true;
572 if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI))
573 return true;
574 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
575 return true;
576 if (ST.hasFuseAddSub2RegAndConstOne() &&
577 isAddSub2RegAndConstOnePair(FirstMI, SecondMI))
578 return true;
579
580 return false;
581}
582
583std::unique_ptr<ScheduleDAGMutation>
584llvm::createAArch64MacroFusionDAGMutation() {
585 return createMacroFusionDAGMutation(Predicates: shouldScheduleAdjacent);
586}
587