1//===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64RegisterBankInfo.h"
15#include "AArch64RegisterInfo.h"
16#include "AArch64Subtarget.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "MCTargetDesc/AArch64MCTargetDesc.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
24#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25#include "llvm/CodeGen/GlobalISel/Utils.h"
26#include "llvm/CodeGen/LowLevelTypeUtils.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachineOperand.h"
30#include "llvm/CodeGen/MachineRegisterInfo.h"
31#include "llvm/CodeGen/MachineSizeOpts.h"
32#include "llvm/CodeGen/RegisterBank.h"
33#include "llvm/CodeGen/RegisterBankInfo.h"
34#include "llvm/CodeGen/TargetOpcodes.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/CodeGen/TargetSubtargetInfo.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/IntrinsicsAArch64.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/Threading.h"
41#include <cassert>
42
43#define GET_TARGET_REGBANK_IMPL
44#include "AArch64GenRegisterBank.inc"
45
46// This file will be TableGen'ed at some point.
47#include "AArch64GenRegisterBankInfo.def"
48
49using namespace llvm;
50using namespace MIPatternMatch;
51static const unsigned CustomMappingID = 1;
52
53AArch64RegisterBankInfo::AArch64RegisterBankInfo(
54 const TargetRegisterInfo &TRI) {
55 static llvm::once_flag InitializeRegisterBankFlag;
56
57 static auto InitializeRegisterBankOnce = [&]() {
58 // We have only one set of register banks, whatever the subtarget
59 // is. Therefore, the initialization of the RegBanks table should be
60 // done only once. Indeed the table of all register banks
61 // (AArch64::RegBanks) is unique in the compiler. At some point, it
62 // will get tablegen'ed and the whole constructor becomes empty.
63
64 const RegisterBank &RBGPR = getRegBank(ID: AArch64::GPRRegBankID);
65 (void)RBGPR;
66 assert(&AArch64::GPRRegBank == &RBGPR &&
67 "The order in RegBanks is messed up");
68
69 const RegisterBank &RBFPR = getRegBank(ID: AArch64::FPRRegBankID);
70 (void)RBFPR;
71 assert(&AArch64::FPRRegBank == &RBFPR &&
72 "The order in RegBanks is messed up");
73
74 const RegisterBank &RBCCR = getRegBank(ID: AArch64::CCRegBankID);
75 (void)RBCCR;
76 assert(&AArch64::CCRegBank == &RBCCR &&
77 "The order in RegBanks is messed up");
78
79 // The GPR register bank is fully defined by all the registers in
80 // GR64all + its subclasses.
81 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
82 "Subclass not added?");
83 assert(getMaximumSize(RBGPR.getID()) == 128 &&
84 "GPRs should hold up to 128-bit");
85
86 // The FPR register bank is fully defined by all the registers in
87 // GR64all + its subclasses.
88 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
89 "Subclass not added?");
90 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
91 "Subclass not added?");
92 assert(getMaximumSize(RBFPR.getID()) == 512 &&
93 "FPRs should hold up to 512-bit via QQQQ sequence");
94
95 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
96 "Class not added?");
97 assert(getMaximumSize(RBCCR.getID()) == 32 &&
98 "CCR should hold up to 32-bit");
99
100 // Check that the TableGen'ed like file is in sync we our expectations.
101 // First, the Idx.
102 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
103 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) &&
104 "PartialMappingIdx's are incorrectly ordered");
105 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
106 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
107 PMI_FPR256, PMI_FPR512}) &&
108 "PartialMappingIdx's are incorrectly ordered");
109// Now, the content.
110// Check partial mapping.
111#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
112 do { \
113 assert( \
114 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
115 #Idx " is incorrectly initialized"); \
116 } while (false)
117
118 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
119 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
120 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);
121 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
122 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
123 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
124 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
125 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
126 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
127
128// Check value mapping.
129#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
130 do { \
131 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
132 PartialMappingIdx::PMI_First##RBName, Size, \
133 Offset) && \
134 #RBName #Size " " #Offset " is incorrectly initialized"); \
135 } while (false)
136
137#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
138
139 CHECK_VALUEMAP(GPR, 32);
140 CHECK_VALUEMAP(GPR, 64);
141 CHECK_VALUEMAP(GPR, 128);
142 CHECK_VALUEMAP(FPR, 16);
143 CHECK_VALUEMAP(FPR, 32);
144 CHECK_VALUEMAP(FPR, 64);
145 CHECK_VALUEMAP(FPR, 128);
146 CHECK_VALUEMAP(FPR, 256);
147 CHECK_VALUEMAP(FPR, 512);
148
149// Check the value mapping for 3-operands instructions where all the operands
150// map to the same value mapping.
151#define CHECK_VALUEMAP_3OPS(RBName, Size) \
152 do { \
153 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
154 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
155 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
156 } while (false)
157
158 CHECK_VALUEMAP_3OPS(GPR, 32);
159 CHECK_VALUEMAP_3OPS(GPR, 64);
160 CHECK_VALUEMAP_3OPS(GPR, 128);
161 CHECK_VALUEMAP_3OPS(FPR, 32);
162 CHECK_VALUEMAP_3OPS(FPR, 64);
163 CHECK_VALUEMAP_3OPS(FPR, 128);
164 CHECK_VALUEMAP_3OPS(FPR, 256);
165 CHECK_VALUEMAP_3OPS(FPR, 512);
166
167#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
168 do { \
169 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
170 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
171 (void)PartialMapDstIdx; \
172 (void)PartialMapSrcIdx; \
173 const ValueMapping *Map = getCopyMapping(AArch64::RBNameDst##RegBankID, \
174 AArch64::RBNameSrc##RegBankID, \
175 TypeSize::getFixed(Size)); \
176 (void)Map; \
177 assert(Map[0].BreakDown == \
178 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
179 Map[0].NumBreakDowns == 1 && \
180 #RBNameDst #Size " Dst is incorrectly initialized"); \
181 assert(Map[1].BreakDown == \
182 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
183 Map[1].NumBreakDowns == 1 && \
184 #RBNameSrc #Size " Src is incorrectly initialized"); \
185 \
186 } while (false)
187
188 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
189 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
190 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
191 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
192 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
193 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
194 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
195 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
196
197#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
198 do { \
199 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
200 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
201 (void)PartialMapDstIdx; \
202 (void)PartialMapSrcIdx; \
203 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
204 (void)Map; \
205 assert(Map[0].BreakDown == \
206 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
207 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
208 " Dst is incorrectly initialized"); \
209 assert(Map[1].BreakDown == \
210 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
211 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
212 " Src is incorrectly initialized"); \
213 \
214 } while (false)
215
216 CHECK_VALUEMAP_FPEXT(32, 16);
217 CHECK_VALUEMAP_FPEXT(64, 16);
218 CHECK_VALUEMAP_FPEXT(64, 32);
219 CHECK_VALUEMAP_FPEXT(128, 64);
220
221 assert(verify(TRI) && "Invalid register bank information");
222 };
223
224 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
225}
226
227unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
228 const RegisterBank &B,
229 const TypeSize Size) const {
230 // What do we do with different size?
231 // copy are same size.
232 // Will introduce other hooks for different size:
233 // * extract cost.
234 // * build_sequence cost.
235
236 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
237 // FIXME: This should be deduced from the scheduling model.
238 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
239 // FMOVXDr or FMOVWSr.
240 return 5;
241 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
242 // FMOVDXr or FMOVSWr.
243 return 4;
244
245 return RegisterBankInfo::copyCost(A, B, Size);
246}
247
248const RegisterBank &
249AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
250 LLT Ty) const {
251 switch (RC.getID()) {
252 case AArch64::GPR64sponlyRegClassID:
253 return AArch64::GPRRegBank;
254 default:
255 return AArch64GenRegisterBankInfo::getRegBankFromRegClass(RC, Ty);
256 }
257}
258
259RegisterBankInfo::InstructionMappings
260AArch64RegisterBankInfo::getInstrAlternativeMappings(
261 const MachineInstr &MI) const {
262 const MachineFunction &MF = *MI.getParent()->getParent();
263 const TargetSubtargetInfo &STI = MF.getSubtarget();
264 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
265 const MachineRegisterInfo &MRI = MF.getRegInfo();
266
267 switch (MI.getOpcode()) {
268 case TargetOpcode::G_OR: {
269 // 32 and 64-bit or can be mapped on either FPR or
270 // GPR for the same cost.
271 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
272 if (Size != 32 && Size != 64)
273 break;
274
275 // If the instruction has any implicit-defs or uses,
276 // do not mess with it.
277 if (MI.getNumOperands() != 3)
278 break;
279 InstructionMappings AltMappings;
280 const InstructionMapping &GPRMapping = getInstructionMapping(
281 /*ID*/ 1, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstGPR, Size),
282 /*NumOperands*/ 3);
283 const InstructionMapping &FPRMapping = getInstructionMapping(
284 /*ID*/ 2, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstFPR, Size),
285 /*NumOperands*/ 3);
286
287 AltMappings.push_back(Elt: &GPRMapping);
288 AltMappings.push_back(Elt: &FPRMapping);
289 return AltMappings;
290 }
291 case TargetOpcode::G_BITCAST: {
292 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
293 if (Size != 32 && Size != 64)
294 break;
295
296 // If the instruction has any implicit-defs or uses,
297 // do not mess with it.
298 if (MI.getNumOperands() != 2)
299 break;
300
301 InstructionMappings AltMappings;
302 const InstructionMapping &GPRMapping = getInstructionMapping(
303 /*ID*/ 1, /*Cost*/ 1,
304 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
305 /*NumOperands*/ 2);
306 const InstructionMapping &FPRMapping = getInstructionMapping(
307 /*ID*/ 2, /*Cost*/ 1,
308 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
309 /*NumOperands*/ 2);
310 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
311 /*ID*/ 3,
312 /*Cost*/
313 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
314 Size: TypeSize::getFixed(ExactSize: Size)),
315 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
316 /*NumOperands*/ 2);
317 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
318 /*ID*/ 3,
319 /*Cost*/
320 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
321 Size: TypeSize::getFixed(ExactSize: Size)),
322 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
323 /*NumOperands*/ 2);
324
325 AltMappings.push_back(Elt: &GPRMapping);
326 AltMappings.push_back(Elt: &FPRMapping);
327 AltMappings.push_back(Elt: &GPRToFPRMapping);
328 AltMappings.push_back(Elt: &FPRToGPRMapping);
329 return AltMappings;
330 }
331 case TargetOpcode::G_LOAD: {
332 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
333 if (Size != 64)
334 break;
335
336 // If the instruction has any implicit-defs or uses,
337 // do not mess with it.
338 if (MI.getNumOperands() != 2)
339 break;
340
341 InstructionMappings AltMappings;
342 const InstructionMapping &GPRMapping = getInstructionMapping(
343 /*ID*/ 1, /*Cost*/ 1,
344 OperandsMapping: getOperandsMapping(
345 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstGPR, Size),
346 // Addresses are GPR 64-bit.
347 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
348 /*NumOperands*/ 2);
349 const InstructionMapping &FPRMapping = getInstructionMapping(
350 /*ID*/ 2, /*Cost*/ 1,
351 OperandsMapping: getOperandsMapping(
352 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstFPR, Size),
353 // Addresses are GPR 64-bit.
354 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
355 /*NumOperands*/ 2);
356
357 AltMappings.push_back(Elt: &GPRMapping);
358 AltMappings.push_back(Elt: &FPRMapping);
359 return AltMappings;
360 }
361 default:
362 break;
363 }
364 return RegisterBankInfo::getInstrAlternativeMappings(MI);
365}
366
367static bool preferGPRForFPImm(const MachineInstr &MI,
368 const MachineRegisterInfo &MRI,
369 const AArch64Subtarget &STI) {
370 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT);
371 Register Dst = MI.getOperand(i: 0).getReg();
372 LLT Ty = MRI.getType(Reg: Dst);
373
374 unsigned Size = Ty.getSizeInBits();
375 if (Size != 16 && Size != 32 && Size != 64)
376 return false;
377
378 EVT VT = EVT::getFloatingPointVT(BitWidth: Size);
379 const AArch64TargetLowering *TLI = STI.getTargetLowering();
380
381 const APFloat Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
382 const APInt ImmBits = Imm.bitcastToAPInt();
383
384 // If all the uses are stores use a gpr constant
385 if (all_of(Range: MRI.use_nodbg_instructions(Reg: Dst), P: [&](const MachineInstr &UseMI) {
386 return UseMI.getOpcode() == TargetOpcode::G_STORE &&
387 UseMI.getOperand(i: 0).getReg() == Dst;
388 }))
389 return true;
390
391 // Check if we can encode this as a movi. Note, we only have one pattern so
392 // far for movis, hence the one check.
393 if (Size == 32) {
394 uint64_t Val = APInt::getSplat(NewLen: 64, V: ImmBits).getZExtValue();
395 if (AArch64_AM::isAdvSIMDModImmType4(Imm: Val))
396 return false;
397 }
398
399 // We want to use GPR when the value cannot be encoded as the immediate value
400 // of a fmov and when it will not result in a constant pool load. As
401 // AArch64TargetLowering::isFPImmLegal is used by the instruction selector
402 // to choose whether to emit a constant pool load, negating this check will
403 // ensure it would not have become a constant pool load.
404 bool OptForSize =
405 shouldOptimizeForSize(F: &MI.getMF()->getFunction(), PSI: nullptr, BFI: nullptr);
406 bool IsLegal = TLI->isFPImmLegal(Imm, VT, ForCodeSize: OptForSize);
407 bool IsFMov = TLI->isFPImmLegalAsFMov(Imm, VT);
408 return !IsFMov && IsLegal;
409}
410
411// Some of the instructions in applyMappingImpl attempt to anyext small values.
412// It may be that these values come from a G_CONSTANT that has been expanded to
413// 32 bits and then truncated. If this is the case, we shouldn't insert an
414// anyext and should instead make use of the G_CONSTANT directly, deleting the
415// trunc if possible.
416static bool foldTruncOfI32Constant(MachineInstr &MI, unsigned OpIdx,
417 MachineRegisterInfo &MRI) {
418 MachineOperand &Op = MI.getOperand(i: OpIdx);
419
420 Register ScalarReg = Op.getReg();
421 MachineInstr *TruncMI = MRI.getVRegDef(Reg: ScalarReg);
422 if (!TruncMI || TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
423 return false;
424
425 Register TruncSrc = TruncMI->getOperand(i: 1).getReg();
426 MachineInstr *SrcDef = MRI.getVRegDef(Reg: TruncSrc);
427 if (!SrcDef || SrcDef->getOpcode() != TargetOpcode::G_CONSTANT)
428 return false;
429
430 LLT TruncSrcTy = MRI.getType(Reg: TruncSrc);
431 if (!TruncSrcTy.isScalar() || TruncSrcTy.getSizeInBits() != 32)
432 return false;
433
434 // Avoid truncating and extending a constant, this helps with selection.
435 Op.setReg(TruncSrc);
436 MRI.setRegBank(Reg: TruncSrc, RegBank: AArch64::GPRRegBank);
437
438 if (MRI.use_empty(RegNo: ScalarReg))
439 TruncMI->eraseFromParent();
440
441 return true;
442}
443
444void AArch64RegisterBankInfo::applyMappingImpl(
445 MachineIRBuilder &Builder, const OperandsMapper &OpdMapper) const {
446 MachineInstr &MI = OpdMapper.getMI();
447 MachineRegisterInfo &MRI = OpdMapper.getMRI();
448
449 switch (MI.getOpcode()) {
450 case TargetOpcode::G_CONSTANT: {
451 Register Dst = MI.getOperand(i: 0).getReg();
452 [[maybe_unused]] LLT DstTy = MRI.getType(Reg: Dst);
453 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank && DstTy.isScalar() &&
454 DstTy.getSizeInBits() < 32 &&
455 "Expected a scalar smaller than 32 bits on a GPR.");
456 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
457 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
458 Builder.buildTrunc(Res: Dst, Op: ExtReg);
459
460 APInt Val = MI.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
461 LLVMContext &Ctx = Builder.getMF().getFunction().getContext();
462 MI.getOperand(i: 1).setCImm(ConstantInt::get(Context&: Ctx, V: Val));
463 MI.getOperand(i: 0).setReg(ExtReg);
464 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
465
466 return applyDefaultMapping(OpdMapper);
467 }
468 case TargetOpcode::G_FCONSTANT: {
469 Register Dst = MI.getOperand(i: 0).getReg();
470 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank &&
471 "Expected Dst to be on a GPR.");
472 const APFloat &Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
473 APInt Bits = Imm.bitcastToAPInt();
474 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
475 if (Bits.getBitWidth() < 32) {
476 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
477 Builder.buildConstant(Res: ExtReg, Val: Bits.zext(width: 32));
478 Builder.buildTrunc(Res: Dst, Op: ExtReg);
479 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
480 } else {
481 Builder.buildConstant(Res: Dst, Val: Bits);
482 }
483 MI.eraseFromParent();
484 return;
485 }
486 case TargetOpcode::G_STORE: {
487 Register Dst = MI.getOperand(i: 0).getReg();
488 LLT Ty = MRI.getType(Reg: Dst);
489
490 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
491 Ty.getSizeInBits() < 32) {
492
493 if (foldTruncOfI32Constant(MI, OpIdx: 0, MRI))
494 return applyDefaultMapping(OpdMapper);
495
496 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
497 auto Ext = Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: Dst);
498 MI.getOperand(i: 0).setReg(Ext.getReg(Idx: 0));
499 MRI.setRegBank(Reg: Ext.getReg(Idx: 0), RegBank: AArch64::GPRRegBank);
500 }
501 return applyDefaultMapping(OpdMapper);
502 }
503 case TargetOpcode::G_LOAD: {
504 Register Dst = MI.getOperand(i: 0).getReg();
505 LLT Ty = MRI.getType(Reg: Dst);
506 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
507 Ty.getSizeInBits() < 32) {
508 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
509 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
510 Builder.buildTrunc(Res: Dst, Op: ExtReg);
511 MI.getOperand(i: 0).setReg(ExtReg);
512 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
513 }
514 [[fallthrough]];
515 }
516 case TargetOpcode::G_OR:
517 case TargetOpcode::G_BITCAST:
518 // Those ID must match getInstrAlternativeMappings.
519 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
520 OpdMapper.getInstrMapping().getID() <= 4) &&
521 "Don't know how to handle that ID");
522 return applyDefaultMapping(OpdMapper);
523 case AArch64::G_DUP: {
524 if (foldTruncOfI32Constant(MI, OpIdx: 1, MRI))
525 return applyDefaultMapping(OpdMapper);
526
527 // Extend smaller gpr to 32-bits
528 assert(MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() < 32 &&
529 "Expected sources smaller than 32-bits");
530 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
531
532 Register ConstReg =
533 Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: MI.getOperand(i: 1).getReg())
534 .getReg(Idx: 0);
535 MRI.setRegBank(Reg: ConstReg, RegBank: AArch64::GPRRegBank);
536 MI.getOperand(i: 1).setReg(ConstReg);
537
538 return applyDefaultMapping(OpdMapper);
539 }
540 default:
541 llvm_unreachable("Don't know how to handle that operation");
542 }
543}
544
545const RegisterBankInfo::InstructionMapping &
546AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
547 const MachineInstr &MI) const {
548 const unsigned Opc = MI.getOpcode();
549 const MachineFunction &MF = *MI.getParent()->getParent();
550 const MachineRegisterInfo &MRI = MF.getRegInfo();
551
552 unsigned NumOperands = MI.getNumOperands();
553 assert(NumOperands <= 3 &&
554 "This code is for instructions with 3 or less operands");
555
556 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
557 TypeSize Size = Ty.getSizeInBits();
558 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
559
560 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
561
562#ifndef NDEBUG
563 // Make sure all the operands are using similar size and type.
564 // Should probably be checked by the machine verifier.
565 // This code won't catch cases where the number of lanes is
566 // different between the operands.
567 // If we want to go to that level of details, it is probably
568 // best to check that the types are the same, period.
569 // Currently, we just check that the register banks are the same
570 // for each types.
571 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
572 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
573 assert(
574 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
575 RBIdx, OpTy.getSizeInBits()) ==
576 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
577 "Operand has incompatible size");
578 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
579 (void)OpIsFPR;
580 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
581 }
582#endif // End NDEBUG.
583
584 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
585 OperandsMapping: getValueMapping(RBIdx, Size), NumOperands);
586}
587
588/// \returns true if a given intrinsic only uses and defines FPRs.
589static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
590 const MachineInstr &MI) {
591 // TODO: Add more intrinsics.
592 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
593 default:
594 return false;
595 case Intrinsic::aarch64_neon_uaddlv:
596 case Intrinsic::aarch64_neon_uaddv:
597 case Intrinsic::aarch64_neon_saddv:
598 case Intrinsic::aarch64_neon_umaxv:
599 case Intrinsic::aarch64_neon_smaxv:
600 case Intrinsic::aarch64_neon_uminv:
601 case Intrinsic::aarch64_neon_sminv:
602 case Intrinsic::aarch64_neon_faddv:
603 case Intrinsic::aarch64_neon_fmaxv:
604 case Intrinsic::aarch64_neon_fminv:
605 case Intrinsic::aarch64_neon_fmaxnmv:
606 case Intrinsic::aarch64_neon_fminnmv:
607 case Intrinsic::aarch64_neon_fmulx:
608 case Intrinsic::aarch64_neon_frecpe:
609 case Intrinsic::aarch64_neon_frecps:
610 case Intrinsic::aarch64_neon_frecpx:
611 case Intrinsic::aarch64_neon_frsqrte:
612 case Intrinsic::aarch64_neon_frsqrts:
613 case Intrinsic::aarch64_neon_facge:
614 case Intrinsic::aarch64_neon_facgt:
615 case Intrinsic::aarch64_neon_fabd:
616 case Intrinsic::aarch64_neon_sqrdmlah:
617 case Intrinsic::aarch64_neon_sqrdmlsh:
618 case Intrinsic::aarch64_neon_sqrdmulh:
619 case Intrinsic::aarch64_neon_suqadd:
620 case Intrinsic::aarch64_neon_usqadd:
621 case Intrinsic::aarch64_neon_uqadd:
622 case Intrinsic::aarch64_neon_sqadd:
623 case Intrinsic::aarch64_neon_uqsub:
624 case Intrinsic::aarch64_neon_sqsub:
625 case Intrinsic::aarch64_neon_sqdmulh:
626 case Intrinsic::aarch64_neon_sqdmulls_scalar:
627 case Intrinsic::aarch64_neon_srshl:
628 case Intrinsic::aarch64_neon_urshl:
629 case Intrinsic::aarch64_neon_sqshl:
630 case Intrinsic::aarch64_neon_uqshl:
631 case Intrinsic::aarch64_neon_sqrshl:
632 case Intrinsic::aarch64_neon_uqrshl:
633 case Intrinsic::aarch64_neon_ushl:
634 case Intrinsic::aarch64_neon_sshl:
635 case Intrinsic::aarch64_neon_sqshrn:
636 case Intrinsic::aarch64_neon_sqshrun:
637 case Intrinsic::aarch64_neon_sqrshrn:
638 case Intrinsic::aarch64_neon_sqrshrun:
639 case Intrinsic::aarch64_neon_uqshrn:
640 case Intrinsic::aarch64_neon_uqrshrn:
641 case Intrinsic::aarch64_neon_sqneg:
642 case Intrinsic::aarch64_neon_sqabs:
643 case Intrinsic::aarch64_neon_scalar_uqxtn:
644 case Intrinsic::aarch64_neon_scalar_sqxtn:
645 case Intrinsic::aarch64_neon_scalar_sqxtun:
646 case Intrinsic::aarch64_crypto_sha1h:
647 case Intrinsic::aarch64_crypto_sha1c:
648 case Intrinsic::aarch64_crypto_sha1p:
649 case Intrinsic::aarch64_crypto_sha1m:
650 case Intrinsic::aarch64_sisd_fcvtxn:
651 case Intrinsic::aarch64_sisd_fabd:
652 return true;
653 case Intrinsic::aarch64_neon_saddlv: {
654 const LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
655 return SrcTy.getElementType().getSizeInBits() >= 16 &&
656 SrcTy.getElementCount().getFixedValue() >= 4;
657 }
658 }
659}
660
661bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
662 const MachineInstr &MI, const MachineRegisterInfo &MRI,
663 const AArch64RegisterInfo &TRI, const unsigned Depth) const {
664 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
665 return false;
666
667 return any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
668 P: [&](const MachineInstr &UseMI) {
669 if (onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1))
670 return true;
671 return isPHIWithFPConstraints(MI: UseMI, MRI, TRI, Depth: Depth + 1);
672 });
673}
674
675bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
676 const MachineRegisterInfo &MRI,
677 const AArch64RegisterInfo &TRI,
678 unsigned Depth) const {
679 unsigned Op = MI.getOpcode();
680 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
681 return true;
682
683 // Do we have an explicit floating point instruction?
684 if (isPreISelGenericFloatingPointOpcode(Opc: Op))
685 return true;
686
687 // No. Check if we have a copy-like instruction. If we do, then we could
688 // still be fed by floating point instructions.
689 if (Op != TargetOpcode::COPY && !MI.isPHI() &&
690 !isPreISelGenericOptimizationHint(Opcode: Op))
691 return false;
692
693 // Check if we already know the register bank.
694 auto *RB = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
695 if (RB == &AArch64::FPRRegBank)
696 return true;
697 if (RB == &AArch64::GPRRegBank)
698 return false;
699
700 // We don't know anything.
701 //
702 // If we have a phi, we may be able to infer that it will be assigned a FPR
703 // based off of its inputs.
704 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
705 return false;
706
707 return any_of(Range: MI.explicit_uses(), P: [&](const MachineOperand &Op) {
708 return Op.isReg() &&
709 onlyDefinesFP(MI: *MRI.getVRegDef(Reg: Op.getReg()), MRI, TRI, Depth: Depth + 1);
710 });
711}
712
713bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
714 const MachineRegisterInfo &MRI,
715 const AArch64RegisterInfo &TRI,
716 unsigned Depth) const {
717 switch (MI.getOpcode()) {
718 case TargetOpcode::G_BITCAST: {
719 Register DstReg = MI.getOperand(i: 0).getReg();
720 return all_of(Range: MRI.use_nodbg_instructions(Reg: DstReg),
721 P: [&](const MachineInstr &UseMI) {
722 return onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1) ||
723 prefersFPUse(MI: UseMI, MRI, TRI);
724 });
725 }
726
727 case TargetOpcode::G_FPTOSI:
728 case TargetOpcode::G_FPTOUI:
729 case TargetOpcode::G_FPTOSI_SAT:
730 case TargetOpcode::G_FPTOUI_SAT:
731 case TargetOpcode::G_FCMP:
732 case TargetOpcode::G_LROUND:
733 case TargetOpcode::G_LLROUND:
734 case TargetOpcode::G_CLMUL:
735 case AArch64::G_PMULL:
736 case AArch64::G_SLI:
737 case AArch64::G_SRI:
738 case AArch64::G_FPTRUNC_ODD:
739 return true;
740 case TargetOpcode::G_INTRINSIC:
741 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
742 case Intrinsic::aarch64_neon_fcvtas:
743 case Intrinsic::aarch64_neon_fcvtau:
744 case Intrinsic::aarch64_neon_fcvtzs:
745 case Intrinsic::aarch64_neon_fcvtzu:
746 case Intrinsic::aarch64_neon_fcvtms:
747 case Intrinsic::aarch64_neon_fcvtmu:
748 case Intrinsic::aarch64_neon_fcvtns:
749 case Intrinsic::aarch64_neon_fcvtnu:
750 case Intrinsic::aarch64_neon_fcvtps:
751 case Intrinsic::aarch64_neon_fcvtpu:
752 return true;
753 default:
754 break;
755 }
756 break;
757 default:
758 break;
759 }
760 return hasFPConstraints(MI, MRI, TRI, Depth);
761}
762
763bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
764 const MachineRegisterInfo &MRI,
765 const AArch64RegisterInfo &TRI,
766 unsigned Depth) const {
767 switch (MI.getOpcode()) {
768 case AArch64::G_DUP:
769 case AArch64::G_SADDLP:
770 case AArch64::G_UADDLP:
771 case TargetOpcode::G_SITOFP:
772 case TargetOpcode::G_UITOFP:
773 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
774 case TargetOpcode::G_INSERT_VECTOR_ELT:
775 case TargetOpcode::G_BUILD_VECTOR:
776 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
777 case AArch64::G_SLI:
778 case AArch64::G_SRI:
779 case AArch64::G_FPTRUNC_ODD:
780 return true;
781 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
782 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
783 case Intrinsic::aarch64_neon_ld1x2:
784 case Intrinsic::aarch64_neon_ld1x3:
785 case Intrinsic::aarch64_neon_ld1x4:
786 case Intrinsic::aarch64_neon_ld2:
787 case Intrinsic::aarch64_neon_ld2lane:
788 case Intrinsic::aarch64_neon_ld2r:
789 case Intrinsic::aarch64_neon_ld3:
790 case Intrinsic::aarch64_neon_ld3lane:
791 case Intrinsic::aarch64_neon_ld3r:
792 case Intrinsic::aarch64_neon_ld4:
793 case Intrinsic::aarch64_neon_ld4lane:
794 case Intrinsic::aarch64_neon_ld4r:
795 return true;
796 default:
797 break;
798 }
799 break;
800 default:
801 break;
802 }
803 return hasFPConstraints(MI, MRI, TRI, Depth);
804}
805
806bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,
807 const MachineRegisterInfo &MRI,
808 const AArch64RegisterInfo &TRI,
809 unsigned Depth) const {
810 switch (MI.getOpcode()) {
811 case TargetOpcode::G_SITOFP:
812 case TargetOpcode::G_UITOFP:
813 return MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() ==
814 MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
815 }
816 return onlyDefinesFP(MI, MRI, TRI, Depth);
817}
818
819bool AArch64RegisterBankInfo::shouldUseFPRForCvtOperand(
820 const MachineInstr &MI, bool BankedOpIsDef, bool ForceFPRForBankedOp16,
821 bool ForceFPRForOtherOp16, bool AllowFPRCVT, bool CheckBankedOpUses) const {
822 const MachineFunction &MF = *MI.getMF();
823 const MachineRegisterInfo &MRI = MF.getRegInfo();
824 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
825 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
826
827 unsigned BankedOpIdx = BankedOpIsDef ? 0 : 2;
828 unsigned OtherOpIdx = BankedOpIsDef ? 2 : 0;
829
830 Register BankedReg = MI.getOperand(i: BankedOpIdx).getReg();
831 if (MRI.getType(Reg: BankedReg).isVector())
832 return true;
833
834 TypeSize BankedSize = getSizeInBits(Reg: BankedReg, MRI, TRI);
835 TypeSize OtherSize =
836 getSizeInBits(Reg: MI.getOperand(i: OtherOpIdx).getReg(), MRI, TRI);
837 if ((ForceFPRForBankedOp16 && BankedSize == 16) ||
838 (ForceFPRForOtherOp16 && OtherSize == 16))
839 return true;
840
841 if (BankedSize != OtherSize && !AllowFPRCVT)
842 return false;
843
844 if (CheckBankedOpUses)
845 return all_of(
846 Range: MRI.use_nodbg_instructions(Reg: BankedReg), P: [&](const MachineInstr &UseMI) {
847 return onlyUsesFP(MI: UseMI, MRI, TRI) || prefersFPUse(MI: UseMI, MRI, TRI);
848 });
849
850 const MachineInstr *DefMI = MRI.getVRegDef(Reg: BankedReg);
851 return DefMI &&
852 (onlyUsesFP(MI: *DefMI, MRI, TRI) || prefersFPUse(MI: *DefMI, MRI, TRI));
853}
854
855bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
856 // GMemOperation because we also want to match indexed loads.
857 auto *MemOp = cast<GMemOperation>(Val: &MI);
858 const Value *LdVal = MemOp->getMMO().getValue();
859 if (!LdVal)
860 return false;
861
862 Type *EltTy = nullptr;
863 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: LdVal)) {
864 EltTy = GV->getValueType();
865 // Look at the first element of the struct to determine the type we are
866 // loading
867 while (StructType *StructEltTy = dyn_cast<StructType>(Val: EltTy)) {
868 if (StructEltTy->getNumElements() == 0)
869 break;
870 EltTy = StructEltTy->getTypeAtIndex(N: 0U);
871 }
872 // Look at the first element of the array to determine its type
873 if (isa<ArrayType>(Val: EltTy))
874 EltTy = EltTy->getArrayElementType();
875 } else if (!isa<Constant>(Val: LdVal)) {
876 // FIXME: grubbing around uses is pretty ugly, but with no more
877 // `getPointerElementType` there's not much else we can do.
878 for (const auto *LdUser : LdVal->users()) {
879 if (isa<LoadInst>(Val: LdUser)) {
880 EltTy = LdUser->getType();
881 break;
882 }
883 if (isa<StoreInst>(Val: LdUser) && LdUser->getOperand(i: 1) == LdVal) {
884 EltTy = LdUser->getOperand(i: 0)->getType();
885 break;
886 }
887 }
888 }
889 return EltTy && EltTy->isFPOrFPVectorTy();
890}
891
892const RegisterBankInfo::InstructionMapping &
893AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
894 const unsigned Opc = MI.getOpcode();
895
896 // Try the default logic for non-generic instructions that are either copies
897 // or already have some operands assigned to banks.
898 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opcode: Opc)) ||
899 Opc == TargetOpcode::G_PHI) {
900 const RegisterBankInfo::InstructionMapping &Mapping =
901 getInstrMappingImpl(MI);
902 if (Mapping.isValid())
903 return Mapping;
904 }
905
906 const MachineFunction &MF = *MI.getParent()->getParent();
907 const MachineRegisterInfo &MRI = MF.getRegInfo();
908 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
909 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
910
911 switch (Opc) {
912 // G_{F|S|U}REM are not listed because they are not legal.
913 // Arithmetic ops.
914 case TargetOpcode::G_ADD:
915 case TargetOpcode::G_SUB:
916 case TargetOpcode::G_PTR_ADD:
917 case TargetOpcode::G_MUL:
918 case TargetOpcode::G_SDIV:
919 case TargetOpcode::G_UDIV:
920 // Bitwise ops.
921 case TargetOpcode::G_AND:
922 case TargetOpcode::G_OR:
923 case TargetOpcode::G_XOR:
924 // Floating point ops.
925 case TargetOpcode::G_FADD:
926 case TargetOpcode::G_FSUB:
927 case TargetOpcode::G_FMUL:
928 case TargetOpcode::G_FDIV:
929 case TargetOpcode::G_FMAXIMUM:
930 case TargetOpcode::G_FMINIMUM:
931 return getSameKindOfOperandsMapping(MI);
932 case TargetOpcode::G_FPEXT: {
933 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
934 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
935 return getInstructionMapping(
936 ID: DefaultMappingID, /*Cost*/ 1,
937 OperandsMapping: getFPExtMapping(DstSize: DstTy.getSizeInBits(), SrcSize: SrcTy.getSizeInBits()),
938 /*NumOperands*/ 2);
939 }
940 // Shifts.
941 case TargetOpcode::G_SHL:
942 case TargetOpcode::G_LSHR:
943 case TargetOpcode::G_ASHR: {
944 LLT ShiftAmtTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
945 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
946 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
947 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
948 OperandsMapping: &ValMappings[Shift64Imm], NumOperands: 3);
949 return getSameKindOfOperandsMapping(MI);
950 }
951 case TargetOpcode::G_BITCAST: {
952 Register SrcReg = MI.getOperand(i: 1).getReg();
953 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
954 if (SrcRB) {
955 TypeSize Size = getSizeInBits(Reg: SrcReg, MRI, TRI);
956 return getInstructionMapping(
957 ID: DefaultMappingID, Cost: 0,
958 OperandsMapping: getCopyMapping(DstBankID: SrcRB->getID(), SrcBankID: SrcRB->getID(), Size),
959 // We only care about the mapping of the destination.
960 /*NumOperands=*/2);
961 }
962 [[fallthrough]];
963 }
964 case TargetOpcode::COPY: {
965 Register DstReg = MI.getOperand(i: 0).getReg();
966 Register SrcReg = MI.getOperand(i: 1).getReg();
967 // Check if one of the register is not a generic register.
968 if ((DstReg.isPhysical() || !MRI.getType(Reg: DstReg).isValid()) ||
969 (SrcReg.isPhysical() || !MRI.getType(Reg: SrcReg).isValid())) {
970 const RegisterBank *DstRB = getRegBank(Reg: DstReg, MRI, TRI);
971 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
972 if (!DstRB)
973 DstRB = SrcRB;
974 else if (!SrcRB)
975 SrcRB = DstRB;
976 // If both RB are null that means both registers are generic.
977 // We shouldn't be here.
978 assert(DstRB && SrcRB && "Both RegBank were nullptr");
979 TypeSize Size = getSizeInBits(Reg: DstReg, MRI, TRI);
980 return getInstructionMapping(
981 ID: DefaultMappingID, Cost: copyCost(A: *DstRB, B: *SrcRB, Size),
982 OperandsMapping: getCopyMapping(DstBankID: DstRB->getID(), SrcBankID: SrcRB->getID(), Size),
983 // We only care about the mapping of the destination.
984 /*NumOperands*/ 1);
985 }
986 // Both registers are generic
987 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
988 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
989 TypeSize Size = DstTy.getSizeInBits();
990 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
991 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
992 const RegisterBank &DstRB =
993 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
994 const RegisterBank &SrcRB =
995 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
996 return getInstructionMapping(
997 ID: DefaultMappingID, Cost: copyCost(A: DstRB, B: SrcRB, Size),
998 OperandsMapping: getCopyMapping(DstBankID: DstRB.getID(), SrcBankID: SrcRB.getID(), Size),
999 // We only care about the mapping of the destination for COPY.
1000 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
1001 }
1002 case TargetOpcode::G_CONSTANT: {
1003 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1004 TypeSize Size = DstTy.getSizeInBits();
1005 if (!DstTy.isPointer() && (!DstTy.isScalar() || Size < 32 || Size > 64))
1006 break;
1007 // Scalar constants materialize in GPRs.
1008 [[fallthrough]];
1009 }
1010 case TargetOpcode::G_BRCOND:
1011 case TargetOpcode::G_FRAME_INDEX: {
1012 // Operand 0 is the only banked operand and is mapped to GPR.
1013 return getInstructionMapping(
1014 ID: DefaultMappingID, /*Cost=*/1,
1015 OperandsMapping: getOperandsMapping(
1016 OpdsMapping: {getValueMapping(
1017 RBIdx: PMI_FirstGPR,
1018 Size: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits()),
1019 nullptr}),
1020 /*NumOperands=*/2);
1021 }
1022 default:
1023 break;
1024 }
1025
1026 unsigned NumOperands = MI.getNumOperands();
1027 unsigned MappingID = DefaultMappingID;
1028
1029 // Track the size and bank of each register. We don't do partial mappings.
1030 SmallVector<unsigned, 4> OpSize(NumOperands);
1031 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
1032 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1033 auto &MO = MI.getOperand(i: Idx);
1034 if (!MO.isReg() || !MO.getReg())
1035 continue;
1036
1037 LLT Ty = MRI.getType(Reg: MO.getReg());
1038 if (!Ty.isValid())
1039 continue;
1040 OpSize[Idx] = Ty.getSizeInBits().getKnownMinValue();
1041
1042 // As a top-level guess, vectors including both scalable and non-scalable
1043 // ones go in FPRs, scalars and pointers in GPRs.
1044 // For floating-point instructions, scalars go in FPRs.
1045 if (Ty.isVector())
1046 OpRegBankIdx[Idx] = PMI_FirstFPR;
1047 else if (isPreISelGenericFloatingPointOpcode(Opc) ||
1048 (MO.isDef() && onlyDefinesFP(MI, MRI, TRI)) ||
1049 (MO.isUse() && onlyUsesFP(MI, MRI, TRI)) ||
1050 Ty.getSizeInBits() > 64)
1051 OpRegBankIdx[Idx] = PMI_FirstFPR;
1052 else
1053 OpRegBankIdx[Idx] = PMI_FirstGPR;
1054 }
1055
1056 unsigned Cost = 1;
1057 // Some of the floating-point instructions have mixed GPR and FPR operands:
1058 // fine-tune the computed mapping.
1059 switch (Opc) {
1060 case TargetOpcode::G_CONSTANT: {
1061 Register Dst = MI.getOperand(i: 0).getReg();
1062 LLT DstTy = MRI.getType(Reg: Dst);
1063 if (DstTy.isScalar() && DstTy.getSizeInBits() < 32)
1064 MappingID = CustomMappingID;
1065 break;
1066 }
1067 case TargetOpcode::G_FCONSTANT: {
1068 if (preferGPRForFPImm(MI, MRI, STI)) {
1069 // Materialize in GPR and rely on later bank copies for FP uses.
1070 MappingID = CustomMappingID;
1071 OpRegBankIdx = {PMI_FirstGPR};
1072 }
1073 break;
1074 }
1075 case AArch64::G_DUP: {
1076 Register ScalarReg = MI.getOperand(i: 1).getReg();
1077 LLT ScalarTy = MRI.getType(Reg: ScalarReg);
1078 auto ScalarDef = MRI.getVRegDef(Reg: ScalarReg);
1079 // We want to select dup(load) into LD1R.
1080 if (ScalarDef->getOpcode() == TargetOpcode::G_LOAD)
1081 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1082 // s8 is an exception for G_DUP, which we always want on gpr.
1083 else if (ScalarTy.getSizeInBits() != 8 &&
1084 (getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
1085 onlyDefinesFP(MI: *ScalarDef, MRI, TRI)))
1086 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1087 else {
1088 if (ScalarTy.getSizeInBits() < 32 &&
1089 getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::GPRRegBank) {
1090 // Calls applyMappingImpl()
1091 MappingID = CustomMappingID;
1092 }
1093 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1094 }
1095 break;
1096 }
1097 case TargetOpcode::G_TRUNC: {
1098 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1099 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
1100 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1101 break;
1102 }
1103 case TargetOpcode::G_SITOFP:
1104 case TargetOpcode::G_UITOFP: {
1105 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
1106 break;
1107 // Integer to FP conversions don't necessarily happen between GPR -> FPR
1108 // regbanks. They can also be done within an FPR register.
1109 Register SrcReg = MI.getOperand(i: 1).getReg();
1110 if (getRegBank(Reg: SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&
1111 MRI.getType(Reg: SrcReg).getSizeInBits() ==
1112 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits())
1113 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1114 else
1115 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1116 break;
1117 }
1118 case TargetOpcode::G_FPTOSI_SAT:
1119 case TargetOpcode::G_FPTOUI_SAT:
1120 case TargetOpcode::G_FPTOSI:
1121 case TargetOpcode::G_FPTOUI:
1122 case TargetOpcode::G_INTRINSIC_LRINT:
1123 case TargetOpcode::G_INTRINSIC_LLRINT:
1124 case TargetOpcode::G_LROUND:
1125 case TargetOpcode::G_LLROUND: {
1126 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1127 if (DstType.isVector())
1128 break;
1129 if (DstType == LLT::scalar(SizeInBits: 16)) {
1130 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1131 break;
1132 }
1133 TypeSize DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
1134 TypeSize SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI);
1135 if (((DstSize == SrcSize) || STI.hasFeature(Feature: AArch64::FeatureFPRCVT)) &&
1136 all_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1137 P: [&](const MachineInstr &UseMI) {
1138 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1139 prefersFPUse(MI: UseMI, MRI, TRI);
1140 }))
1141 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1142 else
1143 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
1144 break;
1145 }
1146 case TargetOpcode::G_FCMP: {
1147 // If the result is a vector, it must use a FPR.
1148 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 =
1149 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector() ? PMI_FirstFPR
1150 : PMI_FirstGPR;
1151 OpRegBankIdx = {Idx0,
1152 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
1153 break;
1154 }
1155 case TargetOpcode::G_BITCAST:
1156 // This is going to be a cross register bank copy and this is expensive.
1157 if (OpRegBankIdx[0] != OpRegBankIdx[1])
1158 Cost = copyCost(
1159 A: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
1160 B: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
1161 Size: TypeSize::getFixed(ExactSize: OpSize[0]));
1162 break;
1163 case TargetOpcode::G_LOAD: {
1164 // Loading in vector unit is slightly more expensive.
1165 // This is actually only true for the LD1R and co instructions,
1166 // but anyway for the fast mode this number does not matter and
1167 // for the greedy mode the cost of the cross bank copy will
1168 // offset this number.
1169 // FIXME: Should be derived from the scheduling model.
1170 if (OpRegBankIdx[0] != PMI_FirstGPR) {
1171 Cost = 2;
1172 break;
1173 }
1174
1175 if (cast<GLoad>(Val: MI).isAtomic()) {
1176 // Atomics always use GPR destinations. Don't refine any further.
1177 OpRegBankIdx[0] = PMI_FirstGPR;
1178 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() < 32)
1179 MappingID = CustomMappingID;
1180 break;
1181 }
1182
1183 // Try to guess the type of the load from the MMO.
1184 if (isLoadFromFPType(MI)) {
1185 OpRegBankIdx[0] = PMI_FirstFPR;
1186 break;
1187 }
1188
1189 // Check if that load feeds fp instructions.
1190 // In that case, we want the default mapping to be on FPR
1191 // instead of blind map every scalar to GPR.
1192 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1193 P: [&](const MachineInstr &UseMI) {
1194 // If we have at least one direct or indirect use
1195 // in a FP instruction,
1196 // assume this was a floating point load in the IR. If it was
1197 // not, we would have had a bitcast before reaching that
1198 // instruction.
1199 //
1200 // Int->FP conversion operations are also captured in
1201 // prefersFPUse().
1202
1203 if (isPHIWithFPConstraints(MI: UseMI, MRI, TRI))
1204 return true;
1205
1206 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1207 prefersFPUse(MI: UseMI, MRI, TRI);
1208 }))
1209 OpRegBankIdx[0] = PMI_FirstFPR;
1210
1211 // On GPR, extend any load < 32bits to 32bit.
1212 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1213 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1214 MappingID = CustomMappingID;
1215 break;
1216 }
1217 case TargetOpcode::G_STORE:
1218 // Check if that store is fed by fp instructions.
1219 if (OpRegBankIdx[0] == PMI_FirstGPR) {
1220 Register VReg = MI.getOperand(i: 0).getReg();
1221 if (VReg) {
1222 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1223 if (onlyDefinesFP(MI: *DefMI, MRI, TRI)) {
1224 OpRegBankIdx[0] = PMI_FirstFPR;
1225 break;
1226 }
1227 }
1228
1229 // On GPR, extend any store < 32bits to 32bit.
1230 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1231 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1232 MappingID = CustomMappingID;
1233 }
1234 break;
1235 case TargetOpcode::G_INDEXED_STORE:
1236 if (OpRegBankIdx[1] == PMI_FirstGPR) {
1237 Register VReg = MI.getOperand(i: 1).getReg();
1238 if (!VReg)
1239 break;
1240 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1241 if (onlyDefinesFP(MI: *DefMI, MRI, TRI))
1242 OpRegBankIdx[1] = PMI_FirstFPR;
1243 break;
1244 }
1245 break;
1246 case TargetOpcode::G_INDEXED_SEXTLOAD:
1247 case TargetOpcode::G_INDEXED_ZEXTLOAD:
1248 // These should always be GPR.
1249 OpRegBankIdx[0] = PMI_FirstGPR;
1250 break;
1251 case TargetOpcode::G_INDEXED_LOAD: {
1252 if (isLoadFromFPType(MI))
1253 OpRegBankIdx[0] = PMI_FirstFPR;
1254 break;
1255 }
1256 case TargetOpcode::G_SELECT: {
1257 // If the destination is FPR, preserve that.
1258 if (OpRegBankIdx[0] != PMI_FirstGPR)
1259 break;
1260
1261 // If we're taking in vectors, we have no choice but to put everything on
1262 // FPRs, except for the condition. The condition must always be on a GPR.
1263 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1264 if (SrcTy.isVector()) {
1265 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1266 break;
1267 }
1268
1269 // Try to minimize the number of copies. If we have more floating point
1270 // constrained values than not, then we'll put everything on FPR. Otherwise,
1271 // everything has to be on GPR.
1272 unsigned NumFP = 0;
1273
1274 // Check if the uses of the result always produce floating point values.
1275 //
1276 // For example:
1277 //
1278 // %z = G_SELECT %cond %x %y
1279 // fpr = G_FOO %z ...
1280 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1281 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
1282 ++NumFP;
1283
1284 // Check if the defs of the source values always produce floating point
1285 // values.
1286 //
1287 // For example:
1288 //
1289 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
1290 // %z = G_SELECT %cond %x %y
1291 //
1292 // Also check whether or not the sources have already been decided to be
1293 // FPR. Keep track of this.
1294 //
1295 // This doesn't check the condition, since it's just whatever is in NZCV.
1296 // This isn't passed explicitly in a register to fcsel/csel.
1297 for (unsigned Idx = 2; Idx < 4; ++Idx) {
1298 Register VReg = MI.getOperand(i: Idx).getReg();
1299 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1300 if (getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank ||
1301 onlyDefinesFP(MI: *DefMI, MRI, TRI))
1302 ++NumFP;
1303 }
1304
1305 // If we have more FP constraints than not, then move everything over to
1306 // FPR.
1307 if (NumFP >= 2)
1308 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1309
1310 break;
1311 }
1312 case TargetOpcode::G_UNMERGE_VALUES: {
1313 // If the first operand belongs to a FPR register bank, then make sure that
1314 // we preserve that.
1315 if (OpRegBankIdx[0] != PMI_FirstGPR)
1316 break;
1317
1318 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: MI.getNumOperands()-1).getReg());
1319 // UNMERGE into scalars from a vector should always use FPR.
1320 // Likewise if any of the uses are FP instructions.
1321 if (SrcTy.isVector() || SrcTy == LLT::scalar(SizeInBits: 128) ||
1322 any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1323 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
1324 // Set the register bank of every operand to FPR.
1325 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
1326 Idx < NumOperands; ++Idx)
1327 OpRegBankIdx[Idx] = PMI_FirstFPR;
1328 }
1329 break;
1330 }
1331 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1332 // Destination and source need to be FPRs.
1333 OpRegBankIdx[0] = PMI_FirstFPR;
1334 OpRegBankIdx[1] = PMI_FirstFPR;
1335
1336 // Index needs to be a GPR.
1337 OpRegBankIdx[2] = PMI_FirstGPR;
1338 break;
1339 case AArch64::G_SQSHLU_I:
1340 // Destination and source need to be FPRs.
1341 OpRegBankIdx[0] = PMI_FirstFPR;
1342 OpRegBankIdx[1] = PMI_FirstFPR;
1343
1344 // Shift Index needs to be a GPR.
1345 OpRegBankIdx[2] = PMI_FirstGPR;
1346 break;
1347
1348 case TargetOpcode::G_INSERT_VECTOR_ELT:
1349 OpRegBankIdx[0] = PMI_FirstFPR;
1350 OpRegBankIdx[1] = PMI_FirstFPR;
1351
1352 // The element may be either a GPR or FPR. Preserve that behaviour.
1353 if (getRegBank(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
1354 OpRegBankIdx[2] = PMI_FirstFPR;
1355 else {
1356 OpRegBankIdx[2] = PMI_FirstGPR;
1357 }
1358
1359 // Index needs to be a GPR.
1360 OpRegBankIdx[3] = PMI_FirstGPR;
1361 break;
1362 case TargetOpcode::G_EXTRACT: {
1363 // For s128 sources we have to use fpr unless we know otherwise.
1364 auto Src = MI.getOperand(i: 1).getReg();
1365 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1366 if (SrcTy.getSizeInBits() != 128)
1367 break;
1368 auto Idx = MRI.getRegClassOrNull(Reg: Src) == &AArch64::XSeqPairsClassRegClass
1369 ? PMI_FirstGPR
1370 : PMI_FirstFPR;
1371 OpRegBankIdx[0] = Idx;
1372 OpRegBankIdx[1] = Idx;
1373 break;
1374 }
1375 case TargetOpcode::G_BUILD_VECTOR: {
1376 // If the first source operand belongs to a FPR register bank, then make
1377 // sure that we preserve that.
1378 if (OpRegBankIdx[1] != PMI_FirstGPR)
1379 break;
1380 Register VReg = MI.getOperand(i: 1).getReg();
1381 if (!VReg)
1382 break;
1383
1384 // Get the instruction that defined the source operand reg, and check if
1385 // it's a floating point operation. Or, if it's a type like s16 which
1386 // doesn't have a exact size gpr register class. The exception is if the
1387 // build_vector has all constant operands, which may be better to leave as
1388 // gpr without copies, so it can be matched in imported patterns.
1389 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1390 unsigned DefOpc = DefMI->getOpcode();
1391 const LLT SrcTy = MRI.getType(Reg: VReg);
1392 if (all_of(Range: MI.operands(), P: [&](const MachineOperand &Op) {
1393 APInt Cst;
1394 return Op.isDef() || mi_match(R: Op.getReg(), MRI, P: m_ICst(Cst));
1395 }))
1396 break;
1397 if (isPreISelGenericFloatingPointOpcode(Opc: DefOpc) ||
1398 SrcTy.getSizeInBits() < 32 ||
1399 getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank) {
1400 // Have a floating point op.
1401 // Make sure every operand gets mapped to a FPR register class.
1402 unsigned NumOperands = MI.getNumOperands();
1403 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
1404 OpRegBankIdx[Idx] = PMI_FirstFPR;
1405 }
1406 break;
1407 }
1408 case TargetOpcode::G_VECREDUCE_FADD:
1409 case TargetOpcode::G_VECREDUCE_FMUL:
1410 case TargetOpcode::G_VECREDUCE_FMAX:
1411 case TargetOpcode::G_VECREDUCE_FMIN:
1412 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
1413 case TargetOpcode::G_VECREDUCE_FMINIMUM:
1414 case TargetOpcode::G_VECREDUCE_ADD:
1415 case TargetOpcode::G_VECREDUCE_MUL:
1416 case TargetOpcode::G_VECREDUCE_AND:
1417 case TargetOpcode::G_VECREDUCE_OR:
1418 case TargetOpcode::G_VECREDUCE_XOR:
1419 case TargetOpcode::G_VECREDUCE_SMAX:
1420 case TargetOpcode::G_VECREDUCE_SMIN:
1421 case TargetOpcode::G_VECREDUCE_UMAX:
1422 case TargetOpcode::G_VECREDUCE_UMIN:
1423 // Reductions produce a scalar value from a vector, the scalar should be on
1424 // FPR bank.
1425 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1426 break;
1427 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
1428 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
1429 // These reductions also take a scalar accumulator input.
1430 // Assign them FPR for now.
1431 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};
1432 break;
1433 case TargetOpcode::G_INTRINSIC:
1434 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
1435 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
1436 case Intrinsic::aarch64_neon_fcvtas:
1437 case Intrinsic::aarch64_neon_fcvtau:
1438 case Intrinsic::aarch64_neon_fcvtzs:
1439 case Intrinsic::aarch64_neon_fcvtzu:
1440 case Intrinsic::aarch64_neon_fcvtms:
1441 case Intrinsic::aarch64_neon_fcvtmu:
1442 case Intrinsic::aarch64_neon_fcvtns:
1443 case Intrinsic::aarch64_neon_fcvtnu:
1444 case Intrinsic::aarch64_neon_fcvtps:
1445 case Intrinsic::aarch64_neon_fcvtpu: {
1446 OpRegBankIdx[2] = PMI_FirstFPR;
1447 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/true,
1448 /*ForceFPRForBankedOp16=*/true,
1449 /*ForceFPRForOtherOp16=*/false,
1450 /*AllowFPRCVT=*/STI.hasFPRCVT(),
1451 /*CheckBankedOpUses=*/true))
1452 OpRegBankIdx[0] = PMI_FirstFPR;
1453 else
1454 OpRegBankIdx[0] = PMI_FirstGPR;
1455 break;
1456 }
1457 case Intrinsic::aarch64_neon_vcvtfxs2fp:
1458 case Intrinsic::aarch64_neon_vcvtfxu2fp: {
1459 OpRegBankIdx[0] = PMI_FirstFPR;
1460 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/false,
1461 /*ForceFPRForBankedOp16=*/false,
1462 /*ForceFPRForOtherOp16=*/true,
1463 /*AllowFPRCVT=*/false,
1464 /*CheckBankedOpUses=*/false))
1465 OpRegBankIdx[2] = PMI_FirstFPR;
1466 else
1467 OpRegBankIdx[2] = PMI_FirstGPR;
1468 break;
1469 }
1470 case Intrinsic::aarch64_neon_vcvtfp2fxs:
1471 case Intrinsic::aarch64_neon_vcvtfp2fxu: {
1472 OpRegBankIdx[2] = PMI_FirstFPR;
1473 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/true,
1474 /*ForceFPRForBankedOp16=*/false,
1475 /*ForceFPRForOtherOp16=*/true,
1476 /*AllowFPRCVT=*/false,
1477 /*CheckBankedOpUses=*/true))
1478 OpRegBankIdx[0] = PMI_FirstFPR;
1479 else
1480 OpRegBankIdx[0] = PMI_FirstGPR;
1481 break;
1482 }
1483 default: {
1484 // Check if we know that the intrinsic has any constraints on its register
1485 // banks. If it does, then update the mapping accordingly.
1486 unsigned Idx = 0;
1487 if (onlyDefinesFP(MI, MRI, TRI))
1488 for (const auto &Op : MI.defs()) {
1489 if (Op.isReg())
1490 OpRegBankIdx[Idx] = PMI_FirstFPR;
1491 ++Idx;
1492 }
1493 else
1494 Idx += MI.getNumExplicitDefs();
1495
1496 if (onlyUsesFP(MI, MRI, TRI))
1497 for (const auto &Op : MI.explicit_uses()) {
1498 if (Op.isReg())
1499 OpRegBankIdx[Idx] = PMI_FirstFPR;
1500 ++Idx;
1501 }
1502 break;
1503 }
1504 }
1505 break;
1506 }
1507 }
1508
1509 // Finally construct the computed mapping.
1510 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
1511 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1512 if (MI.getOperand(i: Idx).isReg() && MI.getOperand(i: Idx).getReg()) {
1513 LLT Ty = MRI.getType(Reg: MI.getOperand(i: Idx).getReg());
1514 if (!Ty.isValid())
1515 continue;
1516 auto Mapping =
1517 getValueMapping(RBIdx: OpRegBankIdx[Idx], Size: TypeSize::getFixed(ExactSize: OpSize[Idx]));
1518 if (!Mapping->isValid())
1519 return getInvalidInstructionMapping();
1520
1521 OpdsMapping[Idx] = Mapping;
1522 }
1523 }
1524
1525 return getInstructionMapping(ID: MappingID, Cost, OperandsMapping: getOperandsMapping(OpdsMapping),
1526 NumOperands);
1527}
1528