1//===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64RegisterBankInfo.h"
15#include "AArch64ExpandImm.h"
16#include "AArch64RegisterInfo.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCTargetDesc.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/LowLevelTypeUtils.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineOperand.h"
31#include "llvm/CodeGen/MachineRegisterInfo.h"
32#include "llvm/CodeGen/MachineSizeOpts.h"
33#include "llvm/CodeGen/RegisterBank.h"
34#include "llvm/CodeGen/RegisterBankInfo.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/CodeGen/TargetSubtargetInfo.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/IntrinsicsAArch64.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/Threading.h"
42#include <cassert>
43
44#define GET_TARGET_REGBANK_IMPL
45#include "AArch64GenRegisterBank.inc"
46
47// This file will be TableGen'ed at some point.
48#include "AArch64GenRegisterBankInfo.def"
49
50using namespace llvm;
51using namespace MIPatternMatch;
52static const unsigned CustomMappingID = 1;
53
54AArch64RegisterBankInfo::AArch64RegisterBankInfo(
55 const TargetRegisterInfo &TRI) {
56 static llvm::once_flag InitializeRegisterBankFlag;
57
58 static auto InitializeRegisterBankOnce = [&]() {
59 // We have only one set of register banks, whatever the subtarget
60 // is. Therefore, the initialization of the RegBanks table should be
61 // done only once. Indeed the table of all register banks
62 // (AArch64::RegBanks) is unique in the compiler. At some point, it
63 // will get tablegen'ed and the whole constructor becomes empty.
64
65 const RegisterBank &RBGPR = getRegBank(ID: AArch64::GPRRegBankID);
66 (void)RBGPR;
67 assert(&AArch64::GPRRegBank == &RBGPR &&
68 "The order in RegBanks is messed up");
69
70 const RegisterBank &RBFPR = getRegBank(ID: AArch64::FPRRegBankID);
71 (void)RBFPR;
72 assert(&AArch64::FPRRegBank == &RBFPR &&
73 "The order in RegBanks is messed up");
74
75 const RegisterBank &RBCCR = getRegBank(ID: AArch64::CCRegBankID);
76 (void)RBCCR;
77 assert(&AArch64::CCRegBank == &RBCCR &&
78 "The order in RegBanks is messed up");
79
80 // The GPR register bank is fully defined by all the registers in
81 // GR64all + its subclasses.
82 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
83 "Subclass not added?");
84 assert(getMaximumSize(RBGPR.getID()) == 128 &&
85 "GPRs should hold up to 128-bit");
86
87 // The FPR register bank is fully defined by all the registers in
88 // GR64all + its subclasses.
89 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
90 "Subclass not added?");
91 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
92 "Subclass not added?");
93 assert(getMaximumSize(RBFPR.getID()) == 512 &&
94 "FPRs should hold up to 512-bit via QQQQ sequence");
95
96 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
97 "Class not added?");
98 assert(getMaximumSize(RBCCR.getID()) == 32 &&
99 "CCR should hold up to 32-bit");
100
101 // Check that the TableGen'ed like file is in sync we our expectations.
102 // First, the Idx.
103 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
104 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) &&
105 "PartialMappingIdx's are incorrectly ordered");
106 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
107 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
108 PMI_FPR256, PMI_FPR512}) &&
109 "PartialMappingIdx's are incorrectly ordered");
110// Now, the content.
111// Check partial mapping.
112#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
113 do { \
114 assert( \
115 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
116 #Idx " is incorrectly initialized"); \
117 } while (false)
118
119 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
120 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
121 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);
122 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
123 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
124 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
125 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
126 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
127 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
128
129// Check value mapping.
130#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
131 do { \
132 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
133 PartialMappingIdx::PMI_First##RBName, Size, \
134 Offset) && \
135 #RBName #Size " " #Offset " is incorrectly initialized"); \
136 } while (false)
137
138#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
139
140 CHECK_VALUEMAP(GPR, 32);
141 CHECK_VALUEMAP(GPR, 64);
142 CHECK_VALUEMAP(GPR, 128);
143 CHECK_VALUEMAP(FPR, 16);
144 CHECK_VALUEMAP(FPR, 32);
145 CHECK_VALUEMAP(FPR, 64);
146 CHECK_VALUEMAP(FPR, 128);
147 CHECK_VALUEMAP(FPR, 256);
148 CHECK_VALUEMAP(FPR, 512);
149
150// Check the value mapping for 3-operands instructions where all the operands
151// map to the same value mapping.
152#define CHECK_VALUEMAP_3OPS(RBName, Size) \
153 do { \
154 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
155 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
156 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
157 } while (false)
158
159 CHECK_VALUEMAP_3OPS(GPR, 32);
160 CHECK_VALUEMAP_3OPS(GPR, 64);
161 CHECK_VALUEMAP_3OPS(GPR, 128);
162 CHECK_VALUEMAP_3OPS(FPR, 32);
163 CHECK_VALUEMAP_3OPS(FPR, 64);
164 CHECK_VALUEMAP_3OPS(FPR, 128);
165 CHECK_VALUEMAP_3OPS(FPR, 256);
166 CHECK_VALUEMAP_3OPS(FPR, 512);
167
168#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
169 do { \
170 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
171 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
172 (void)PartialMapDstIdx; \
173 (void)PartialMapSrcIdx; \
174 const ValueMapping *Map = getCopyMapping(AArch64::RBNameDst##RegBankID, \
175 AArch64::RBNameSrc##RegBankID, \
176 TypeSize::getFixed(Size)); \
177 (void)Map; \
178 assert(Map[0].BreakDown == \
179 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
180 Map[0].NumBreakDowns == 1 && \
181 #RBNameDst #Size " Dst is incorrectly initialized"); \
182 assert(Map[1].BreakDown == \
183 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
184 Map[1].NumBreakDowns == 1 && \
185 #RBNameSrc #Size " Src is incorrectly initialized"); \
186 \
187 } while (false)
188
189 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
190 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
191 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
192 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
193 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
194 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
195 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
196 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
197
198#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
199 do { \
200 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
201 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
202 (void)PartialMapDstIdx; \
203 (void)PartialMapSrcIdx; \
204 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
205 (void)Map; \
206 assert(Map[0].BreakDown == \
207 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
208 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
209 " Dst is incorrectly initialized"); \
210 assert(Map[1].BreakDown == \
211 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
212 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
213 " Src is incorrectly initialized"); \
214 \
215 } while (false)
216
217 CHECK_VALUEMAP_FPEXT(32, 16);
218 CHECK_VALUEMAP_FPEXT(64, 16);
219 CHECK_VALUEMAP_FPEXT(64, 32);
220 CHECK_VALUEMAP_FPEXT(128, 64);
221
222 assert(verify(TRI) && "Invalid register bank information");
223 };
224
225 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
226}
227
228unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
229 const RegisterBank &B,
230 const TypeSize Size) const {
231 // What do we do with different size?
232 // copy are same size.
233 // Will introduce other hooks for different size:
234 // * extract cost.
235 // * build_sequence cost.
236
237 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
238 // FIXME: This should be deduced from the scheduling model.
239 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
240 // FMOVXDr or FMOVWSr.
241 return 5;
242 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
243 // FMOVDXr or FMOVSWr.
244 return 4;
245
246 return RegisterBankInfo::copyCost(A, B, Size);
247}
248
249const RegisterBank &
250AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
251 LLT Ty) const {
252 switch (RC.getID()) {
253 case AArch64::GPR64sponlyRegClassID:
254 return AArch64::GPRRegBank;
255 default:
256 return AArch64GenRegisterBankInfo::getRegBankFromRegClass(RC, Ty);
257 }
258}
259
260RegisterBankInfo::InstructionMappings
261AArch64RegisterBankInfo::getInstrAlternativeMappings(
262 const MachineInstr &MI) const {
263 const MachineFunction &MF = *MI.getParent()->getParent();
264 const TargetSubtargetInfo &STI = MF.getSubtarget();
265 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
266 const MachineRegisterInfo &MRI = MF.getRegInfo();
267
268 switch (MI.getOpcode()) {
269 case TargetOpcode::G_OR: {
270 // 32 and 64-bit or can be mapped on either FPR or
271 // GPR for the same cost.
272 TypeSize Size = TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
273 if (Size != 32 && Size != 64)
274 break;
275
276 // If the instruction has any implicit-defs or uses,
277 // do not mess with it.
278 if (MI.getNumOperands() != 3)
279 break;
280 InstructionMappings AltMappings;
281 const InstructionMapping &GPRMapping = getInstructionMapping(
282 /*ID*/ 1, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstGPR, Size),
283 /*NumOperands*/ 3);
284 const InstructionMapping &FPRMapping = getInstructionMapping(
285 /*ID*/ 2, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstFPR, Size),
286 /*NumOperands*/ 3);
287
288 AltMappings.push_back(Elt: &GPRMapping);
289 AltMappings.push_back(Elt: &FPRMapping);
290 return AltMappings;
291 }
292 case TargetOpcode::G_BITCAST: {
293 TypeSize Size = TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
294 if (Size != 32 && Size != 64)
295 break;
296
297 // If the instruction has any implicit-defs or uses,
298 // do not mess with it.
299 if (MI.getNumOperands() != 2)
300 break;
301
302 InstructionMappings AltMappings;
303 const InstructionMapping &GPRMapping = getInstructionMapping(
304 /*ID*/ 1, /*Cost*/ 1,
305 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
306 /*NumOperands*/ 2);
307 const InstructionMapping &FPRMapping = getInstructionMapping(
308 /*ID*/ 2, /*Cost*/ 1,
309 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
310 /*NumOperands*/ 2);
311 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
312 /*ID*/ 3,
313 /*Cost*/
314 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
315 Size: TypeSize::getFixed(ExactSize: Size)),
316 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
317 /*NumOperands*/ 2);
318 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
319 /*ID*/ 3,
320 /*Cost*/
321 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
322 Size: TypeSize::getFixed(ExactSize: Size)),
323 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
324 /*NumOperands*/ 2);
325
326 AltMappings.push_back(Elt: &GPRMapping);
327 AltMappings.push_back(Elt: &FPRMapping);
328 AltMappings.push_back(Elt: &GPRToFPRMapping);
329 AltMappings.push_back(Elt: &FPRToGPRMapping);
330 return AltMappings;
331 }
332 case TargetOpcode::G_LOAD: {
333 TypeSize Size = TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
334 if (Size != 64)
335 break;
336
337 // If the instruction has any implicit-defs or uses,
338 // do not mess with it.
339 if (MI.getNumOperands() != 2)
340 break;
341
342 InstructionMappings AltMappings;
343 const InstructionMapping &GPRMapping = getInstructionMapping(
344 /*ID*/ 1, /*Cost*/ 1,
345 OperandsMapping: getOperandsMapping(
346 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstGPR, Size),
347 // Addresses are GPR 64-bit.
348 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
349 /*NumOperands*/ 2);
350 const InstructionMapping &FPRMapping = getInstructionMapping(
351 /*ID*/ 2, /*Cost*/ 1,
352 OperandsMapping: getOperandsMapping(
353 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstFPR, Size),
354 // Addresses are GPR 64-bit.
355 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
356 /*NumOperands*/ 2);
357
358 AltMappings.push_back(Elt: &GPRMapping);
359 AltMappings.push_back(Elt: &FPRMapping);
360 return AltMappings;
361 }
362 default:
363 break;
364 }
365 return RegisterBankInfo::getInstrAlternativeMappings(MI);
366}
367
368static bool preferGPRForFPImm(const MachineInstr &MI,
369 const MachineRegisterInfo &MRI,
370 const AArch64Subtarget &STI) {
371 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT);
372 Register Dst = MI.getOperand(i: 0).getReg();
373 LLT Ty = MRI.getType(Reg: Dst);
374
375 unsigned Size = Ty.getSizeInBits();
376 if (Size != 16 && Size != 32 && Size != 64)
377 return false;
378
379 EVT VT = EVT::getFloatingPointVT(BitWidth: Size);
380 const AArch64TargetLowering *TLI = STI.getTargetLowering();
381
382 const APFloat Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
383 const APInt ImmBits = Imm.bitcastToAPInt();
384
385 // If all the uses are stores use a gpr constant
386 if (all_of(Range: MRI.use_nodbg_instructions(Reg: Dst), P: [&](const MachineInstr &UseMI) {
387 return UseMI.getOpcode() == TargetOpcode::G_STORE &&
388 UseMI.getOperand(i: 0).getReg() == Dst;
389 }))
390 return true;
391
392 // Check if we can encode this as a movi.
393 SmallVector<AArch64_IMM::ImmInsnModel> Insn;
394 if (AArch64_IMM::expandVectorMOVImm(Imm: APInt::getSplat(NewLen: 64, V: ImmBits), Insn, ST: STI))
395 return false;
396
397 // We want to use GPR when the value cannot be encoded as the immediate value
398 // of a fmov and when it will not result in a constant pool load. As
399 // AArch64TargetLowering::isFPImmLegal is used by the instruction selector
400 // to choose whether to emit a constant pool load, negating this check will
401 // ensure it would not have become a constant pool load.
402 bool OptForSize =
403 shouldOptimizeForSize(F: &MI.getMF()->getFunction(), PSI: nullptr, BFI: nullptr);
404 bool IsLegal = TLI->isFPImmLegal(Imm, VT, ForCodeSize: OptForSize);
405 bool IsFMov = TLI->isFPImmLegalAsFMov(Imm, VT);
406 return !IsFMov && IsLegal;
407}
408
409// Some of the instructions in applyMappingImpl attempt to anyext small values.
410// It may be that these values come from a G_CONSTANT that has been expanded to
411// 32 bits and then truncated. If this is the case, we shouldn't insert an
412// anyext and should instead make use of the G_CONSTANT directly, deleting the
413// trunc if possible.
414static bool foldTruncOfI32Constant(MachineInstr &MI, unsigned OpIdx,
415 MachineRegisterInfo &MRI) {
416 MachineOperand &Op = MI.getOperand(i: OpIdx);
417
418 Register ScalarReg = Op.getReg();
419 MachineInstr *TruncMI = MRI.getVRegDef(Reg: ScalarReg);
420 if (!TruncMI || TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
421 return false;
422
423 Register TruncSrc = TruncMI->getOperand(i: 1).getReg();
424 MachineInstr *SrcDef = MRI.getVRegDef(Reg: TruncSrc);
425 if (!SrcDef || SrcDef->getOpcode() != TargetOpcode::G_CONSTANT)
426 return false;
427
428 LLT TruncSrcTy = MRI.getType(Reg: TruncSrc);
429 if (!TruncSrcTy.isScalar() || TruncSrcTy.getSizeInBits() != 32)
430 return false;
431
432 // Avoid truncating and extending a constant, this helps with selection.
433 Op.setReg(TruncSrc);
434 MRI.setRegBank(Reg: TruncSrc, RegBank: AArch64::GPRRegBank);
435
436 if (MRI.use_empty(RegNo: ScalarReg))
437 TruncMI->eraseFromParent();
438
439 return true;
440}
441
442void AArch64RegisterBankInfo::applyMappingImpl(
443 MachineIRBuilder &Builder, const OperandsMapper &OpdMapper) const {
444 MachineInstr &MI = OpdMapper.getMI();
445 MachineRegisterInfo &MRI = OpdMapper.getMRI();
446
447 switch (MI.getOpcode()) {
448 case TargetOpcode::G_CONSTANT: {
449 Register Dst = MI.getOperand(i: 0).getReg();
450 [[maybe_unused]] LLT DstTy = MRI.getType(Reg: Dst);
451 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank && DstTy.isScalar() &&
452 DstTy.getSizeInBits() < 32 &&
453 "Expected a scalar smaller than 32 bits on a GPR.");
454 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
455 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
456 Builder.buildTrunc(Res: Dst, Op: ExtReg);
457
458 APInt Val = MI.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
459 LLVMContext &Ctx = Builder.getMF().getFunction().getContext();
460 MI.getOperand(i: 1).setCImm(ConstantInt::get(Context&: Ctx, V: Val));
461 MI.getOperand(i: 0).setReg(ExtReg);
462 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
463
464 return applyDefaultMapping(OpdMapper);
465 }
466 case TargetOpcode::G_FCONSTANT: {
467 Register Dst = MI.getOperand(i: 0).getReg();
468 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank &&
469 "Expected Dst to be on a GPR.");
470 const APFloat &Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
471 APInt Bits = Imm.bitcastToAPInt();
472 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
473 if (Bits.getBitWidth() < 32) {
474 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
475 Builder.buildConstant(Res: ExtReg, Val: Bits.zext(width: 32));
476 Builder.buildTrunc(Res: Dst, Op: ExtReg);
477 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
478 } else {
479 Builder.buildConstant(Res: Dst, Val: Bits);
480 }
481 MI.eraseFromParent();
482 return;
483 }
484 case TargetOpcode::G_STORE: {
485 Register Dst = MI.getOperand(i: 0).getReg();
486 LLT Ty = MRI.getType(Reg: Dst);
487
488 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
489 Ty.getSizeInBits() < 32) {
490
491 if (foldTruncOfI32Constant(MI, OpIdx: 0, MRI))
492 return applyDefaultMapping(OpdMapper);
493
494 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
495 auto Ext = Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: Dst);
496 MI.getOperand(i: 0).setReg(Ext.getReg(Idx: 0));
497 MRI.setRegBank(Reg: Ext.getReg(Idx: 0), RegBank: AArch64::GPRRegBank);
498 }
499 return applyDefaultMapping(OpdMapper);
500 }
501 case TargetOpcode::G_LOAD: {
502 Register Dst = MI.getOperand(i: 0).getReg();
503 LLT Ty = MRI.getType(Reg: Dst);
504 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
505 Ty.getSizeInBits() < 32) {
506 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
507 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
508 Builder.buildTrunc(Res: Dst, Op: ExtReg);
509 MI.getOperand(i: 0).setReg(ExtReg);
510 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
511 }
512 [[fallthrough]];
513 }
514 case TargetOpcode::G_OR:
515 case TargetOpcode::G_BITCAST:
516 // Those ID must match getInstrAlternativeMappings.
517 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
518 OpdMapper.getInstrMapping().getID() <= 4) &&
519 "Don't know how to handle that ID");
520 return applyDefaultMapping(OpdMapper);
521 case TargetOpcode::G_SPLAT_VECTOR: {
522 if (foldTruncOfI32Constant(MI, OpIdx: 1, MRI))
523 return applyDefaultMapping(OpdMapper);
524
525 // Extend smaller gpr to 32-bits
526 assert(MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() < 32 &&
527 "Expected sources smaller than 32-bits");
528 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
529
530 Register ConstReg =
531 Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: MI.getOperand(i: 1).getReg())
532 .getReg(Idx: 0);
533 MRI.setRegBank(Reg: ConstReg, RegBank: AArch64::GPRRegBank);
534 MI.getOperand(i: 1).setReg(ConstReg);
535
536 return applyDefaultMapping(OpdMapper);
537 }
538 default:
539 llvm_unreachable("Don't know how to handle that operation");
540 }
541}
542
543const RegisterBankInfo::InstructionMapping &
544AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
545 const MachineInstr &MI) const {
546 const unsigned Opc = MI.getOpcode();
547 const MachineFunction &MF = *MI.getParent()->getParent();
548 const MachineRegisterInfo &MRI = MF.getRegInfo();
549
550 unsigned NumOperands = MI.getNumOperands();
551 assert(NumOperands <= 3 &&
552 "This code is for instructions with 3 or less operands");
553
554 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
555 TypeSize Size = Ty.getSizeInBits();
556 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
557
558 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
559
560#ifndef NDEBUG
561 // Make sure all the operands are using similar size and type.
562 // Should probably be checked by the machine verifier.
563 // This code won't catch cases where the number of lanes is
564 // different between the operands.
565 // If we want to go to that level of details, it is probably
566 // best to check that the types are the same, period.
567 // Currently, we just check that the register banks are the same
568 // for each types.
569 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
570 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
571 assert(
572 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
573 RBIdx, OpTy.getSizeInBits()) ==
574 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
575 "Operand has incompatible size");
576 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
577 (void)OpIsFPR;
578 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
579 }
580#endif // End NDEBUG.
581
582 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
583 OperandsMapping: getValueMapping(RBIdx, Size), NumOperands);
584}
585
586/// \returns true if a given intrinsic only uses and defines FPRs.
587static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
588 const MachineInstr &MI) {
589 // TODO: Add more intrinsics.
590 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
591 default:
592 return false;
593 case Intrinsic::aarch64_neon_uaddlv:
594 case Intrinsic::aarch64_neon_uaddv:
595 case Intrinsic::aarch64_neon_saddv:
596 case Intrinsic::aarch64_neon_umaxv:
597 case Intrinsic::aarch64_neon_smaxv:
598 case Intrinsic::aarch64_neon_uminv:
599 case Intrinsic::aarch64_neon_sminv:
600 case Intrinsic::aarch64_neon_faddv:
601 case Intrinsic::aarch64_neon_fmaxv:
602 case Intrinsic::aarch64_neon_fminv:
603 case Intrinsic::aarch64_neon_fmaxnmv:
604 case Intrinsic::aarch64_neon_fminnmv:
605 case Intrinsic::aarch64_neon_fmulx:
606 case Intrinsic::aarch64_neon_frecpe:
607 case Intrinsic::aarch64_neon_frecps:
608 case Intrinsic::aarch64_neon_frecpx:
609 case Intrinsic::aarch64_neon_frsqrte:
610 case Intrinsic::aarch64_neon_frsqrts:
611 case Intrinsic::aarch64_neon_facge:
612 case Intrinsic::aarch64_neon_facgt:
613 case Intrinsic::aarch64_neon_fabd:
614 case Intrinsic::aarch64_neon_sqrdmlah:
615 case Intrinsic::aarch64_neon_sqrdmlsh:
616 case Intrinsic::aarch64_neon_sqrdmulh:
617 case Intrinsic::aarch64_neon_suqadd:
618 case Intrinsic::aarch64_neon_usqadd:
619 case Intrinsic::aarch64_neon_uqadd:
620 case Intrinsic::aarch64_neon_sqadd:
621 case Intrinsic::aarch64_neon_uqsub:
622 case Intrinsic::aarch64_neon_sqsub:
623 case Intrinsic::aarch64_neon_sqdmulh:
624 case Intrinsic::aarch64_neon_sqdmulls_scalar:
625 case Intrinsic::aarch64_neon_srshl:
626 case Intrinsic::aarch64_neon_urshl:
627 case Intrinsic::aarch64_neon_sqshl:
628 case Intrinsic::aarch64_neon_uqshl:
629 case Intrinsic::aarch64_neon_sqrshl:
630 case Intrinsic::aarch64_neon_uqrshl:
631 case Intrinsic::aarch64_neon_ushl:
632 case Intrinsic::aarch64_neon_sshl:
633 case Intrinsic::aarch64_neon_sqshrn:
634 case Intrinsic::aarch64_neon_sqshrun:
635 case Intrinsic::aarch64_neon_sqrshrn:
636 case Intrinsic::aarch64_neon_sqrshrun:
637 case Intrinsic::aarch64_neon_uqshrn:
638 case Intrinsic::aarch64_neon_uqrshrn:
639 case Intrinsic::aarch64_neon_sqneg:
640 case Intrinsic::aarch64_neon_sqabs:
641 case Intrinsic::aarch64_neon_scalar_uqxtn:
642 case Intrinsic::aarch64_neon_scalar_sqxtn:
643 case Intrinsic::aarch64_neon_scalar_sqxtun:
644 case Intrinsic::aarch64_crypto_sha1h:
645 case Intrinsic::aarch64_crypto_sha1c:
646 case Intrinsic::aarch64_crypto_sha1p:
647 case Intrinsic::aarch64_crypto_sha1m:
648 case Intrinsic::aarch64_sisd_fcvtxn:
649 case Intrinsic::aarch64_sisd_fabd:
650 return true;
651 case Intrinsic::aarch64_neon_saddlv: {
652 const LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
653 return SrcTy.getElementType().getSizeInBits() >= 16 &&
654 SrcTy.getElementCount().getFixedValue() >= 4;
655 }
656 }
657}
658
659bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
660 const MachineInstr &MI, const MachineRegisterInfo &MRI,
661 const AArch64RegisterInfo &TRI, const unsigned Depth) const {
662 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
663 return false;
664
665 return any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
666 P: [&](const MachineInstr &UseMI) {
667 if (onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1))
668 return true;
669 return isPHIWithFPConstraints(MI: UseMI, MRI, TRI, Depth: Depth + 1);
670 });
671}
672
673bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
674 const MachineRegisterInfo &MRI,
675 const AArch64RegisterInfo &TRI,
676 unsigned Depth) const {
677 unsigned Op = MI.getOpcode();
678 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
679 return true;
680
681 // Do we have an explicit floating point instruction?
682 if (isPreISelGenericFloatingPointOpcode(Opc: Op))
683 return true;
684
685 // No. Check if we have a copy-like instruction. If we do, then we could
686 // still be fed by floating point instructions.
687 if (Op != TargetOpcode::COPY && !MI.isPHI() &&
688 !isPreISelGenericOptimizationHint(Opcode: Op))
689 return false;
690
691 // Check if we already know the register bank.
692 auto *RB = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
693 if (RB == &AArch64::FPRRegBank)
694 return true;
695 if (RB == &AArch64::GPRRegBank)
696 return false;
697
698 // We don't know anything.
699 //
700 // If we have a phi, we may be able to infer that it will be assigned a FPR
701 // based off of its inputs.
702 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
703 return false;
704
705 return any_of(Range: MI.explicit_uses(), P: [&](const MachineOperand &Op) {
706 return Op.isReg() &&
707 onlyDefinesFP(MI: *MRI.getVRegDef(Reg: Op.getReg()), MRI, TRI, Depth: Depth + 1);
708 });
709}
710
711bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
712 const MachineRegisterInfo &MRI,
713 const AArch64RegisterInfo &TRI,
714 unsigned Depth) const {
715 switch (MI.getOpcode()) {
716 case TargetOpcode::G_BITCAST: {
717 Register DstReg = MI.getOperand(i: 0).getReg();
718 return all_of(Range: MRI.use_nodbg_instructions(Reg: DstReg),
719 P: [&](const MachineInstr &UseMI) {
720 return onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1) ||
721 prefersFPUse(MI: UseMI, MRI, TRI);
722 });
723 }
724
725 case TargetOpcode::G_FPTOSI:
726 case TargetOpcode::G_FPTOUI:
727 case TargetOpcode::G_FPTOSI_SAT:
728 case TargetOpcode::G_FPTOUI_SAT:
729 case TargetOpcode::G_FCMP:
730 case TargetOpcode::G_LROUND:
731 case TargetOpcode::G_LLROUND:
732 case TargetOpcode::G_CLMUL:
733 case AArch64::G_PMULL:
734 case AArch64::G_SLI:
735 case AArch64::G_SRI:
736 case AArch64::G_FPTRUNC_ODD:
737 return true;
738 case TargetOpcode::G_INTRINSIC:
739 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
740 case Intrinsic::aarch64_neon_fcvtas:
741 case Intrinsic::aarch64_neon_fcvtau:
742 case Intrinsic::aarch64_neon_fcvtzs:
743 case Intrinsic::aarch64_neon_fcvtzu:
744 case Intrinsic::aarch64_neon_fcvtms:
745 case Intrinsic::aarch64_neon_fcvtmu:
746 case Intrinsic::aarch64_neon_fcvtns:
747 case Intrinsic::aarch64_neon_fcvtnu:
748 case Intrinsic::aarch64_neon_fcvtps:
749 case Intrinsic::aarch64_neon_fcvtpu:
750 return true;
751 default:
752 break;
753 }
754 break;
755 default:
756 break;
757 }
758 return hasFPConstraints(MI, MRI, TRI, Depth);
759}
760
761bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
762 const MachineRegisterInfo &MRI,
763 const AArch64RegisterInfo &TRI,
764 unsigned Depth) const {
765 switch (MI.getOpcode()) {
766 case TargetOpcode::G_SPLAT_VECTOR:
767 case AArch64::G_SADDLP:
768 case AArch64::G_UADDLP:
769 case TargetOpcode::G_SITOFP:
770 case TargetOpcode::G_UITOFP:
771 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
772 case TargetOpcode::G_INSERT_VECTOR_ELT:
773 case TargetOpcode::G_BUILD_VECTOR:
774 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
775 case AArch64::G_SLI:
776 case AArch64::G_SRI:
777 case AArch64::G_FPTRUNC_ODD:
778 return true;
779 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
780 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
781 case Intrinsic::aarch64_neon_ld1x2:
782 case Intrinsic::aarch64_neon_ld1x3:
783 case Intrinsic::aarch64_neon_ld1x4:
784 case Intrinsic::aarch64_neon_ld2:
785 case Intrinsic::aarch64_neon_ld2lane:
786 case Intrinsic::aarch64_neon_ld2r:
787 case Intrinsic::aarch64_neon_ld3:
788 case Intrinsic::aarch64_neon_ld3lane:
789 case Intrinsic::aarch64_neon_ld3r:
790 case Intrinsic::aarch64_neon_ld4:
791 case Intrinsic::aarch64_neon_ld4lane:
792 case Intrinsic::aarch64_neon_ld4r:
793 return true;
794 default:
795 break;
796 }
797 break;
798 default:
799 break;
800 }
801 return hasFPConstraints(MI, MRI, TRI, Depth);
802}
803
804bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,
805 const MachineRegisterInfo &MRI,
806 const AArch64RegisterInfo &TRI,
807 unsigned Depth) const {
808 switch (MI.getOpcode()) {
809 case TargetOpcode::G_SITOFP:
810 case TargetOpcode::G_UITOFP:
811 return MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() ==
812 MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
813 }
814 return onlyDefinesFP(MI, MRI, TRI, Depth);
815}
816
817bool AArch64RegisterBankInfo::shouldUseFPRForCvtOperand(
818 const MachineInstr &MI, bool BankedOpIsDef, bool ForceFPRForBankedOp16,
819 bool ForceFPRForOtherOp16, bool AllowFPRCVT, bool CheckBankedOpUses) const {
820 const MachineFunction &MF = *MI.getMF();
821 const MachineRegisterInfo &MRI = MF.getRegInfo();
822 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
823 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
824
825 unsigned BankedOpIdx = BankedOpIsDef ? 0 : 2;
826 unsigned OtherOpIdx = BankedOpIsDef ? 2 : 0;
827
828 Register BankedReg = MI.getOperand(i: BankedOpIdx).getReg();
829 if (MRI.getType(Reg: BankedReg).isVector())
830 return true;
831
832 TypeSize BankedSize = TRI.getRegSizeInBits(Reg: BankedReg, MRI);
833 TypeSize OtherSize =
834 TRI.getRegSizeInBits(Reg: MI.getOperand(i: OtherOpIdx).getReg(), MRI);
835 if ((ForceFPRForBankedOp16 && BankedSize == 16) ||
836 (ForceFPRForOtherOp16 && OtherSize == 16))
837 return true;
838
839 if (BankedSize != OtherSize && !AllowFPRCVT)
840 return false;
841
842 if (CheckBankedOpUses)
843 return all_of(
844 Range: MRI.use_nodbg_instructions(Reg: BankedReg), P: [&](const MachineInstr &UseMI) {
845 return onlyUsesFP(MI: UseMI, MRI, TRI) || prefersFPUse(MI: UseMI, MRI, TRI);
846 });
847
848 const MachineInstr *DefMI = MRI.getVRegDef(Reg: BankedReg);
849 return DefMI &&
850 (onlyUsesFP(MI: *DefMI, MRI, TRI) || prefersFPUse(MI: *DefMI, MRI, TRI));
851}
852
853bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
854 // GMemOperation because we also want to match indexed loads.
855 auto *MemOp = cast<GMemOperation>(Val: &MI);
856 const Value *LdVal = MemOp->getMMO().getValue();
857 if (!LdVal)
858 return false;
859
860 Type *EltTy = nullptr;
861 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: LdVal)) {
862 EltTy = GV->getValueType();
863 // Look at the first element of the struct to determine the type we are
864 // loading
865 while (StructType *StructEltTy = dyn_cast<StructType>(Val: EltTy)) {
866 if (StructEltTy->getNumElements() == 0)
867 break;
868 EltTy = StructEltTy->getTypeAtIndex(N: 0U);
869 }
870 // Look at the first element of the array to determine its type
871 if (isa<ArrayType>(Val: EltTy))
872 EltTy = EltTy->getArrayElementType();
873 } else if (!isa<Constant>(Val: LdVal)) {
874 // FIXME: grubbing around uses is pretty ugly, but with no more
875 // `getPointerElementType` there's not much else we can do.
876 for (const auto *LdUser : LdVal->users()) {
877 if (isa<LoadInst>(Val: LdUser)) {
878 EltTy = LdUser->getType();
879 break;
880 }
881 if (isa<StoreInst>(Val: LdUser) && LdUser->getOperand(i: 1) == LdVal) {
882 EltTy = LdUser->getOperand(i: 0)->getType();
883 break;
884 }
885 }
886 }
887 return EltTy && EltTy->isFPOrFPVectorTy();
888}
889
890const RegisterBankInfo::InstructionMapping &
891AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
892 const unsigned Opc = MI.getOpcode();
893
894 // Try the default logic for non-generic instructions that are either copies
895 // or already have some operands assigned to banks.
896 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opcode: Opc)) ||
897 Opc == TargetOpcode::G_PHI) {
898 const RegisterBankInfo::InstructionMapping &Mapping =
899 getInstrMappingImpl(MI);
900 if (Mapping.isValid())
901 return Mapping;
902 }
903
904 const MachineFunction &MF = *MI.getParent()->getParent();
905 const MachineRegisterInfo &MRI = MF.getRegInfo();
906 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
907 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
908
909 switch (Opc) {
910 // G_{F|S|U}REM are not listed because they are not legal.
911 // Arithmetic ops.
912 case TargetOpcode::G_ADD:
913 case TargetOpcode::G_SUB:
914 case TargetOpcode::G_PTR_ADD:
915 case TargetOpcode::G_MUL:
916 case TargetOpcode::G_SDIV:
917 case TargetOpcode::G_UDIV:
918 // Bitwise ops.
919 case TargetOpcode::G_AND:
920 case TargetOpcode::G_OR:
921 case TargetOpcode::G_XOR:
922 // Floating point ops.
923 case TargetOpcode::G_FADD:
924 case TargetOpcode::G_FSUB:
925 case TargetOpcode::G_FMUL:
926 case TargetOpcode::G_FDIV:
927 case TargetOpcode::G_FMAXIMUM:
928 case TargetOpcode::G_FMINIMUM:
929 return getSameKindOfOperandsMapping(MI);
930 case TargetOpcode::G_FPEXT: {
931 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
932 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
933 return getInstructionMapping(
934 ID: DefaultMappingID, /*Cost*/ 1,
935 OperandsMapping: getFPExtMapping(DstSize: DstTy.getSizeInBits(), SrcSize: SrcTy.getSizeInBits()),
936 /*NumOperands*/ 2);
937 }
938 // Shifts.
939 case TargetOpcode::G_SHL:
940 case TargetOpcode::G_LSHR:
941 case TargetOpcode::G_ASHR: {
942 LLT ShiftAmtTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
943 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
944 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
945 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
946 OperandsMapping: &ValMappings[Shift64Imm], NumOperands: 3);
947 return getSameKindOfOperandsMapping(MI);
948 }
949 case TargetOpcode::G_BITCAST: {
950 Register SrcReg = MI.getOperand(i: 1).getReg();
951 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
952 if (SrcRB) {
953 TypeSize Size = TRI.getRegSizeInBits(Reg: SrcReg, MRI);
954 return getInstructionMapping(
955 ID: DefaultMappingID, Cost: 0,
956 OperandsMapping: getCopyMapping(DstBankID: SrcRB->getID(), SrcBankID: SrcRB->getID(), Size),
957 // We only care about the mapping of the destination.
958 /*NumOperands=*/2);
959 }
960 [[fallthrough]];
961 }
962 case TargetOpcode::COPY: {
963 Register DstReg = MI.getOperand(i: 0).getReg();
964 Register SrcReg = MI.getOperand(i: 1).getReg();
965 // Check if one of the register is not a generic register.
966 if ((DstReg.isPhysical() || !MRI.getType(Reg: DstReg).isValid()) ||
967 (SrcReg.isPhysical() || !MRI.getType(Reg: SrcReg).isValid())) {
968 const RegisterBank *DstRB = getRegBank(Reg: DstReg, MRI, TRI);
969 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
970 if (!DstRB)
971 DstRB = SrcRB;
972 else if (!SrcRB)
973 SrcRB = DstRB;
974 // If both RB are null that means both registers are generic.
975 // We shouldn't be here.
976 assert(DstRB && SrcRB && "Both RegBank were nullptr");
977 TypeSize Size = TRI.getRegSizeInBits(Reg: DstReg, MRI);
978 return getInstructionMapping(
979 ID: DefaultMappingID, Cost: copyCost(A: *DstRB, B: *SrcRB, Size),
980 OperandsMapping: getCopyMapping(DstBankID: DstRB->getID(), SrcBankID: SrcRB->getID(), Size),
981 // We only care about the mapping of the destination.
982 /*NumOperands*/ 1);
983 }
984 // Both registers are generic
985 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
986 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
987 TypeSize Size = DstTy.getSizeInBits();
988 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
989 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
990 const RegisterBank &DstRB =
991 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
992 const RegisterBank &SrcRB =
993 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
994 return getInstructionMapping(
995 ID: DefaultMappingID, Cost: copyCost(A: DstRB, B: SrcRB, Size),
996 OperandsMapping: getCopyMapping(DstBankID: DstRB.getID(), SrcBankID: SrcRB.getID(), Size),
997 // We only care about the mapping of the destination for COPY.
998 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
999 }
1000 case TargetOpcode::G_CONSTANT: {
1001 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1002 TypeSize Size = DstTy.getSizeInBits();
1003 if (!DstTy.isPointer() && (!DstTy.isScalar() || Size < 32 || Size > 64))
1004 break;
1005 // Scalar constants materialize in GPRs.
1006 [[fallthrough]];
1007 }
1008 case TargetOpcode::G_BRCOND:
1009 case TargetOpcode::G_FRAME_INDEX: {
1010 // Operand 0 is the only banked operand and is mapped to GPR.
1011 return getInstructionMapping(
1012 ID: DefaultMappingID, /*Cost=*/1,
1013 OperandsMapping: getOperandsMapping(
1014 OpdsMapping: {getValueMapping(
1015 RBIdx: PMI_FirstGPR,
1016 Size: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits()),
1017 nullptr}),
1018 /*NumOperands=*/2);
1019 }
1020 default:
1021 break;
1022 }
1023
1024 unsigned NumOperands = MI.getNumOperands();
1025 unsigned MappingID = DefaultMappingID;
1026
1027 // Track the size and bank of each register. We don't do partial mappings.
1028 SmallVector<unsigned, 4> OpSize(NumOperands);
1029 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
1030 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1031 auto &MO = MI.getOperand(i: Idx);
1032 if (!MO.isReg() || !MO.getReg())
1033 continue;
1034
1035 LLT Ty = MRI.getType(Reg: MO.getReg());
1036 if (!Ty.isValid())
1037 continue;
1038 OpSize[Idx] = Ty.getSizeInBits().getKnownMinValue();
1039
1040 // As a top-level guess, vectors including both scalable and non-scalable
1041 // ones go in FPRs, scalars and pointers in GPRs.
1042 // For floating-point instructions, scalars go in FPRs.
1043 if (Ty.isVector())
1044 OpRegBankIdx[Idx] = PMI_FirstFPR;
1045 else if (isPreISelGenericFloatingPointOpcode(Opc) ||
1046 (MO.isDef() && onlyDefinesFP(MI, MRI, TRI)) ||
1047 (MO.isUse() && onlyUsesFP(MI, MRI, TRI)) ||
1048 Ty.getSizeInBits() > 64)
1049 OpRegBankIdx[Idx] = PMI_FirstFPR;
1050 else
1051 OpRegBankIdx[Idx] = PMI_FirstGPR;
1052 }
1053
1054 unsigned Cost = 1;
1055 // Some of the floating-point instructions have mixed GPR and FPR operands:
1056 // fine-tune the computed mapping.
1057 switch (Opc) {
1058 case TargetOpcode::G_CONSTANT: {
1059 Register Dst = MI.getOperand(i: 0).getReg();
1060 LLT DstTy = MRI.getType(Reg: Dst);
1061 if (DstTy.isScalar() && DstTy.getSizeInBits() < 32)
1062 MappingID = CustomMappingID;
1063 break;
1064 }
1065 case TargetOpcode::G_FCONSTANT: {
1066 if (preferGPRForFPImm(MI, MRI, STI)) {
1067 // Materialize in GPR and rely on later bank copies for FP uses.
1068 MappingID = CustomMappingID;
1069 OpRegBankIdx = {PMI_FirstGPR};
1070 }
1071 break;
1072 }
1073 case TargetOpcode::G_SPLAT_VECTOR: {
1074 Register ScalarReg = MI.getOperand(i: 1).getReg();
1075 LLT ScalarTy = MRI.getType(Reg: ScalarReg);
1076 auto ScalarDef = MRI.getVRegDef(Reg: ScalarReg);
1077 // We want to select dup(load) into LD1R.
1078 if (ScalarDef->getOpcode() == TargetOpcode::G_LOAD)
1079 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1080 else if (getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
1081 onlyDefinesFP(MI: *ScalarDef, MRI, TRI))
1082 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1083 else {
1084 if (ScalarTy.getSizeInBits() < 32 &&
1085 getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::GPRRegBank) {
1086 // Calls applyMappingImpl()
1087 MappingID = CustomMappingID;
1088 }
1089 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1090 }
1091 break;
1092 }
1093 case TargetOpcode::G_TRUNC: {
1094 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1095 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
1096 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1097 break;
1098 }
1099 case TargetOpcode::G_SITOFP:
1100 case TargetOpcode::G_UITOFP: {
1101 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
1102 break;
1103 // Integer to FP conversions don't necessarily happen between GPR -> FPR
1104 // regbanks. They can also be done within an FPR register.
1105 Register SrcReg = MI.getOperand(i: 1).getReg();
1106 if (getRegBank(Reg: SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&
1107 MRI.getType(Reg: SrcReg).getSizeInBits() ==
1108 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits())
1109 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1110 else
1111 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1112 break;
1113 }
1114 case TargetOpcode::G_FPTOSI_SAT:
1115 case TargetOpcode::G_FPTOUI_SAT:
1116 case TargetOpcode::G_FPTOSI:
1117 case TargetOpcode::G_FPTOUI:
1118 case TargetOpcode::G_INTRINSIC_LRINT:
1119 case TargetOpcode::G_INTRINSIC_LLRINT:
1120 case TargetOpcode::G_LROUND:
1121 case TargetOpcode::G_LLROUND: {
1122 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1123 if (DstType.isVector())
1124 break;
1125 if (DstType == LLT::scalar(SizeInBits: 16)) {
1126 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1127 break;
1128 }
1129 TypeSize DstSize = TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
1130 TypeSize SrcSize = TRI.getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
1131 if (((DstSize == SrcSize) || STI.hasFeature(Feature: AArch64::FeatureFPRCVT)) &&
1132 all_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1133 P: [&](const MachineInstr &UseMI) {
1134 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1135 prefersFPUse(MI: UseMI, MRI, TRI);
1136 }))
1137 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1138 else
1139 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
1140 break;
1141 }
1142 case TargetOpcode::G_FCMP: {
1143 // If the result is a vector, it must use a FPR.
1144 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 =
1145 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector() ? PMI_FirstFPR
1146 : PMI_FirstGPR;
1147 OpRegBankIdx = {Idx0,
1148 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
1149 break;
1150 }
1151 case TargetOpcode::G_BITCAST:
1152 // This is going to be a cross register bank copy and this is expensive.
1153 if (OpRegBankIdx[0] != OpRegBankIdx[1])
1154 Cost = copyCost(
1155 A: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
1156 B: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
1157 Size: TypeSize::getFixed(ExactSize: OpSize[0]));
1158 break;
1159 case TargetOpcode::G_LOAD: {
1160 // Loading in vector unit is slightly more expensive.
1161 // This is actually only true for the LD1R and co instructions,
1162 // but anyway for the fast mode this number does not matter and
1163 // for the greedy mode the cost of the cross bank copy will
1164 // offset this number.
1165 // FIXME: Should be derived from the scheduling model.
1166 if (OpRegBankIdx[0] != PMI_FirstGPR) {
1167 Cost = 2;
1168 break;
1169 }
1170
1171 if (cast<GLoad>(Val: MI).isAtomic()) {
1172 // Atomics always use GPR destinations. Don't refine any further.
1173 OpRegBankIdx[0] = PMI_FirstGPR;
1174 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() < 32)
1175 MappingID = CustomMappingID;
1176 break;
1177 }
1178
1179 // Try to guess the type of the load from the MMO.
1180 if (isLoadFromFPType(MI)) {
1181 OpRegBankIdx[0] = PMI_FirstFPR;
1182 break;
1183 }
1184
1185 // Check if that load feeds fp instructions.
1186 // In that case, we want the default mapping to be on FPR
1187 // instead of blind map every scalar to GPR.
1188 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1189 P: [&](const MachineInstr &UseMI) {
1190 // If we have at least one direct or indirect use
1191 // in a FP instruction,
1192 // assume this was a floating point load in the IR. If it was
1193 // not, we would have had a bitcast before reaching that
1194 // instruction.
1195 //
1196 // Int->FP conversion operations are also captured in
1197 // prefersFPUse().
1198
1199 if (isPHIWithFPConstraints(MI: UseMI, MRI, TRI))
1200 return true;
1201
1202 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1203 prefersFPUse(MI: UseMI, MRI, TRI);
1204 }))
1205 OpRegBankIdx[0] = PMI_FirstFPR;
1206
1207 // On GPR, extend any load < 32bits to 32bit.
1208 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1209 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1210 MappingID = CustomMappingID;
1211 break;
1212 }
1213 case TargetOpcode::G_STORE:
1214 // Check if that store is fed by fp instructions.
1215 if (OpRegBankIdx[0] == PMI_FirstGPR) {
1216 Register VReg = MI.getOperand(i: 0).getReg();
1217 if (VReg) {
1218 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1219 if (onlyDefinesFP(MI: *DefMI, MRI, TRI)) {
1220 OpRegBankIdx[0] = PMI_FirstFPR;
1221 break;
1222 }
1223 }
1224
1225 // On GPR, extend any store < 32bits to 32bit.
1226 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1227 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1228 MappingID = CustomMappingID;
1229 }
1230 break;
1231 case TargetOpcode::G_INDEXED_STORE:
1232 if (OpRegBankIdx[1] == PMI_FirstGPR) {
1233 Register VReg = MI.getOperand(i: 1).getReg();
1234 if (!VReg)
1235 break;
1236 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1237 if (onlyDefinesFP(MI: *DefMI, MRI, TRI))
1238 OpRegBankIdx[1] = PMI_FirstFPR;
1239 break;
1240 }
1241 break;
1242 case TargetOpcode::G_INDEXED_SEXTLOAD:
1243 case TargetOpcode::G_INDEXED_ZEXTLOAD:
1244 // These should always be GPR.
1245 OpRegBankIdx[0] = PMI_FirstGPR;
1246 break;
1247 case TargetOpcode::G_INDEXED_LOAD: {
1248 if (isLoadFromFPType(MI))
1249 OpRegBankIdx[0] = PMI_FirstFPR;
1250 break;
1251 }
1252 case TargetOpcode::G_SELECT: {
1253 // If the destination is FPR, preserve that.
1254 if (OpRegBankIdx[0] != PMI_FirstGPR)
1255 break;
1256
1257 // If we're taking in vectors, we have no choice but to put everything on
1258 // FPRs, except for the condition. The condition must always be on a GPR.
1259 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1260 if (SrcTy.isVector()) {
1261 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1262 break;
1263 }
1264
1265 // Try to minimize the number of copies. If we have more floating point
1266 // constrained values than not, then we'll put everything on FPR. Otherwise,
1267 // everything has to be on GPR.
1268 unsigned NumFP = 0;
1269
1270 // Check if the uses of the result always produce floating point values.
1271 //
1272 // For example:
1273 //
1274 // %z = G_SELECT %cond %x %y
1275 // fpr = G_FOO %z ...
1276 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1277 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
1278 ++NumFP;
1279
1280 // Check if the defs of the source values always produce floating point
1281 // values.
1282 //
1283 // For example:
1284 //
1285 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
1286 // %z = G_SELECT %cond %x %y
1287 //
1288 // Also check whether or not the sources have already been decided to be
1289 // FPR. Keep track of this.
1290 //
1291 // This doesn't check the condition, since it's just whatever is in NZCV.
1292 // This isn't passed explicitly in a register to fcsel/csel.
1293 for (unsigned Idx = 2; Idx < 4; ++Idx) {
1294 Register VReg = MI.getOperand(i: Idx).getReg();
1295 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1296 if (getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank ||
1297 onlyDefinesFP(MI: *DefMI, MRI, TRI))
1298 ++NumFP;
1299 }
1300
1301 // If we have more FP constraints than not, then move everything over to
1302 // FPR.
1303 if (NumFP >= 2)
1304 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1305
1306 break;
1307 }
1308 case TargetOpcode::G_UNMERGE_VALUES: {
1309 // If the first operand belongs to a FPR register bank, then make sure that
1310 // we preserve that.
1311 if (OpRegBankIdx[0] != PMI_FirstGPR)
1312 break;
1313
1314 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: MI.getNumOperands()-1).getReg());
1315 // UNMERGE into scalars from a vector should always use FPR.
1316 // Likewise if any of the uses are FP instructions.
1317 if (SrcTy.isVector() || SrcTy == LLT::scalar(SizeInBits: 128) ||
1318 any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1319 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
1320 // Set the register bank of every operand to FPR.
1321 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
1322 Idx < NumOperands; ++Idx)
1323 OpRegBankIdx[Idx] = PMI_FirstFPR;
1324 }
1325 break;
1326 }
1327 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1328 // Destination and source need to be FPRs.
1329 OpRegBankIdx[0] = PMI_FirstFPR;
1330 OpRegBankIdx[1] = PMI_FirstFPR;
1331
1332 // Index needs to be a GPR.
1333 OpRegBankIdx[2] = PMI_FirstGPR;
1334 break;
1335 case AArch64::G_SQSHLU_I:
1336 // Destination and source need to be FPRs.
1337 OpRegBankIdx[0] = PMI_FirstFPR;
1338 OpRegBankIdx[1] = PMI_FirstFPR;
1339
1340 // Shift Index needs to be a GPR.
1341 OpRegBankIdx[2] = PMI_FirstGPR;
1342 break;
1343
1344 case TargetOpcode::G_INSERT_VECTOR_ELT:
1345 OpRegBankIdx[0] = PMI_FirstFPR;
1346 OpRegBankIdx[1] = PMI_FirstFPR;
1347
1348 // The element may be either a GPR or FPR. Preserve that behaviour.
1349 if (getRegBank(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
1350 OpRegBankIdx[2] = PMI_FirstFPR;
1351 else {
1352 OpRegBankIdx[2] = PMI_FirstGPR;
1353 }
1354
1355 // Index needs to be a GPR.
1356 OpRegBankIdx[3] = PMI_FirstGPR;
1357 break;
1358 case TargetOpcode::G_EXTRACT: {
1359 // For s128 sources we have to use fpr unless we know otherwise.
1360 auto Src = MI.getOperand(i: 1).getReg();
1361 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1362 if (SrcTy.getSizeInBits() != 128)
1363 break;
1364 auto Idx = MRI.getRegClassOrNull(Reg: Src) == &AArch64::XSeqPairsClassRegClass
1365 ? PMI_FirstGPR
1366 : PMI_FirstFPR;
1367 OpRegBankIdx[0] = Idx;
1368 OpRegBankIdx[1] = Idx;
1369 break;
1370 }
1371 case TargetOpcode::G_BUILD_VECTOR: {
1372 // If the first source operand belongs to a FPR register bank, then make
1373 // sure that we preserve that.
1374 if (OpRegBankIdx[1] != PMI_FirstGPR)
1375 break;
1376 Register VReg = MI.getOperand(i: 1).getReg();
1377 if (!VReg)
1378 break;
1379
1380 // Get the instruction that defined the source operand reg, and check if
1381 // it's a floating point operation. Or, if it's a type like s16 which
1382 // doesn't have a exact size gpr register class. The exception is if the
1383 // build_vector has all constant operands, which may be better to leave as
1384 // gpr without copies, so it can be matched in imported patterns.
1385 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1386 unsigned DefOpc = DefMI->getOpcode();
1387 const LLT SrcTy = MRI.getType(Reg: VReg);
1388 if (all_of(Range: MI.operands(), P: [&](const MachineOperand &Op) {
1389 APInt Cst;
1390 return Op.isDef() || mi_match(R: Op.getReg(), MRI, P: m_ICst(Cst));
1391 }))
1392 break;
1393 if (isPreISelGenericFloatingPointOpcode(Opc: DefOpc) ||
1394 SrcTy.getSizeInBits() < 32 ||
1395 getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank) {
1396 // Have a floating point op.
1397 // Make sure every operand gets mapped to a FPR register class.
1398 unsigned NumOperands = MI.getNumOperands();
1399 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
1400 OpRegBankIdx[Idx] = PMI_FirstFPR;
1401 }
1402 break;
1403 }
1404 case TargetOpcode::G_VECREDUCE_FADD:
1405 case TargetOpcode::G_VECREDUCE_FMUL:
1406 case TargetOpcode::G_VECREDUCE_FMAX:
1407 case TargetOpcode::G_VECREDUCE_FMIN:
1408 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
1409 case TargetOpcode::G_VECREDUCE_FMINIMUM:
1410 case TargetOpcode::G_VECREDUCE_ADD:
1411 case TargetOpcode::G_VECREDUCE_MUL:
1412 case TargetOpcode::G_VECREDUCE_AND:
1413 case TargetOpcode::G_VECREDUCE_OR:
1414 case TargetOpcode::G_VECREDUCE_XOR:
1415 case TargetOpcode::G_VECREDUCE_SMAX:
1416 case TargetOpcode::G_VECREDUCE_SMIN:
1417 case TargetOpcode::G_VECREDUCE_UMAX:
1418 case TargetOpcode::G_VECREDUCE_UMIN:
1419 // Reductions produce a scalar value from a vector, the scalar should be on
1420 // FPR bank.
1421 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1422 break;
1423 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
1424 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
1425 // These reductions also take a scalar accumulator input.
1426 // Assign them FPR for now.
1427 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};
1428 break;
1429 case TargetOpcode::G_INTRINSIC:
1430 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
1431 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
1432 case Intrinsic::aarch64_neon_fcvtas:
1433 case Intrinsic::aarch64_neon_fcvtau:
1434 case Intrinsic::aarch64_neon_fcvtzs:
1435 case Intrinsic::aarch64_neon_fcvtzu:
1436 case Intrinsic::aarch64_neon_fcvtms:
1437 case Intrinsic::aarch64_neon_fcvtmu:
1438 case Intrinsic::aarch64_neon_fcvtns:
1439 case Intrinsic::aarch64_neon_fcvtnu:
1440 case Intrinsic::aarch64_neon_fcvtps:
1441 case Intrinsic::aarch64_neon_fcvtpu: {
1442 OpRegBankIdx[2] = PMI_FirstFPR;
1443 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/true,
1444 /*ForceFPRForBankedOp16=*/true,
1445 /*ForceFPRForOtherOp16=*/false,
1446 /*AllowFPRCVT=*/STI.hasFPRCVT(),
1447 /*CheckBankedOpUses=*/true))
1448 OpRegBankIdx[0] = PMI_FirstFPR;
1449 else
1450 OpRegBankIdx[0] = PMI_FirstGPR;
1451 break;
1452 }
1453 case Intrinsic::aarch64_neon_vcvtfxs2fp:
1454 case Intrinsic::aarch64_neon_vcvtfxu2fp: {
1455 OpRegBankIdx[0] = PMI_FirstFPR;
1456 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/false,
1457 /*ForceFPRForBankedOp16=*/false,
1458 /*ForceFPRForOtherOp16=*/true,
1459 /*AllowFPRCVT=*/false,
1460 /*CheckBankedOpUses=*/false))
1461 OpRegBankIdx[2] = PMI_FirstFPR;
1462 else
1463 OpRegBankIdx[2] = PMI_FirstGPR;
1464 break;
1465 }
1466 case Intrinsic::aarch64_neon_vcvtfp2fxs:
1467 case Intrinsic::aarch64_neon_vcvtfp2fxu: {
1468 OpRegBankIdx[2] = PMI_FirstFPR;
1469 if (shouldUseFPRForCvtOperand(MI, /*BankedOpIsDef=*/true,
1470 /*ForceFPRForBankedOp16=*/false,
1471 /*ForceFPRForOtherOp16=*/true,
1472 /*AllowFPRCVT=*/false,
1473 /*CheckBankedOpUses=*/true))
1474 OpRegBankIdx[0] = PMI_FirstFPR;
1475 else
1476 OpRegBankIdx[0] = PMI_FirstGPR;
1477 break;
1478 }
1479 default: {
1480 // Check if we know that the intrinsic has any constraints on its register
1481 // banks. If it does, then update the mapping accordingly.
1482 unsigned Idx = 0;
1483 if (onlyDefinesFP(MI, MRI, TRI))
1484 for (const auto &Op : MI.defs()) {
1485 if (Op.isReg())
1486 OpRegBankIdx[Idx] = PMI_FirstFPR;
1487 ++Idx;
1488 }
1489 else
1490 Idx += MI.getNumExplicitDefs();
1491
1492 if (onlyUsesFP(MI, MRI, TRI))
1493 for (const auto &Op : MI.explicit_uses()) {
1494 if (Op.isReg())
1495 OpRegBankIdx[Idx] = PMI_FirstFPR;
1496 ++Idx;
1497 }
1498 break;
1499 }
1500 }
1501 break;
1502 }
1503 }
1504
1505 // Finally construct the computed mapping.
1506 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
1507 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1508 if (MI.getOperand(i: Idx).isReg() && MI.getOperand(i: Idx).getReg()) {
1509 LLT Ty = MRI.getType(Reg: MI.getOperand(i: Idx).getReg());
1510 if (!Ty.isValid())
1511 continue;
1512 auto Mapping =
1513 getValueMapping(RBIdx: OpRegBankIdx[Idx], Size: TypeSize::getFixed(ExactSize: OpSize[Idx]));
1514 if (!Mapping->isValid())
1515 return getInvalidInstructionMapping();
1516
1517 OpdsMapping[Idx] = Mapping;
1518 }
1519 }
1520
1521 return getInstructionMapping(ID: MappingID, Cost, OperandsMapping: getOperandsMapping(OpdsMapping),
1522 NumOperands);
1523}
1524