1//===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64RegisterBankInfo.h"
15#include "AArch64ExpandImm.h"
16#include "AArch64RegisterInfo.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCTargetDesc.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/LowLevelTypeUtils.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineOperand.h"
31#include "llvm/CodeGen/MachineRegisterInfo.h"
32#include "llvm/CodeGen/MachineSizeOpts.h"
33#include "llvm/CodeGen/RegisterBank.h"
34#include "llvm/CodeGen/RegisterBankInfo.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/CodeGen/TargetSubtargetInfo.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/IntrinsicsAArch64.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/Threading.h"
42#include <cassert>
43
44#define GET_TARGET_REGBANK_IMPL
45#include "AArch64GenRegisterBank.inc"
46
47// This file will be TableGen'ed at some point.
48#include "AArch64GenRegisterBankInfo.def"
49
50using namespace llvm;
51using namespace MIPatternMatch;
52static const unsigned CustomMappingID = 1;
53
54AArch64RegisterBankInfo::AArch64RegisterBankInfo(
55 const TargetRegisterInfo &TRI) {
56 static llvm::once_flag InitializeRegisterBankFlag;
57
58 static auto InitializeRegisterBankOnce = [&]() {
59 // We have only one set of register banks, whatever the subtarget
60 // is. Therefore, the initialization of the RegBanks table should be
61 // done only once. Indeed the table of all register banks
62 // (AArch64::RegBanks) is unique in the compiler. At some point, it
63 // will get tablegen'ed and the whole constructor becomes empty.
64
65 const RegisterBank &RBGPR = getRegBank(ID: AArch64::GPRRegBankID);
66 (void)RBGPR;
67 assert(&AArch64::GPRRegBank == &RBGPR &&
68 "The order in RegBanks is messed up");
69
70 const RegisterBank &RBFPR = getRegBank(ID: AArch64::FPRRegBankID);
71 (void)RBFPR;
72 assert(&AArch64::FPRRegBank == &RBFPR &&
73 "The order in RegBanks is messed up");
74
75 const RegisterBank &RBCCR = getRegBank(ID: AArch64::CCRegBankID);
76 (void)RBCCR;
77 assert(&AArch64::CCRegBank == &RBCCR &&
78 "The order in RegBanks is messed up");
79
80 // The GPR register bank is fully defined by all the registers in
81 // GR64all + its subclasses.
82 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
83 "Subclass not added?");
84 assert(getMaximumSize(RBGPR.getID()) == 128 &&
85 "GPRs should hold up to 128-bit");
86
87 // The FPR register bank is fully defined by all the registers in
88 // GR64all + its subclasses.
89 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
90 "Subclass not added?");
91 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
92 "Subclass not added?");
93 assert(getMaximumSize(RBFPR.getID()) == 512 &&
94 "FPRs should hold up to 512-bit via QQQQ sequence");
95
96 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
97 "Class not added?");
98 assert(getMaximumSize(RBCCR.getID()) == 32 &&
99 "CCR should hold up to 32-bit");
100
101 // Check that the TableGen'ed like file is in sync we our expectations.
102 // First, the Idx.
103 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
104 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) &&
105 "PartialMappingIdx's are incorrectly ordered");
106 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
107 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
108 PMI_FPR256, PMI_FPR512}) &&
109 "PartialMappingIdx's are incorrectly ordered");
110// Now, the content.
111// Check partial mapping.
112#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
113 do { \
114 assert( \
115 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
116 #Idx " is incorrectly initialized"); \
117 } while (false)
118
119 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
120 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
121 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);
122 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
123 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
124 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
125 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
126 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
127 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
128
129// Check value mapping.
130#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
131 do { \
132 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
133 PartialMappingIdx::PMI_First##RBName, Size, \
134 Offset) && \
135 #RBName #Size " " #Offset " is incorrectly initialized"); \
136 } while (false)
137
138#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
139
140 CHECK_VALUEMAP(GPR, 32);
141 CHECK_VALUEMAP(GPR, 64);
142 CHECK_VALUEMAP(GPR, 128);
143 CHECK_VALUEMAP(FPR, 16);
144 CHECK_VALUEMAP(FPR, 32);
145 CHECK_VALUEMAP(FPR, 64);
146 CHECK_VALUEMAP(FPR, 128);
147 CHECK_VALUEMAP(FPR, 256);
148 CHECK_VALUEMAP(FPR, 512);
149
150// Check the value mapping for 3-operands instructions where all the operands
151// map to the same value mapping.
152#define CHECK_VALUEMAP_3OPS(RBName, Size) \
153 do { \
154 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
155 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
156 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
157 } while (false)
158
159 CHECK_VALUEMAP_3OPS(GPR, 32);
160 CHECK_VALUEMAP_3OPS(GPR, 64);
161 CHECK_VALUEMAP_3OPS(GPR, 128);
162 CHECK_VALUEMAP_3OPS(FPR, 32);
163 CHECK_VALUEMAP_3OPS(FPR, 64);
164 CHECK_VALUEMAP_3OPS(FPR, 128);
165 CHECK_VALUEMAP_3OPS(FPR, 256);
166 CHECK_VALUEMAP_3OPS(FPR, 512);
167
168#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
169 do { \
170 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
171 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
172 (void)PartialMapDstIdx; \
173 (void)PartialMapSrcIdx; \
174 const ValueMapping *Map = getCopyMapping(AArch64::RBNameDst##RegBankID, \
175 AArch64::RBNameSrc##RegBankID, \
176 TypeSize::getFixed(Size)); \
177 (void)Map; \
178 assert(Map[0].BreakDown == \
179 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
180 Map[0].NumBreakDowns == 1 && \
181 #RBNameDst #Size " Dst is incorrectly initialized"); \
182 assert(Map[1].BreakDown == \
183 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
184 Map[1].NumBreakDowns == 1 && \
185 #RBNameSrc #Size " Src is incorrectly initialized"); \
186 \
187 } while (false)
188
189 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
190 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
191 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
192 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
193 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
194 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
195 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
196 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
197
198#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
199 do { \
200 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
201 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
202 (void)PartialMapDstIdx; \
203 (void)PartialMapSrcIdx; \
204 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
205 (void)Map; \
206 assert(Map[0].BreakDown == \
207 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
208 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
209 " Dst is incorrectly initialized"); \
210 assert(Map[1].BreakDown == \
211 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
212 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
213 " Src is incorrectly initialized"); \
214 \
215 } while (false)
216
217 CHECK_VALUEMAP_FPEXT(32, 16);
218 CHECK_VALUEMAP_FPEXT(64, 16);
219 CHECK_VALUEMAP_FPEXT(64, 32);
220 CHECK_VALUEMAP_FPEXT(128, 64);
221
222 assert(verify(TRI) && "Invalid register bank information");
223 };
224
225 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
226}
227
228unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
229 const RegisterBank &B,
230 const TypeSize Size) const {
231 // What do we do with different size?
232 // copy are same size.
233 // Will introduce other hooks for different size:
234 // * extract cost.
235 // * build_sequence cost.
236
237 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
238 // FIXME: This should be deduced from the scheduling model.
239 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
240 // FMOVXDr or FMOVWSr.
241 return 5;
242 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
243 // FMOVDXr or FMOVSWr.
244 return 4;
245
246 return RegisterBankInfo::copyCost(A, B, Size);
247}
248
249const RegisterBank &
250AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
251 LLT Ty) const {
252 switch (RC.getID()) {
253 case AArch64::GPR64sponlyRegClassID:
254 return AArch64::GPRRegBank;
255 default:
256 return AArch64GenRegisterBankInfo::getRegBankFromRegClass(RC, Ty);
257 }
258}
259
260RegisterBankInfo::InstructionMappings
261AArch64RegisterBankInfo::getInstrAlternativeMappings(
262 const MachineInstr &MI) const {
263 const MachineFunction &MF = *MI.getParent()->getParent();
264 const TargetSubtargetInfo &STI = MF.getSubtarget();
265 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
266 const MachineRegisterInfo &MRI = MF.getRegInfo();
267
268 switch (MI.getOpcode()) {
269 case TargetOpcode::G_OR: {
270 // 32 and 64-bit or can be mapped on either FPR or
271 // GPR for the same cost.
272 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
273 if (Size != 32 && Size != 64)
274 break;
275
276 // If the instruction has any implicit-defs or uses,
277 // do not mess with it.
278 if (MI.getNumOperands() != 3)
279 break;
280 InstructionMappings AltMappings;
281 const InstructionMapping &GPRMapping = getInstructionMapping(
282 /*ID*/ 1, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstGPR, Size),
283 /*NumOperands*/ 3);
284 const InstructionMapping &FPRMapping = getInstructionMapping(
285 /*ID*/ 2, /*Cost*/ 1, OperandsMapping: getValueMapping(RBIdx: PMI_FirstFPR, Size),
286 /*NumOperands*/ 3);
287
288 AltMappings.push_back(Elt: &GPRMapping);
289 AltMappings.push_back(Elt: &FPRMapping);
290 return AltMappings;
291 }
292 case TargetOpcode::G_BITCAST: {
293 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
294 if (Size != 32 && Size != 64)
295 break;
296
297 // If the instruction has any implicit-defs or uses,
298 // do not mess with it.
299 if (MI.getNumOperands() != 2)
300 break;
301
302 InstructionMappings AltMappings;
303 const InstructionMapping &GPRMapping = getInstructionMapping(
304 /*ID*/ 1, /*Cost*/ 1,
305 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
306 /*NumOperands*/ 2);
307 const InstructionMapping &FPRMapping = getInstructionMapping(
308 /*ID*/ 2, /*Cost*/ 1,
309 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
310 /*NumOperands*/ 2);
311 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
312 /*ID*/ 3,
313 /*Cost*/
314 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
315 Size: TypeSize::getFixed(ExactSize: Size)),
316 OperandsMapping: getCopyMapping(DstBankID: AArch64::FPRRegBankID, SrcBankID: AArch64::GPRRegBankID, Size),
317 /*NumOperands*/ 2);
318 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
319 /*ID*/ 3,
320 /*Cost*/
321 copyCost(A: AArch64::GPRRegBank, B: AArch64::FPRRegBank,
322 Size: TypeSize::getFixed(ExactSize: Size)),
323 OperandsMapping: getCopyMapping(DstBankID: AArch64::GPRRegBankID, SrcBankID: AArch64::FPRRegBankID, Size),
324 /*NumOperands*/ 2);
325
326 AltMappings.push_back(Elt: &GPRMapping);
327 AltMappings.push_back(Elt: &FPRMapping);
328 AltMappings.push_back(Elt: &GPRToFPRMapping);
329 AltMappings.push_back(Elt: &FPRToGPRMapping);
330 return AltMappings;
331 }
332 case TargetOpcode::G_LOAD: {
333 TypeSize Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
334 if (Size != 64)
335 break;
336
337 // If the instruction has any implicit-defs or uses,
338 // do not mess with it.
339 if (MI.getNumOperands() != 2)
340 break;
341
342 InstructionMappings AltMappings;
343 const InstructionMapping &GPRMapping = getInstructionMapping(
344 /*ID*/ 1, /*Cost*/ 1,
345 OperandsMapping: getOperandsMapping(
346 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstGPR, Size),
347 // Addresses are GPR 64-bit.
348 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
349 /*NumOperands*/ 2);
350 const InstructionMapping &FPRMapping = getInstructionMapping(
351 /*ID*/ 2, /*Cost*/ 1,
352 OperandsMapping: getOperandsMapping(
353 OpdsMapping: {getValueMapping(RBIdx: PMI_FirstFPR, Size),
354 // Addresses are GPR 64-bit.
355 getValueMapping(RBIdx: PMI_FirstGPR, Size: TypeSize::getFixed(ExactSize: 64))}),
356 /*NumOperands*/ 2);
357
358 AltMappings.push_back(Elt: &GPRMapping);
359 AltMappings.push_back(Elt: &FPRMapping);
360 return AltMappings;
361 }
362 default:
363 break;
364 }
365 return RegisterBankInfo::getInstrAlternativeMappings(MI);
366}
367
368static bool preferGPRForFPImm(const MachineInstr &MI,
369 const MachineRegisterInfo &MRI,
370 const AArch64Subtarget &STI) {
371 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT);
372 Register Dst = MI.getOperand(i: 0).getReg();
373 LLT Ty = MRI.getType(Reg: Dst);
374
375 unsigned Size = Ty.getSizeInBits();
376 if (Size != 16 && Size != 32 && Size != 64)
377 return false;
378
379 EVT VT = EVT::getFloatingPointVT(BitWidth: Size);
380 const AArch64TargetLowering *TLI = STI.getTargetLowering();
381
382 const APFloat Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
383 const APInt ImmBits = Imm.bitcastToAPInt();
384
385 // If all the uses are stores use a gpr constant
386 if (all_of(Range: MRI.use_nodbg_instructions(Reg: Dst), P: [&](const MachineInstr &UseMI) {
387 return UseMI.getOpcode() == TargetOpcode::G_STORE &&
388 UseMI.getOperand(i: 0).getReg() == Dst;
389 }))
390 return true;
391
392 // Check if we can encode this as a movi. Note, we only have one pattern so
393 // far for movis, hence the one check.
394 if (Size == 32) {
395 uint64_t Val = APInt::getSplat(NewLen: 64, V: ImmBits).getZExtValue();
396 if (AArch64_AM::isAdvSIMDModImmType4(Imm: Val))
397 return false;
398 }
399
400 // We want to use GPR when the value cannot be encoded as the immediate value
401 // of a fmov and when it will not result in a constant pool load. As
402 // AArch64TargetLowering::isFPImmLegal is used by the instruction selector
403 // to choose whether to emit a constant pool load, negating this check will
404 // ensure it would not have become a constant pool load.
405 bool OptForSize =
406 shouldOptimizeForSize(F: &MI.getMF()->getFunction(), PSI: nullptr, BFI: nullptr);
407 bool IsLegal = TLI->isFPImmLegal(Imm, VT, ForCodeSize: OptForSize);
408 bool IsFMov = TLI->isFPImmLegalAsFMov(Imm, VT);
409 return !IsFMov && IsLegal;
410}
411
412// Some of the instructions in applyMappingImpl attempt to anyext small values.
413// It may be that these values come from a G_CONSTANT that has been expanded to
414// 32 bits and then truncated. If this is the case, we shouldn't insert an
415// anyext and should instead make use of the G_CONSTANT directly, deleting the
416// trunc if possible.
417static bool foldTruncOfI32Constant(MachineInstr &MI, unsigned OpIdx,
418 MachineRegisterInfo &MRI) {
419 MachineOperand &Op = MI.getOperand(i: OpIdx);
420
421 Register ScalarReg = Op.getReg();
422 MachineInstr *TruncMI = MRI.getVRegDef(Reg: ScalarReg);
423 if (!TruncMI || TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
424 return false;
425
426 Register TruncSrc = TruncMI->getOperand(i: 1).getReg();
427 MachineInstr *SrcDef = MRI.getVRegDef(Reg: TruncSrc);
428 if (!SrcDef || SrcDef->getOpcode() != TargetOpcode::G_CONSTANT)
429 return false;
430
431 LLT TruncSrcTy = MRI.getType(Reg: TruncSrc);
432 if (!TruncSrcTy.isScalar() || TruncSrcTy.getSizeInBits() != 32)
433 return false;
434
435 // Avoid truncating and extending a constant, this helps with selection.
436 Op.setReg(TruncSrc);
437 MRI.setRegBank(Reg: TruncSrc, RegBank: AArch64::GPRRegBank);
438
439 if (MRI.use_empty(RegNo: ScalarReg))
440 TruncMI->eraseFromParent();
441
442 return true;
443}
444
445void AArch64RegisterBankInfo::applyMappingImpl(
446 MachineIRBuilder &Builder, const OperandsMapper &OpdMapper) const {
447 MachineInstr &MI = OpdMapper.getMI();
448 MachineRegisterInfo &MRI = OpdMapper.getMRI();
449
450 switch (MI.getOpcode()) {
451 case TargetOpcode::G_CONSTANT: {
452 Register Dst = MI.getOperand(i: 0).getReg();
453 [[maybe_unused]] LLT DstTy = MRI.getType(Reg: Dst);
454 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank && DstTy.isScalar() &&
455 DstTy.getSizeInBits() < 32 &&
456 "Expected a scalar smaller than 32 bits on a GPR.");
457 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
458 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
459 Builder.buildTrunc(Res: Dst, Op: ExtReg);
460
461 APInt Val = MI.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
462 LLVMContext &Ctx = Builder.getMF().getFunction().getContext();
463 MI.getOperand(i: 1).setCImm(ConstantInt::get(Context&: Ctx, V: Val));
464 MI.getOperand(i: 0).setReg(ExtReg);
465 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
466
467 return applyDefaultMapping(OpdMapper);
468 }
469 case TargetOpcode::G_FCONSTANT: {
470 Register Dst = MI.getOperand(i: 0).getReg();
471 assert(MRI.getRegBank(Dst) == &AArch64::GPRRegBank &&
472 "Expected Dst to be on a GPR.");
473 const APFloat &Imm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
474 APInt Bits = Imm.bitcastToAPInt();
475 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
476 if (Bits.getBitWidth() < 32) {
477 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
478 Builder.buildConstant(Res: ExtReg, Val: Bits.zext(width: 32));
479 Builder.buildTrunc(Res: Dst, Op: ExtReg);
480 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
481 } else {
482 Builder.buildConstant(Res: Dst, Val: Bits);
483 }
484 MI.eraseFromParent();
485 return;
486 }
487 case TargetOpcode::G_STORE: {
488 Register Dst = MI.getOperand(i: 0).getReg();
489 LLT Ty = MRI.getType(Reg: Dst);
490
491 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
492 Ty.getSizeInBits() < 32) {
493
494 if (foldTruncOfI32Constant(MI, OpIdx: 0, MRI))
495 return applyDefaultMapping(OpdMapper);
496
497 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
498 auto Ext = Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: Dst);
499 MI.getOperand(i: 0).setReg(Ext.getReg(Idx: 0));
500 MRI.setRegBank(Reg: Ext.getReg(Idx: 0), RegBank: AArch64::GPRRegBank);
501 }
502 return applyDefaultMapping(OpdMapper);
503 }
504 case TargetOpcode::G_LOAD: {
505 Register Dst = MI.getOperand(i: 0).getReg();
506 LLT Ty = MRI.getType(Reg: Dst);
507 if (MRI.getRegBank(Reg: Dst) == &AArch64::GPRRegBank && Ty.isScalar() &&
508 Ty.getSizeInBits() < 32) {
509 Builder.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
510 Register ExtReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
511 Builder.buildTrunc(Res: Dst, Op: ExtReg);
512 MI.getOperand(i: 0).setReg(ExtReg);
513 MRI.setRegBank(Reg: ExtReg, RegBank: AArch64::GPRRegBank);
514 }
515 [[fallthrough]];
516 }
517 case TargetOpcode::G_OR:
518 case TargetOpcode::G_BITCAST:
519 // Those ID must match getInstrAlternativeMappings.
520 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
521 OpdMapper.getInstrMapping().getID() <= 4) &&
522 "Don't know how to handle that ID");
523 return applyDefaultMapping(OpdMapper);
524 case AArch64::G_DUP: {
525 if (foldTruncOfI32Constant(MI, OpIdx: 1, MRI))
526 return applyDefaultMapping(OpdMapper);
527
528 // Extend smaller gpr to 32-bits
529 assert(MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() < 32 &&
530 "Expected sources smaller than 32-bits");
531 Builder.setInsertPt(MBB&: *MI.getParent(), II: MI.getIterator());
532
533 Register ConstReg =
534 Builder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: MI.getOperand(i: 1).getReg())
535 .getReg(Idx: 0);
536 MRI.setRegBank(Reg: ConstReg, RegBank: AArch64::GPRRegBank);
537 MI.getOperand(i: 1).setReg(ConstReg);
538
539 return applyDefaultMapping(OpdMapper);
540 }
541 default:
542 llvm_unreachable("Don't know how to handle that operation");
543 }
544}
545
546const RegisterBankInfo::InstructionMapping &
547AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
548 const MachineInstr &MI) const {
549 const unsigned Opc = MI.getOpcode();
550 const MachineFunction &MF = *MI.getParent()->getParent();
551 const MachineRegisterInfo &MRI = MF.getRegInfo();
552
553 unsigned NumOperands = MI.getNumOperands();
554 assert(NumOperands <= 3 &&
555 "This code is for instructions with 3 or less operands");
556
557 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
558 TypeSize Size = Ty.getSizeInBits();
559 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
560
561 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
562
563#ifndef NDEBUG
564 // Make sure all the operands are using similar size and type.
565 // Should probably be checked by the machine verifier.
566 // This code won't catch cases where the number of lanes is
567 // different between the operands.
568 // If we want to go to that level of details, it is probably
569 // best to check that the types are the same, period.
570 // Currently, we just check that the register banks are the same
571 // for each types.
572 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
573 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
574 assert(
575 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
576 RBIdx, OpTy.getSizeInBits()) ==
577 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
578 "Operand has incompatible size");
579 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
580 (void)OpIsFPR;
581 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
582 }
583#endif // End NDEBUG.
584
585 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
586 OperandsMapping: getValueMapping(RBIdx, Size), NumOperands);
587}
588
589/// \returns true if a given intrinsic only uses and defines FPRs.
590static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
591 const MachineInstr &MI) {
592 // TODO: Add more intrinsics.
593 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
594 default:
595 return false;
596 case Intrinsic::aarch64_neon_uaddlv:
597 case Intrinsic::aarch64_neon_uaddv:
598 case Intrinsic::aarch64_neon_saddv:
599 case Intrinsic::aarch64_neon_umaxv:
600 case Intrinsic::aarch64_neon_smaxv:
601 case Intrinsic::aarch64_neon_uminv:
602 case Intrinsic::aarch64_neon_sminv:
603 case Intrinsic::aarch64_neon_faddv:
604 case Intrinsic::aarch64_neon_fmaxv:
605 case Intrinsic::aarch64_neon_fminv:
606 case Intrinsic::aarch64_neon_fmaxnmv:
607 case Intrinsic::aarch64_neon_fminnmv:
608 case Intrinsic::aarch64_neon_fmulx:
609 case Intrinsic::aarch64_neon_frecpe:
610 case Intrinsic::aarch64_neon_frecps:
611 case Intrinsic::aarch64_neon_frecpx:
612 case Intrinsic::aarch64_neon_frsqrte:
613 case Intrinsic::aarch64_neon_frsqrts:
614 case Intrinsic::aarch64_neon_facge:
615 case Intrinsic::aarch64_neon_facgt:
616 case Intrinsic::aarch64_neon_fabd:
617 case Intrinsic::aarch64_neon_sqrdmlah:
618 case Intrinsic::aarch64_neon_sqrdmlsh:
619 case Intrinsic::aarch64_neon_sqrdmulh:
620 case Intrinsic::aarch64_neon_suqadd:
621 case Intrinsic::aarch64_neon_usqadd:
622 case Intrinsic::aarch64_neon_uqadd:
623 case Intrinsic::aarch64_neon_sqadd:
624 case Intrinsic::aarch64_neon_uqsub:
625 case Intrinsic::aarch64_neon_sqsub:
626 case Intrinsic::aarch64_neon_sqdmulh:
627 case Intrinsic::aarch64_neon_sqdmulls_scalar:
628 case Intrinsic::aarch64_neon_srshl:
629 case Intrinsic::aarch64_neon_urshl:
630 case Intrinsic::aarch64_neon_sqshl:
631 case Intrinsic::aarch64_neon_uqshl:
632 case Intrinsic::aarch64_neon_sqrshl:
633 case Intrinsic::aarch64_neon_uqrshl:
634 case Intrinsic::aarch64_neon_ushl:
635 case Intrinsic::aarch64_neon_sshl:
636 case Intrinsic::aarch64_neon_sqshrn:
637 case Intrinsic::aarch64_neon_sqshrun:
638 case Intrinsic::aarch64_neon_sqrshrn:
639 case Intrinsic::aarch64_neon_sqrshrun:
640 case Intrinsic::aarch64_neon_uqshrn:
641 case Intrinsic::aarch64_neon_uqrshrn:
642 case Intrinsic::aarch64_neon_sqneg:
643 case Intrinsic::aarch64_neon_sqabs:
644 case Intrinsic::aarch64_neon_scalar_uqxtn:
645 case Intrinsic::aarch64_neon_scalar_sqxtn:
646 case Intrinsic::aarch64_neon_scalar_sqxtun:
647 case Intrinsic::aarch64_crypto_sha1h:
648 case Intrinsic::aarch64_crypto_sha1c:
649 case Intrinsic::aarch64_crypto_sha1p:
650 case Intrinsic::aarch64_crypto_sha1m:
651 case Intrinsic::aarch64_sisd_fcvtxn:
652 case Intrinsic::aarch64_sisd_fabd:
653 return true;
654 case Intrinsic::aarch64_neon_saddlv: {
655 const LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
656 return SrcTy.getElementType().getSizeInBits() >= 16 &&
657 SrcTy.getElementCount().getFixedValue() >= 4;
658 }
659 }
660}
661
662bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
663 const MachineInstr &MI, const MachineRegisterInfo &MRI,
664 const AArch64RegisterInfo &TRI, const unsigned Depth) const {
665 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
666 return false;
667
668 return any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
669 P: [&](const MachineInstr &UseMI) {
670 if (onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1))
671 return true;
672 return isPHIWithFPConstraints(MI: UseMI, MRI, TRI, Depth: Depth + 1);
673 });
674}
675
676bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
677 const MachineRegisterInfo &MRI,
678 const AArch64RegisterInfo &TRI,
679 unsigned Depth) const {
680 unsigned Op = MI.getOpcode();
681 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
682 return true;
683
684 // Do we have an explicit floating point instruction?
685 if (isPreISelGenericFloatingPointOpcode(Opc: Op))
686 return true;
687
688 // No. Check if we have a copy-like instruction. If we do, then we could
689 // still be fed by floating point instructions.
690 if (Op != TargetOpcode::COPY && !MI.isPHI() &&
691 !isPreISelGenericOptimizationHint(Opcode: Op))
692 return false;
693
694 // Check if we already know the register bank.
695 auto *RB = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
696 if (RB == &AArch64::FPRRegBank)
697 return true;
698 if (RB == &AArch64::GPRRegBank)
699 return false;
700
701 // We don't know anything.
702 //
703 // If we have a phi, we may be able to infer that it will be assigned a FPR
704 // based off of its inputs.
705 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
706 return false;
707
708 return any_of(Range: MI.explicit_uses(), P: [&](const MachineOperand &Op) {
709 return Op.isReg() &&
710 onlyDefinesFP(MI: *MRI.getVRegDef(Reg: Op.getReg()), MRI, TRI, Depth: Depth + 1);
711 });
712}
713
714bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
715 const MachineRegisterInfo &MRI,
716 const AArch64RegisterInfo &TRI,
717 unsigned Depth) const {
718 switch (MI.getOpcode()) {
719 case TargetOpcode::G_BITCAST: {
720 Register DstReg = MI.getOperand(i: 0).getReg();
721 return all_of(Range: MRI.use_nodbg_instructions(Reg: DstReg),
722 P: [&](const MachineInstr &UseMI) {
723 return onlyUsesFP(MI: UseMI, MRI, TRI, Depth: Depth + 1) ||
724 prefersFPUse(MI: UseMI, MRI, TRI);
725 });
726 }
727
728 case TargetOpcode::G_FPTOSI:
729 case TargetOpcode::G_FPTOUI:
730 case TargetOpcode::G_FPTOSI_SAT:
731 case TargetOpcode::G_FPTOUI_SAT:
732 case TargetOpcode::G_FCMP:
733 case TargetOpcode::G_LROUND:
734 case TargetOpcode::G_LLROUND:
735 case TargetOpcode::G_CLMUL:
736 case AArch64::G_PMULL:
737 case AArch64::G_SLI:
738 case AArch64::G_SRI:
739 case AArch64::G_FPTRUNC_ODD:
740 return true;
741 case TargetOpcode::G_INTRINSIC:
742 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
743 case Intrinsic::aarch64_neon_fcvtas:
744 case Intrinsic::aarch64_neon_fcvtau:
745 case Intrinsic::aarch64_neon_fcvtzs:
746 case Intrinsic::aarch64_neon_fcvtzu:
747 case Intrinsic::aarch64_neon_fcvtms:
748 case Intrinsic::aarch64_neon_fcvtmu:
749 case Intrinsic::aarch64_neon_fcvtns:
750 case Intrinsic::aarch64_neon_fcvtnu:
751 case Intrinsic::aarch64_neon_fcvtps:
752 case Intrinsic::aarch64_neon_fcvtpu:
753 return true;
754 default:
755 break;
756 }
757 break;
758 default:
759 break;
760 }
761 return hasFPConstraints(MI, MRI, TRI, Depth);
762}
763
764bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
765 const MachineRegisterInfo &MRI,
766 const AArch64RegisterInfo &TRI,
767 unsigned Depth) const {
768 switch (MI.getOpcode()) {
769 case AArch64::G_DUP:
770 case AArch64::G_SADDLP:
771 case AArch64::G_UADDLP:
772 case TargetOpcode::G_SITOFP:
773 case TargetOpcode::G_UITOFP:
774 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
775 case TargetOpcode::G_INSERT_VECTOR_ELT:
776 case TargetOpcode::G_BUILD_VECTOR:
777 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
778 case AArch64::G_SLI:
779 case AArch64::G_SRI:
780 case AArch64::G_FPTRUNC_ODD:
781 return true;
782 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
783 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
784 case Intrinsic::aarch64_neon_ld1x2:
785 case Intrinsic::aarch64_neon_ld1x3:
786 case Intrinsic::aarch64_neon_ld1x4:
787 case Intrinsic::aarch64_neon_ld2:
788 case Intrinsic::aarch64_neon_ld2lane:
789 case Intrinsic::aarch64_neon_ld2r:
790 case Intrinsic::aarch64_neon_ld3:
791 case Intrinsic::aarch64_neon_ld3lane:
792 case Intrinsic::aarch64_neon_ld3r:
793 case Intrinsic::aarch64_neon_ld4:
794 case Intrinsic::aarch64_neon_ld4lane:
795 case Intrinsic::aarch64_neon_ld4r:
796 return true;
797 default:
798 break;
799 }
800 break;
801 default:
802 break;
803 }
804 return hasFPConstraints(MI, MRI, TRI, Depth);
805}
806
807bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,
808 const MachineRegisterInfo &MRI,
809 const AArch64RegisterInfo &TRI,
810 unsigned Depth) const {
811 switch (MI.getOpcode()) {
812 case TargetOpcode::G_SITOFP:
813 case TargetOpcode::G_UITOFP:
814 return MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() ==
815 MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
816 }
817 return onlyDefinesFP(MI, MRI, TRI, Depth);
818}
819
820bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
821 // GMemOperation because we also want to match indexed loads.
822 auto *MemOp = cast<GMemOperation>(Val: &MI);
823 const Value *LdVal = MemOp->getMMO().getValue();
824 if (!LdVal)
825 return false;
826
827 Type *EltTy = nullptr;
828 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: LdVal)) {
829 EltTy = GV->getValueType();
830 // Look at the first element of the struct to determine the type we are
831 // loading
832 while (StructType *StructEltTy = dyn_cast<StructType>(Val: EltTy)) {
833 if (StructEltTy->getNumElements() == 0)
834 break;
835 EltTy = StructEltTy->getTypeAtIndex(N: 0U);
836 }
837 // Look at the first element of the array to determine its type
838 if (isa<ArrayType>(Val: EltTy))
839 EltTy = EltTy->getArrayElementType();
840 } else if (!isa<Constant>(Val: LdVal)) {
841 // FIXME: grubbing around uses is pretty ugly, but with no more
842 // `getPointerElementType` there's not much else we can do.
843 for (const auto *LdUser : LdVal->users()) {
844 if (isa<LoadInst>(Val: LdUser)) {
845 EltTy = LdUser->getType();
846 break;
847 }
848 if (isa<StoreInst>(Val: LdUser) && LdUser->getOperand(i: 1) == LdVal) {
849 EltTy = LdUser->getOperand(i: 0)->getType();
850 break;
851 }
852 }
853 }
854 return EltTy && EltTy->isFPOrFPVectorTy();
855}
856
857const RegisterBankInfo::InstructionMapping &
858AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
859 const unsigned Opc = MI.getOpcode();
860
861 // Try the default logic for non-generic instructions that are either copies
862 // or already have some operands assigned to banks.
863 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opcode: Opc)) ||
864 Opc == TargetOpcode::G_PHI) {
865 const RegisterBankInfo::InstructionMapping &Mapping =
866 getInstrMappingImpl(MI);
867 if (Mapping.isValid())
868 return Mapping;
869 }
870
871 const MachineFunction &MF = *MI.getParent()->getParent();
872 const MachineRegisterInfo &MRI = MF.getRegInfo();
873 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
874 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
875
876 switch (Opc) {
877 // G_{F|S|U}REM are not listed because they are not legal.
878 // Arithmetic ops.
879 case TargetOpcode::G_ADD:
880 case TargetOpcode::G_SUB:
881 case TargetOpcode::G_PTR_ADD:
882 case TargetOpcode::G_MUL:
883 case TargetOpcode::G_SDIV:
884 case TargetOpcode::G_UDIV:
885 // Bitwise ops.
886 case TargetOpcode::G_AND:
887 case TargetOpcode::G_OR:
888 case TargetOpcode::G_XOR:
889 // Floating point ops.
890 case TargetOpcode::G_FADD:
891 case TargetOpcode::G_FSUB:
892 case TargetOpcode::G_FMUL:
893 case TargetOpcode::G_FDIV:
894 case TargetOpcode::G_FMAXIMUM:
895 case TargetOpcode::G_FMINIMUM:
896 return getSameKindOfOperandsMapping(MI);
897 case TargetOpcode::G_FPEXT: {
898 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
899 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
900 return getInstructionMapping(
901 ID: DefaultMappingID, /*Cost*/ 1,
902 OperandsMapping: getFPExtMapping(DstSize: DstTy.getSizeInBits(), SrcSize: SrcTy.getSizeInBits()),
903 /*NumOperands*/ 2);
904 }
905 // Shifts.
906 case TargetOpcode::G_SHL:
907 case TargetOpcode::G_LSHR:
908 case TargetOpcode::G_ASHR: {
909 LLT ShiftAmtTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
910 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
911 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
912 return getInstructionMapping(ID: DefaultMappingID, Cost: 1,
913 OperandsMapping: &ValMappings[Shift64Imm], NumOperands: 3);
914 return getSameKindOfOperandsMapping(MI);
915 }
916 case TargetOpcode::G_BITCAST: {
917 Register SrcReg = MI.getOperand(i: 1).getReg();
918 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
919 if (SrcRB) {
920 TypeSize Size = getSizeInBits(Reg: SrcReg, MRI, TRI);
921 return getInstructionMapping(
922 ID: DefaultMappingID, Cost: 0,
923 OperandsMapping: getCopyMapping(DstBankID: SrcRB->getID(), SrcBankID: SrcRB->getID(), Size),
924 // We only care about the mapping of the destination.
925 /*NumOperands=*/2);
926 }
927 [[fallthrough]];
928 }
929 case TargetOpcode::COPY: {
930 Register DstReg = MI.getOperand(i: 0).getReg();
931 Register SrcReg = MI.getOperand(i: 1).getReg();
932 // Check if one of the register is not a generic register.
933 if ((DstReg.isPhysical() || !MRI.getType(Reg: DstReg).isValid()) ||
934 (SrcReg.isPhysical() || !MRI.getType(Reg: SrcReg).isValid())) {
935 const RegisterBank *DstRB = getRegBank(Reg: DstReg, MRI, TRI);
936 const RegisterBank *SrcRB = getRegBank(Reg: SrcReg, MRI, TRI);
937 if (!DstRB)
938 DstRB = SrcRB;
939 else if (!SrcRB)
940 SrcRB = DstRB;
941 // If both RB are null that means both registers are generic.
942 // We shouldn't be here.
943 assert(DstRB && SrcRB && "Both RegBank were nullptr");
944 TypeSize Size = getSizeInBits(Reg: DstReg, MRI, TRI);
945 return getInstructionMapping(
946 ID: DefaultMappingID, Cost: copyCost(A: *DstRB, B: *SrcRB, Size),
947 OperandsMapping: getCopyMapping(DstBankID: DstRB->getID(), SrcBankID: SrcRB->getID(), Size),
948 // We only care about the mapping of the destination.
949 /*NumOperands*/ 1);
950 }
951 // Both registers are generic
952 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
953 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
954 TypeSize Size = DstTy.getSizeInBits();
955 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
956 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
957 const RegisterBank &DstRB =
958 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
959 const RegisterBank &SrcRB =
960 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
961 return getInstructionMapping(
962 ID: DefaultMappingID, Cost: copyCost(A: DstRB, B: SrcRB, Size),
963 OperandsMapping: getCopyMapping(DstBankID: DstRB.getID(), SrcBankID: SrcRB.getID(), Size),
964 // We only care about the mapping of the destination for COPY.
965 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
966 }
967 case TargetOpcode::G_CONSTANT: {
968 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
969 TypeSize Size = DstTy.getSizeInBits();
970 if (!DstTy.isPointer() && (!DstTy.isScalar() || Size < 32 || Size > 64))
971 break;
972 // Scalar constants materialize in GPRs.
973 [[fallthrough]];
974 }
975 case TargetOpcode::G_BRCOND:
976 case TargetOpcode::G_FRAME_INDEX: {
977 // Operand 0 is the only banked operand and is mapped to GPR.
978 return getInstructionMapping(
979 ID: DefaultMappingID, /*Cost=*/1,
980 OperandsMapping: getOperandsMapping(
981 OpdsMapping: {getValueMapping(
982 RBIdx: PMI_FirstGPR,
983 Size: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits()),
984 nullptr}),
985 /*NumOperands=*/2);
986 }
987 default:
988 break;
989 }
990
991 unsigned NumOperands = MI.getNumOperands();
992 unsigned MappingID = DefaultMappingID;
993
994 // Track the size and bank of each register. We don't do partial mappings.
995 SmallVector<unsigned, 4> OpSize(NumOperands);
996 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
997 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
998 auto &MO = MI.getOperand(i: Idx);
999 if (!MO.isReg() || !MO.getReg())
1000 continue;
1001
1002 LLT Ty = MRI.getType(Reg: MO.getReg());
1003 if (!Ty.isValid())
1004 continue;
1005 OpSize[Idx] = Ty.getSizeInBits().getKnownMinValue();
1006
1007 // As a top-level guess, vectors including both scalable and non-scalable
1008 // ones go in FPRs, scalars and pointers in GPRs.
1009 // For floating-point instructions, scalars go in FPRs.
1010 if (Ty.isVector())
1011 OpRegBankIdx[Idx] = PMI_FirstFPR;
1012 else if (isPreISelGenericFloatingPointOpcode(Opc) ||
1013 (MO.isDef() && onlyDefinesFP(MI, MRI, TRI)) ||
1014 (MO.isUse() && onlyUsesFP(MI, MRI, TRI)) ||
1015 Ty.getSizeInBits() > 64)
1016 OpRegBankIdx[Idx] = PMI_FirstFPR;
1017 else
1018 OpRegBankIdx[Idx] = PMI_FirstGPR;
1019 }
1020
1021 unsigned Cost = 1;
1022 // Some of the floating-point instructions have mixed GPR and FPR operands:
1023 // fine-tune the computed mapping.
1024 switch (Opc) {
1025 case TargetOpcode::G_CONSTANT: {
1026 Register Dst = MI.getOperand(i: 0).getReg();
1027 LLT DstTy = MRI.getType(Reg: Dst);
1028 if (DstTy.isScalar() && DstTy.getSizeInBits() < 32)
1029 MappingID = CustomMappingID;
1030 break;
1031 }
1032 case TargetOpcode::G_FCONSTANT: {
1033 if (preferGPRForFPImm(MI, MRI, STI)) {
1034 // Materialize in GPR and rely on later bank copies for FP uses.
1035 MappingID = CustomMappingID;
1036 OpRegBankIdx = {PMI_FirstGPR};
1037 }
1038 break;
1039 }
1040 case AArch64::G_DUP: {
1041 Register ScalarReg = MI.getOperand(i: 1).getReg();
1042 LLT ScalarTy = MRI.getType(Reg: ScalarReg);
1043 auto ScalarDef = MRI.getVRegDef(Reg: ScalarReg);
1044 // We want to select dup(load) into LD1R.
1045 if (ScalarDef->getOpcode() == TargetOpcode::G_LOAD)
1046 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1047 // s8 is an exception for G_DUP, which we always want on gpr.
1048 else if (ScalarTy.getSizeInBits() != 8 &&
1049 (getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
1050 onlyDefinesFP(MI: *ScalarDef, MRI, TRI)))
1051 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1052 else {
1053 if (ScalarTy.getSizeInBits() < 32 &&
1054 getRegBank(Reg: ScalarReg, MRI, TRI) == &AArch64::GPRRegBank) {
1055 // Calls applyMappingImpl()
1056 MappingID = CustomMappingID;
1057 }
1058 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1059 }
1060 break;
1061 }
1062 case TargetOpcode::G_TRUNC: {
1063 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1064 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
1065 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1066 break;
1067 }
1068 case TargetOpcode::G_SITOFP:
1069 case TargetOpcode::G_UITOFP: {
1070 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
1071 break;
1072 // Integer to FP conversions don't necessarily happen between GPR -> FPR
1073 // regbanks. They can also be done within an FPR register.
1074 Register SrcReg = MI.getOperand(i: 1).getReg();
1075 if (getRegBank(Reg: SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&
1076 MRI.getType(Reg: SrcReg).getSizeInBits() ==
1077 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits())
1078 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1079 else
1080 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
1081 break;
1082 }
1083 case TargetOpcode::G_FPTOSI_SAT:
1084 case TargetOpcode::G_FPTOUI_SAT:
1085 case TargetOpcode::G_FPTOSI:
1086 case TargetOpcode::G_FPTOUI:
1087 case TargetOpcode::G_INTRINSIC_LRINT:
1088 case TargetOpcode::G_INTRINSIC_LLRINT:
1089 case TargetOpcode::G_LROUND:
1090 case TargetOpcode::G_LLROUND: {
1091 LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1092 if (DstType.isVector())
1093 break;
1094 if (DstType == LLT::scalar(SizeInBits: 16)) {
1095 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1096 break;
1097 }
1098 TypeSize DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
1099 TypeSize SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI);
1100 if (((DstSize == SrcSize) || STI.hasFeature(Feature: AArch64::FeatureFPRCVT)) &&
1101 all_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1102 P: [&](const MachineInstr &UseMI) {
1103 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1104 prefersFPUse(MI: UseMI, MRI, TRI);
1105 }))
1106 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1107 else
1108 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
1109 break;
1110 }
1111 case TargetOpcode::G_FCMP: {
1112 // If the result is a vector, it must use a FPR.
1113 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 =
1114 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector() ? PMI_FirstFPR
1115 : PMI_FirstGPR;
1116 OpRegBankIdx = {Idx0,
1117 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
1118 break;
1119 }
1120 case TargetOpcode::G_BITCAST:
1121 // This is going to be a cross register bank copy and this is expensive.
1122 if (OpRegBankIdx[0] != OpRegBankIdx[1])
1123 Cost = copyCost(
1124 A: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
1125 B: *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
1126 Size: TypeSize::getFixed(ExactSize: OpSize[0]));
1127 break;
1128 case TargetOpcode::G_LOAD: {
1129 // Loading in vector unit is slightly more expensive.
1130 // This is actually only true for the LD1R and co instructions,
1131 // but anyway for the fast mode this number does not matter and
1132 // for the greedy mode the cost of the cross bank copy will
1133 // offset this number.
1134 // FIXME: Should be derived from the scheduling model.
1135 if (OpRegBankIdx[0] != PMI_FirstGPR) {
1136 Cost = 2;
1137 break;
1138 }
1139
1140 if (cast<GLoad>(Val: MI).isAtomic()) {
1141 // Atomics always use GPR destinations. Don't refine any further.
1142 OpRegBankIdx[0] = PMI_FirstGPR;
1143 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() < 32)
1144 MappingID = CustomMappingID;
1145 break;
1146 }
1147
1148 // Try to guess the type of the load from the MMO.
1149 if (isLoadFromFPType(MI)) {
1150 OpRegBankIdx[0] = PMI_FirstFPR;
1151 break;
1152 }
1153
1154 // Check if that load feeds fp instructions.
1155 // In that case, we want the default mapping to be on FPR
1156 // instead of blind map every scalar to GPR.
1157 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1158 P: [&](const MachineInstr &UseMI) {
1159 // If we have at least one direct or indirect use
1160 // in a FP instruction,
1161 // assume this was a floating point load in the IR. If it was
1162 // not, we would have had a bitcast before reaching that
1163 // instruction.
1164 //
1165 // Int->FP conversion operations are also captured in
1166 // prefersFPUse().
1167
1168 if (isPHIWithFPConstraints(MI: UseMI, MRI, TRI))
1169 return true;
1170
1171 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1172 prefersFPUse(MI: UseMI, MRI, TRI);
1173 }))
1174 OpRegBankIdx[0] = PMI_FirstFPR;
1175
1176 // On GPR, extend any load < 32bits to 32bit.
1177 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1178 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1179 MappingID = CustomMappingID;
1180 break;
1181 }
1182 case TargetOpcode::G_STORE:
1183 // Check if that store is fed by fp instructions.
1184 if (OpRegBankIdx[0] == PMI_FirstGPR) {
1185 Register VReg = MI.getOperand(i: 0).getReg();
1186 if (VReg) {
1187 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1188 if (onlyDefinesFP(MI: *DefMI, MRI, TRI)) {
1189 OpRegBankIdx[0] = PMI_FirstFPR;
1190 break;
1191 }
1192 }
1193
1194 // On GPR, extend any store < 32bits to 32bit.
1195 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1196 if (Ty.isScalar() && Ty.getSizeInBits() < 32)
1197 MappingID = CustomMappingID;
1198 }
1199 break;
1200 case TargetOpcode::G_INDEXED_STORE:
1201 if (OpRegBankIdx[1] == PMI_FirstGPR) {
1202 Register VReg = MI.getOperand(i: 1).getReg();
1203 if (!VReg)
1204 break;
1205 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1206 if (onlyDefinesFP(MI: *DefMI, MRI, TRI))
1207 OpRegBankIdx[1] = PMI_FirstFPR;
1208 break;
1209 }
1210 break;
1211 case TargetOpcode::G_INDEXED_SEXTLOAD:
1212 case TargetOpcode::G_INDEXED_ZEXTLOAD:
1213 // These should always be GPR.
1214 OpRegBankIdx[0] = PMI_FirstGPR;
1215 break;
1216 case TargetOpcode::G_INDEXED_LOAD: {
1217 if (isLoadFromFPType(MI))
1218 OpRegBankIdx[0] = PMI_FirstFPR;
1219 break;
1220 }
1221 case TargetOpcode::G_SELECT: {
1222 // If the destination is FPR, preserve that.
1223 if (OpRegBankIdx[0] != PMI_FirstGPR)
1224 break;
1225
1226 // If we're taking in vectors, we have no choice but to put everything on
1227 // FPRs, except for the condition. The condition must always be on a GPR.
1228 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
1229 if (SrcTy.isVector()) {
1230 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1231 break;
1232 }
1233
1234 // Try to minimize the number of copies. If we have more floating point
1235 // constrained values than not, then we'll put everything on FPR. Otherwise,
1236 // everything has to be on GPR.
1237 unsigned NumFP = 0;
1238
1239 // Check if the uses of the result always produce floating point values.
1240 //
1241 // For example:
1242 //
1243 // %z = G_SELECT %cond %x %y
1244 // fpr = G_FOO %z ...
1245 if (any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1246 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
1247 ++NumFP;
1248
1249 // Check if the defs of the source values always produce floating point
1250 // values.
1251 //
1252 // For example:
1253 //
1254 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
1255 // %z = G_SELECT %cond %x %y
1256 //
1257 // Also check whether or not the sources have already been decided to be
1258 // FPR. Keep track of this.
1259 //
1260 // This doesn't check the condition, since it's just whatever is in NZCV.
1261 // This isn't passed explicitly in a register to fcsel/csel.
1262 for (unsigned Idx = 2; Idx < 4; ++Idx) {
1263 Register VReg = MI.getOperand(i: Idx).getReg();
1264 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1265 if (getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank ||
1266 onlyDefinesFP(MI: *DefMI, MRI, TRI))
1267 ++NumFP;
1268 }
1269
1270 // If we have more FP constraints than not, then move everything over to
1271 // FPR.
1272 if (NumFP >= 2)
1273 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
1274
1275 break;
1276 }
1277 case TargetOpcode::G_UNMERGE_VALUES: {
1278 // If the first operand belongs to a FPR register bank, then make sure that
1279 // we preserve that.
1280 if (OpRegBankIdx[0] != PMI_FirstGPR)
1281 break;
1282
1283 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: MI.getNumOperands()-1).getReg());
1284 // UNMERGE into scalars from a vector should always use FPR.
1285 // Likewise if any of the uses are FP instructions.
1286 if (SrcTy.isVector() || SrcTy == LLT::scalar(SizeInBits: 128) ||
1287 any_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1288 P: [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
1289 // Set the register bank of every operand to FPR.
1290 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
1291 Idx < NumOperands; ++Idx)
1292 OpRegBankIdx[Idx] = PMI_FirstFPR;
1293 }
1294 break;
1295 }
1296 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1297 // Destination and source need to be FPRs.
1298 OpRegBankIdx[0] = PMI_FirstFPR;
1299 OpRegBankIdx[1] = PMI_FirstFPR;
1300
1301 // Index needs to be a GPR.
1302 OpRegBankIdx[2] = PMI_FirstGPR;
1303 break;
1304 case AArch64::G_SQSHLU_I:
1305 // Destination and source need to be FPRs.
1306 OpRegBankIdx[0] = PMI_FirstFPR;
1307 OpRegBankIdx[1] = PMI_FirstFPR;
1308
1309 // Shift Index needs to be a GPR.
1310 OpRegBankIdx[2] = PMI_FirstGPR;
1311 break;
1312
1313 case TargetOpcode::G_INSERT_VECTOR_ELT:
1314 OpRegBankIdx[0] = PMI_FirstFPR;
1315 OpRegBankIdx[1] = PMI_FirstFPR;
1316
1317 // The element may be either a GPR or FPR. Preserve that behaviour.
1318 if (getRegBank(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
1319 OpRegBankIdx[2] = PMI_FirstFPR;
1320 else {
1321 OpRegBankIdx[2] = PMI_FirstGPR;
1322 }
1323
1324 // Index needs to be a GPR.
1325 OpRegBankIdx[3] = PMI_FirstGPR;
1326 break;
1327 case TargetOpcode::G_EXTRACT: {
1328 // For s128 sources we have to use fpr unless we know otherwise.
1329 auto Src = MI.getOperand(i: 1).getReg();
1330 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1331 if (SrcTy.getSizeInBits() != 128)
1332 break;
1333 auto Idx = MRI.getRegClassOrNull(Reg: Src) == &AArch64::XSeqPairsClassRegClass
1334 ? PMI_FirstGPR
1335 : PMI_FirstFPR;
1336 OpRegBankIdx[0] = Idx;
1337 OpRegBankIdx[1] = Idx;
1338 break;
1339 }
1340 case TargetOpcode::G_BUILD_VECTOR: {
1341 // If the first source operand belongs to a FPR register bank, then make
1342 // sure that we preserve that.
1343 if (OpRegBankIdx[1] != PMI_FirstGPR)
1344 break;
1345 Register VReg = MI.getOperand(i: 1).getReg();
1346 if (!VReg)
1347 break;
1348
1349 // Get the instruction that defined the source operand reg, and check if
1350 // it's a floating point operation. Or, if it's a type like s16 which
1351 // doesn't have a exact size gpr register class. The exception is if the
1352 // build_vector has all constant operands, which may be better to leave as
1353 // gpr without copies, so it can be matched in imported patterns.
1354 MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1355 unsigned DefOpc = DefMI->getOpcode();
1356 const LLT SrcTy = MRI.getType(Reg: VReg);
1357 if (all_of(Range: MI.operands(), P: [&](const MachineOperand &Op) {
1358 APInt Cst;
1359 return Op.isDef() || mi_match(R: Op.getReg(), MRI, P: m_ICst(Cst));
1360 }))
1361 break;
1362 if (isPreISelGenericFloatingPointOpcode(Opc: DefOpc) ||
1363 SrcTy.getSizeInBits() < 32 ||
1364 getRegBank(Reg: VReg, MRI, TRI) == &AArch64::FPRRegBank) {
1365 // Have a floating point op.
1366 // Make sure every operand gets mapped to a FPR register class.
1367 unsigned NumOperands = MI.getNumOperands();
1368 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
1369 OpRegBankIdx[Idx] = PMI_FirstFPR;
1370 }
1371 break;
1372 }
1373 case TargetOpcode::G_VECREDUCE_FADD:
1374 case TargetOpcode::G_VECREDUCE_FMUL:
1375 case TargetOpcode::G_VECREDUCE_FMAX:
1376 case TargetOpcode::G_VECREDUCE_FMIN:
1377 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
1378 case TargetOpcode::G_VECREDUCE_FMINIMUM:
1379 case TargetOpcode::G_VECREDUCE_ADD:
1380 case TargetOpcode::G_VECREDUCE_MUL:
1381 case TargetOpcode::G_VECREDUCE_AND:
1382 case TargetOpcode::G_VECREDUCE_OR:
1383 case TargetOpcode::G_VECREDUCE_XOR:
1384 case TargetOpcode::G_VECREDUCE_SMAX:
1385 case TargetOpcode::G_VECREDUCE_SMIN:
1386 case TargetOpcode::G_VECREDUCE_UMAX:
1387 case TargetOpcode::G_VECREDUCE_UMIN:
1388 // Reductions produce a scalar value from a vector, the scalar should be on
1389 // FPR bank.
1390 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1391 break;
1392 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
1393 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
1394 // These reductions also take a scalar accumulator input.
1395 // Assign them FPR for now.
1396 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};
1397 break;
1398 case TargetOpcode::G_INTRINSIC:
1399 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
1400 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
1401 case Intrinsic::aarch64_neon_fcvtas:
1402 case Intrinsic::aarch64_neon_fcvtau:
1403 case Intrinsic::aarch64_neon_fcvtzs:
1404 case Intrinsic::aarch64_neon_fcvtzu:
1405 case Intrinsic::aarch64_neon_fcvtms:
1406 case Intrinsic::aarch64_neon_fcvtmu:
1407 case Intrinsic::aarch64_neon_fcvtns:
1408 case Intrinsic::aarch64_neon_fcvtnu:
1409 case Intrinsic::aarch64_neon_fcvtps:
1410 case Intrinsic::aarch64_neon_fcvtpu: {
1411 OpRegBankIdx[2] = PMI_FirstFPR;
1412 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector()) {
1413 OpRegBankIdx[0] = PMI_FirstFPR;
1414 break;
1415 }
1416 TypeSize DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
1417 TypeSize SrcSize = getSizeInBits(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI);
1418 // Fp conversions to i16 must be kept on fp register banks to ensure
1419 // proper saturation, as there are no 16-bit gprs.
1420 // In addition, conversion intrinsics have fpr output when the input
1421 // size matches the output size, or FPRCVT is present.
1422 if (DstSize == 16 ||
1423 ((DstSize == SrcSize || STI.hasFeature(Feature: AArch64::FeatureFPRCVT)) &&
1424 all_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1425 P: [&](const MachineInstr &UseMI) {
1426 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1427 prefersFPUse(MI: UseMI, MRI, TRI);
1428 })))
1429 OpRegBankIdx[0] = PMI_FirstFPR;
1430 else
1431 OpRegBankIdx[0] = PMI_FirstGPR;
1432 break;
1433 }
1434 case Intrinsic::aarch64_neon_vcvtfxs2fp:
1435 case Intrinsic::aarch64_neon_vcvtfxu2fp:
1436 // Override these intrinsics, because they would have a partial
1437 // mapping. This is needed for 'half' types, which otherwise don't
1438 // get legalised correctly.
1439 OpRegBankIdx[0] = PMI_FirstFPR;
1440 OpRegBankIdx[2] = PMI_FirstFPR;
1441 // OpRegBankIdx[1] is the intrinsic ID.
1442 // OpRegBankIdx[3] is an integer immediate.
1443 break;
1444 case Intrinsic::aarch64_neon_vcvtfp2fxs:
1445 case Intrinsic::aarch64_neon_vcvtfp2fxu: {
1446 OpRegBankIdx[2] = PMI_FirstFPR;
1447 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector()) {
1448 OpRegBankIdx[0] = PMI_FirstFPR;
1449 break;
1450 }
1451
1452 TypeSize DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI);
1453 TypeSize SrcSize = getSizeInBits(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI);
1454
1455 // Half-precision fixed-point FP-to-int scalar intrinsics are specified as
1456 // producing an H-register result. The LLVM intrinsic may still return an
1457 // i32/i64 type, so check the source size for 16 bits.
1458 if (SrcSize == 16 ||
1459 ((DstSize == SrcSize) &&
1460 all_of(Range: MRI.use_nodbg_instructions(Reg: MI.getOperand(i: 0).getReg()),
1461 P: [&](const MachineInstr &UseMI) {
1462 return onlyUsesFP(MI: UseMI, MRI, TRI) ||
1463 prefersFPUse(MI: UseMI, MRI, TRI);
1464 })))
1465 OpRegBankIdx[0] = PMI_FirstFPR;
1466 else
1467 OpRegBankIdx[0] = PMI_FirstGPR;
1468 break;
1469 }
1470 default: {
1471 // Check if we know that the intrinsic has any constraints on its register
1472 // banks. If it does, then update the mapping accordingly.
1473 unsigned Idx = 0;
1474 if (onlyDefinesFP(MI, MRI, TRI))
1475 for (const auto &Op : MI.defs()) {
1476 if (Op.isReg())
1477 OpRegBankIdx[Idx] = PMI_FirstFPR;
1478 ++Idx;
1479 }
1480 else
1481 Idx += MI.getNumExplicitDefs();
1482
1483 if (onlyUsesFP(MI, MRI, TRI))
1484 for (const auto &Op : MI.explicit_uses()) {
1485 if (Op.isReg())
1486 OpRegBankIdx[Idx] = PMI_FirstFPR;
1487 ++Idx;
1488 }
1489 break;
1490 }
1491 }
1492 break;
1493 }
1494 }
1495
1496 // Finally construct the computed mapping.
1497 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
1498 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1499 if (MI.getOperand(i: Idx).isReg() && MI.getOperand(i: Idx).getReg()) {
1500 LLT Ty = MRI.getType(Reg: MI.getOperand(i: Idx).getReg());
1501 if (!Ty.isValid())
1502 continue;
1503 auto Mapping =
1504 getValueMapping(RBIdx: OpRegBankIdx[Idx], Size: TypeSize::getFixed(ExactSize: OpSize[Idx]));
1505 if (!Mapping->isValid())
1506 return getInvalidInstructionMapping();
1507
1508 OpdsMapping[Idx] = Mapping;
1509 }
1510 }
1511
1512 return getInstructionMapping(ID: MappingID, Cost, OperandsMapping: getOperandsMapping(OpdsMapping),
1513 NumOperands);
1514}
1515