1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64GlobalISelUtils.h"
15#include "AArch64InstrInfo.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterBankInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetMachine.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64MCTargetDesc.h"
23#include "llvm/BinaryFormat/Dwarf.h"
24#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
25#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
29#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
30#include "llvm/CodeGen/GlobalISel/Utils.h"
31#include "llvm/CodeGen/MachineBasicBlock.h"
32#include "llvm/CodeGen/MachineConstantPool.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstr.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/TargetOpcodes.h"
41#include "llvm/CodeGen/TargetRegisterInfo.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/Instructions.h"
45#include "llvm/IR/IntrinsicsAArch64.h"
46#include "llvm/IR/Type.h"
47#include "llvm/Pass.h"
48#include "llvm/Support/Debug.h"
49#include "llvm/Support/raw_ostream.h"
50#include <optional>
51
52#define DEBUG_TYPE "aarch64-isel"
53
54using namespace llvm;
55using namespace MIPatternMatch;
56using namespace AArch64GISelUtils;
57
58namespace llvm {
59class BlockFrequencyInfo;
60class ProfileSummaryInfo;
61}
62
63namespace {
64
65#define GET_GLOBALISEL_PREDICATE_BITSET
66#include "AArch64GenGlobalISel.inc"
67#undef GET_GLOBALISEL_PREDICATE_BITSET
68
69
70class AArch64InstructionSelector : public InstructionSelector {
71public:
72 AArch64InstructionSelector(const AArch64TargetMachine &TM,
73 const AArch64Subtarget &STI,
74 const AArch64RegisterBankInfo &RBI);
75
76 bool select(MachineInstr &I) override;
77 static const char *getName() { return DEBUG_TYPE; }
78
79 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
80 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
81 BlockFrequencyInfo *BFI) override {
82 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
83 MIB.setMF(MF);
84
85 // hasFnAttribute() is expensive to call on every BRCOND selection, so
86 // cache it here for each run of the selector.
87 ProduceNonFlagSettingCondBr =
88 !MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening);
89 MFReturnAddr = Register();
90
91 processPHIs(MF);
92 }
93
94private:
95 /// tblgen-erated 'select' implementation, used as the initial selector for
96 /// the patterns that don't require complex C++.
97 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
98
99 // A lowering phase that runs before any selection attempts.
100 // Returns true if the instruction was modified.
101 bool preISelLower(MachineInstr &I);
102
103 // An early selection function that runs before the selectImpl() call.
104 bool earlySelect(MachineInstr &I);
105
106 /// Save state that is shared between select calls, call select on \p I and
107 /// then restore the saved state. This can be used to recursively call select
108 /// within a select call.
109 bool selectAndRestoreState(MachineInstr &I);
110
111 // Do some preprocessing of G_PHIs before we begin selection.
112 void processPHIs(MachineFunction &MF);
113
114 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);
115
116 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
117 bool contractCrossBankCopyIntoStore(MachineInstr &I,
118 MachineRegisterInfo &MRI);
119
120 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
121
122 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
123 MachineRegisterInfo &MRI) const;
124 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
125 MachineRegisterInfo &MRI) const;
126
127 ///@{
128 /// Helper functions for selectCompareBranch.
129 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
130 MachineIRBuilder &MIB) const;
131 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
132 MachineIRBuilder &MIB) const;
133 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
134 MachineIRBuilder &MIB) const;
135 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
136 MachineBasicBlock *DstMBB,
137 MachineIRBuilder &MIB) const;
138 ///@}
139
140 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
141 MachineRegisterInfo &MRI);
142
143 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
144 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);
145
146 // Helper to generate an equivalent of scalar_to_vector into a new register,
147 // returned via 'Dst'.
148 MachineInstr *emitScalarToVector(unsigned EltSize,
149 const TargetRegisterClass *DstRC,
150 Register Scalar,
151 MachineIRBuilder &MIRBuilder) const;
152 /// Helper to narrow vector that was widened by emitScalarToVector.
153 /// Copy lowest part of 128-bit or 64-bit vector to 64-bit or 32-bit
154 /// vector, correspondingly.
155 MachineInstr *emitNarrowVector(Register DstReg, Register SrcReg,
156 MachineIRBuilder &MIRBuilder,
157 MachineRegisterInfo &MRI) const;
158
159 /// Emit a lane insert into \p DstReg, or a new vector register if
160 /// std::nullopt is provided.
161 ///
162 /// The lane inserted into is defined by \p LaneIdx. The vector source
163 /// register is given by \p SrcReg. The register containing the element is
164 /// given by \p EltReg.
165 MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
166 Register EltReg, unsigned LaneIdx,
167 const RegisterBank &RB,
168 MachineIRBuilder &MIRBuilder) const;
169
170 /// Emit a sequence of instructions representing a constant \p CV for a
171 /// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
172 ///
173 /// \returns the last instruction in the sequence on success, and nullptr
174 /// otherwise.
175 MachineInstr *emitConstantVector(Register Dst, Constant *CV,
176 MachineIRBuilder &MIRBuilder,
177 MachineRegisterInfo &MRI);
178
179 MachineInstr *tryAdvSIMDModImm8(Register Dst, unsigned DstSize, APInt Bits,
180 MachineIRBuilder &MIRBuilder);
181
182 MachineInstr *tryAdvSIMDModImm16(Register Dst, unsigned DstSize, APInt Bits,
183 MachineIRBuilder &MIRBuilder, bool Inv);
184
185 MachineInstr *tryAdvSIMDModImm32(Register Dst, unsigned DstSize, APInt Bits,
186 MachineIRBuilder &MIRBuilder, bool Inv);
187 MachineInstr *tryAdvSIMDModImm64(Register Dst, unsigned DstSize, APInt Bits,
188 MachineIRBuilder &MIRBuilder);
189 MachineInstr *tryAdvSIMDModImm321s(Register Dst, unsigned DstSize, APInt Bits,
190 MachineIRBuilder &MIRBuilder, bool Inv);
191 MachineInstr *tryAdvSIMDModImmFP(Register Dst, unsigned DstSize, APInt Bits,
192 MachineIRBuilder &MIRBuilder);
193
194 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
195 MachineRegisterInfo &MRI);
196 /// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
197 /// SUBREG_TO_REG.
198 bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
199 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
200 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
201 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
202
203 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
204 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
205 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
206 bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);
207
208 /// Helper function to select vector load intrinsics like
209 /// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
210 /// \p Opc is the opcode that the selected instruction should use.
211 /// \p NumVecs is the number of vector destinations for the instruction.
212 /// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
213 bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
214 MachineInstr &I);
215 bool selectVectorLoadLaneIntrinsic(unsigned Opc, unsigned NumVecs,
216 MachineInstr &I);
217 void selectVectorStoreIntrinsic(MachineInstr &I, unsigned NumVecs,
218 unsigned Opc);
219 bool selectVectorStoreLaneIntrinsic(MachineInstr &I, unsigned NumVecs,
220 unsigned Opc);
221 bool selectIntrinsicWithSideEffects(MachineInstr &I,
222 MachineRegisterInfo &MRI);
223 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
224 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
225 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
226 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
227 bool selectPtrAuthGlobalValue(MachineInstr &I,
228 MachineRegisterInfo &MRI) const;
229 bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);
230 bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
231 bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
232 void SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVecs,
233 unsigned Opc1, unsigned Opc2, bool isExt);
234
235 bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
236 bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
237 bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);
238
239 unsigned emitConstantPoolEntry(const Constant *CPVal,
240 MachineFunction &MF) const;
241 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
242 MachineIRBuilder &MIRBuilder) const;
243
244 // Emit a vector concat operation.
245 MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
246 Register Op2,
247 MachineIRBuilder &MIRBuilder) const;
248
249 // Emit an integer compare between LHS and RHS, which checks for Predicate.
250 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
251 MachineOperand &Predicate,
252 MachineIRBuilder &MIRBuilder) const;
253
254 /// Emit a floating point comparison between \p LHS and \p RHS.
255 /// \p Pred if given is the intended predicate to use.
256 MachineInstr *
257 emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
258 std::optional<CmpInst::Predicate> = std::nullopt) const;
259
260 MachineInstr *
261 emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
262 std::initializer_list<llvm::SrcOp> SrcOps,
263 MachineIRBuilder &MIRBuilder,
264 const ComplexRendererFns &RenderFns = std::nullopt) const;
265 /// Helper function to emit an add or sub instruction.
266 ///
267 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
268 /// in a specific order.
269 ///
270 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
271 ///
272 /// \code
273 /// const std::array<std::array<unsigned, 2>, 4> Table {
274 /// {{AArch64::ADDXri, AArch64::ADDWri},
275 /// {AArch64::ADDXrs, AArch64::ADDWrs},
276 /// {AArch64::ADDXrr, AArch64::ADDWrr},
277 /// {AArch64::SUBXri, AArch64::SUBWri},
278 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
279 /// \endcode
280 ///
281 /// Each row in the table corresponds to a different addressing mode. Each
282 /// column corresponds to a different register size.
283 ///
284 /// \attention Rows must be structured as follows:
285 /// - Row 0: The ri opcode variants
286 /// - Row 1: The rs opcode variants
287 /// - Row 2: The rr opcode variants
288 /// - Row 3: The ri opcode variants for negative immediates
289 /// - Row 4: The rx opcode variants
290 ///
291 /// \attention Columns must be structured as follows:
292 /// - Column 0: The 64-bit opcode variants
293 /// - Column 1: The 32-bit opcode variants
294 ///
295 /// \p Dst is the destination register of the binop to emit.
296 /// \p LHS is the left-hand operand of the binop to emit.
297 /// \p RHS is the right-hand operand of the binop to emit.
298 MachineInstr *emitAddSub(
299 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
300 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
301 MachineIRBuilder &MIRBuilder) const;
302 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
303 MachineOperand &RHS,
304 MachineIRBuilder &MIRBuilder) const;
305 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
306 MachineIRBuilder &MIRBuilder) const;
307 MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
308 MachineIRBuilder &MIRBuilder) const;
309 MachineInstr *emitADCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
310 MachineIRBuilder &MIRBuilder) const;
311 MachineInstr *emitSBCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
312 MachineIRBuilder &MIRBuilder) const;
313 MachineInstr *emitCMP(MachineOperand &LHS, MachineOperand &RHS,
314 MachineIRBuilder &MIRBuilder) const;
315 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
316 MachineIRBuilder &MIRBuilder) const;
317 MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
318 MachineIRBuilder &MIRBuilder) const;
319 MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
320 AArch64CC::CondCode CC,
321 MachineIRBuilder &MIRBuilder) const;
322 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
323 const RegisterBank &DstRB, LLT ScalarTy,
324 Register VecReg, unsigned LaneIdx,
325 MachineIRBuilder &MIRBuilder) const;
326 MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
327 AArch64CC::CondCode Pred,
328 MachineIRBuilder &MIRBuilder) const;
329 /// Emit a CSet for a FP compare.
330 ///
331 /// \p Dst is expected to be a 32-bit scalar register.
332 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
333 MachineIRBuilder &MIRBuilder) const;
334
335 /// Emit an instruction that sets NZCV to the carry-in expected by \p I.
336 /// Might elide the instruction if the previous instruction already sets NZCV
337 /// correctly.
338 MachineInstr *emitCarryIn(MachineInstr &I, Register CarryReg);
339
340 /// Emit the overflow op for \p Opcode.
341 ///
342 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
343 /// G_USUBO, etc.
344 std::pair<MachineInstr *, AArch64CC::CondCode>
345 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
346 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
347
348 bool selectOverflowOp(MachineInstr &I, MachineRegisterInfo &MRI);
349
350 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
351 /// In some cases this is even possible with OR operations in the expression.
352 MachineInstr *emitConjunction(Register Val, AArch64CC::CondCode &OutCC,
353 MachineIRBuilder &MIB) const;
354 MachineInstr *emitConditionalComparison(Register LHS, Register RHS,
355 CmpInst::Predicate CC,
356 AArch64CC::CondCode Predicate,
357 AArch64CC::CondCode OutCC,
358 MachineIRBuilder &MIB) const;
359 MachineInstr *emitConjunctionRec(Register Val, AArch64CC::CondCode &OutCC,
360 bool Negate, Register CCOp,
361 AArch64CC::CondCode Predicate,
362 MachineIRBuilder &MIB) const;
363
364 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
365 /// \p IsNegative is true if the test should be "not zero".
366 /// This will also optimize the test bit instruction when possible.
367 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
368 MachineBasicBlock *DstMBB,
369 MachineIRBuilder &MIB) const;
370
371 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
372 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
373 MachineBasicBlock *DestMBB,
374 MachineIRBuilder &MIB) const;
375
376 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
377 // We use these manually instead of using the importer since it doesn't
378 // support SDNodeXForm.
379 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
380 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
381 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
382 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
383
384 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
385 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
386 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
387
388 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
389 unsigned Size) const;
390
391 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
392 return selectAddrModeUnscaled(Root, Size: 1);
393 }
394 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
395 return selectAddrModeUnscaled(Root, Size: 2);
396 }
397 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
398 return selectAddrModeUnscaled(Root, Size: 4);
399 }
400 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
401 return selectAddrModeUnscaled(Root, Size: 8);
402 }
403 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
404 return selectAddrModeUnscaled(Root, Size: 16);
405 }
406
407 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
408 /// from complex pattern matchers like selectAddrModeIndexed().
409 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
410 MachineRegisterInfo &MRI) const;
411
412 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
413 unsigned Size) const;
414 template <int Width>
415 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
416 return selectAddrModeIndexed(Root, Size: Width / 8);
417 }
418
419 std::optional<bool>
420 isWorthFoldingIntoAddrMode(const MachineInstr &MI,
421 const MachineRegisterInfo &MRI) const;
422
423 bool isWorthFoldingIntoExtendedReg(const MachineInstr &MI,
424 const MachineRegisterInfo &MRI,
425 bool IsAddrOperand) const;
426 ComplexRendererFns
427 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
428 unsigned SizeInBytes) const;
429
430 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
431 /// or not a shift + extend should be folded into an addressing mode. Returns
432 /// None when this is not profitable or possible.
433 ComplexRendererFns
434 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
435 MachineOperand &Offset, unsigned SizeInBytes,
436 bool WantsExt) const;
437 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
438 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
439 unsigned SizeInBytes) const;
440 template <int Width>
441 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
442 return selectAddrModeXRO(Root, SizeInBytes: Width / 8);
443 }
444
445 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
446 unsigned SizeInBytes) const;
447 template <int Width>
448 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
449 return selectAddrModeWRO(Root, SizeInBytes: Width / 8);
450 }
451
452 ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
453 bool AllowROR = false) const;
454
455 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
456 return selectShiftedRegister(Root);
457 }
458
459 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
460 return selectShiftedRegister(Root, AllowROR: true);
461 }
462
463 /// Given an extend instruction, determine the correct shift-extend type for
464 /// that instruction.
465 ///
466 /// If the instruction is going to be used in a load or store, pass
467 /// \p IsLoadStore = true.
468 AArch64_AM::ShiftExtendType
469 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
470 bool IsLoadStore = false) const;
471
472 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
473 ///
474 /// \returns Either \p Reg if no change was necessary, or the new register
475 /// created by moving \p Reg.
476 ///
477 /// Note: This uses emitCopy right now.
478 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
479 MachineIRBuilder &MIB) const;
480
481 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
482
483 ComplexRendererFns selectExtractHigh(MachineOperand &Root) const;
484 template <unsigned Width>
485 ComplexRendererFns selectCVTFixedPoint(MachineOperand &Root) const;
486 ComplexRendererFns selectCVTFixedPointBase(const MachineOperand &Root,
487 unsigned width,
488 bool isReciprocal = false) const;
489 ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
490 ComplexRendererFns
491 selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
492 void renderFixedPointScalarXForm(MachineInstrBuilder &MIB,
493 const MachineInstr &MI, int OpIdx) const;
494 unsigned getFixedPointWidthFromOperand(const MachineOperand &Root) const;
495 void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
496 int OpIdx = -1) const;
497 void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
498 const MachineInstr &MI, int OpIdx = -1) const;
499 void renderFixedPointImm(MachineInstrBuilder &MIB, const MachineOperand &Root,
500 unsigned Width, bool isReciprocal) const;
501 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
502 int OpIdx = -1) const;
503 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
504 int OpIdx = -1) const;
505 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
506 int OpIdx = -1) const;
507 void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
508 int OpIdx) const;
509 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
510 int OpIdx = -1) const;
511 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
512 int OpIdx = -1) const;
513 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
514 int OpIdx = -1) const;
515 void renderFPImm32SIMDModImmType4(MachineInstrBuilder &MIB,
516 const MachineInstr &MI,
517 int OpIdx = -1) const;
518
519 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
520 void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);
521
522 // Optimization methods.
523 bool tryOptSelect(GSelect &Sel);
524 bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
525 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
526 MachineOperand &Predicate,
527 MachineIRBuilder &MIRBuilder) const;
528
529 /// Return true if \p MI is a load or store of \p NumBytes bytes.
530 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
531
532 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
533 /// register zeroed out. In other words, the result of MI has been explicitly
534 /// zero extended.
535 bool isDef32(const MachineInstr &MI) const;
536
537 const AArch64TargetMachine &TM;
538 const AArch64Subtarget &STI;
539 const AArch64InstrInfo &TII;
540 const AArch64RegisterInfo &TRI;
541 const AArch64RegisterBankInfo &RBI;
542
543 bool ProduceNonFlagSettingCondBr = false;
544
545 // Some cached values used during selection.
546 // We use LR as a live-in register, and we keep track of it here as it can be
547 // clobbered by calls.
548 Register MFReturnAddr;
549
550 MachineIRBuilder MIB;
551
552#define GET_GLOBALISEL_PREDICATES_DECL
553#include "AArch64GenGlobalISel.inc"
554#undef GET_GLOBALISEL_PREDICATES_DECL
555
556// We declare the temporaries used by selectImpl() in the class to minimize the
557// cost of constructing placeholder values.
558#define GET_GLOBALISEL_TEMPORARIES_DECL
559#include "AArch64GenGlobalISel.inc"
560#undef GET_GLOBALISEL_TEMPORARIES_DECL
561};
562
563} // end anonymous namespace
564
565#define GET_GLOBALISEL_IMPL
566#include "AArch64GenGlobalISel.inc"
567#undef GET_GLOBALISEL_IMPL
568
569AArch64InstructionSelector::AArch64InstructionSelector(
570 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
571 const AArch64RegisterBankInfo &RBI)
572 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
573 RBI(RBI),
574#define GET_GLOBALISEL_PREDICATES_INIT
575#include "AArch64GenGlobalISel.inc"
576#undef GET_GLOBALISEL_PREDICATES_INIT
577#define GET_GLOBALISEL_TEMPORARIES_INIT
578#include "AArch64GenGlobalISel.inc"
579#undef GET_GLOBALISEL_TEMPORARIES_INIT
580{
581}
582
583// FIXME: This should be target-independent, inferred from the types declared
584// for each class in the bank.
585//
586/// Given a register bank, and a type, return the smallest register class that
587/// can represent that combination.
588static const TargetRegisterClass *
589getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
590 bool GetAllRegSet = false) {
591 if (RB.getID() == AArch64::GPRRegBankID) {
592 if (Ty.getSizeInBits() <= 32)
593 return GetAllRegSet ? &AArch64::GPR32allRegClass
594 : &AArch64::GPR32RegClass;
595 if (Ty.getSizeInBits() == 64)
596 return GetAllRegSet ? &AArch64::GPR64allRegClass
597 : &AArch64::GPR64RegClass;
598 if (Ty.getSizeInBits() == 128)
599 return &AArch64::XSeqPairsClassRegClass;
600 return nullptr;
601 }
602
603 if (RB.getID() == AArch64::FPRRegBankID) {
604 switch (Ty.getSizeInBits()) {
605 case 8:
606 return &AArch64::FPR8RegClass;
607 case 16:
608 return &AArch64::FPR16RegClass;
609 case 32:
610 return &AArch64::FPR32RegClass;
611 case 64:
612 return &AArch64::FPR64RegClass;
613 case 128:
614 return &AArch64::FPR128RegClass;
615 }
616 return nullptr;
617 }
618
619 return nullptr;
620}
621
622/// Given a register bank, and size in bits, return the smallest register class
623/// that can represent that combination.
624static const TargetRegisterClass *
625getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
626 bool GetAllRegSet = false) {
627 if (SizeInBits.isScalable()) {
628 assert(RB.getID() == AArch64::FPRRegBankID &&
629 "Expected FPR regbank for scalable type size");
630 return &AArch64::ZPRRegClass;
631 }
632
633 unsigned RegBankID = RB.getID();
634
635 if (RegBankID == AArch64::GPRRegBankID) {
636 assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
637 if (SizeInBits <= 32)
638 return GetAllRegSet ? &AArch64::GPR32allRegClass
639 : &AArch64::GPR32RegClass;
640 if (SizeInBits == 64)
641 return GetAllRegSet ? &AArch64::GPR64allRegClass
642 : &AArch64::GPR64RegClass;
643 if (SizeInBits == 128)
644 return &AArch64::XSeqPairsClassRegClass;
645 }
646
647 if (RegBankID == AArch64::FPRRegBankID) {
648 if (SizeInBits.isScalable()) {
649 assert(SizeInBits == TypeSize::getScalable(128) &&
650 "Unexpected scalable register size");
651 return &AArch64::ZPRRegClass;
652 }
653
654 switch (SizeInBits) {
655 default:
656 return nullptr;
657 case 8:
658 return &AArch64::FPR8RegClass;
659 case 16:
660 return &AArch64::FPR16RegClass;
661 case 32:
662 return &AArch64::FPR32RegClass;
663 case 64:
664 return &AArch64::FPR64RegClass;
665 case 128:
666 return &AArch64::FPR128RegClass;
667 }
668 }
669
670 return nullptr;
671}
672
673/// Returns the correct subregister to use for a given register class.
674static bool getSubRegForClass(const TargetRegisterClass *RC,
675 const TargetRegisterInfo &TRI, unsigned &SubReg) {
676 switch (TRI.getRegSizeInBits(RC: *RC)) {
677 case 8:
678 SubReg = AArch64::bsub;
679 break;
680 case 16:
681 SubReg = AArch64::hsub;
682 break;
683 case 32:
684 if (RC != &AArch64::FPR32RegClass)
685 SubReg = AArch64::sub_32;
686 else
687 SubReg = AArch64::ssub;
688 break;
689 case 64:
690 SubReg = AArch64::dsub;
691 break;
692 default:
693 LLVM_DEBUG(
694 dbgs() << "Couldn't find appropriate subregister for register class.");
695 return false;
696 }
697
698 return true;
699}
700
701/// Returns the minimum size the given register bank can hold.
702static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
703 switch (RB.getID()) {
704 case AArch64::GPRRegBankID:
705 return 32;
706 case AArch64::FPRRegBankID:
707 return 8;
708 default:
709 llvm_unreachable("Tried to get minimum size for unknown register bank.");
710 }
711}
712
713/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
714/// Helper function for functions like createDTuple and createQTuple.
715///
716/// \p RegClassIDs - The list of register class IDs available for some tuple of
717/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
718/// expected to contain between 2 and 4 tuple classes.
719///
720/// \p SubRegs - The list of subregister classes associated with each register
721/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
722/// subregister class. The index of each subregister class is expected to
723/// correspond with the index of each register class.
724///
725/// \returns Either the destination register of REG_SEQUENCE instruction that
726/// was created, or the 0th element of \p Regs if \p Regs contains a single
727/// element.
728static Register createTuple(ArrayRef<Register> Regs,
729 const unsigned RegClassIDs[],
730 const unsigned SubRegs[], MachineIRBuilder &MIB) {
731 unsigned NumRegs = Regs.size();
732 if (NumRegs == 1)
733 return Regs[0];
734 assert(NumRegs >= 2 && NumRegs <= 4 &&
735 "Only support between two and 4 registers in a tuple!");
736 const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
737 auto *DesiredClass = TRI->getRegClass(i: RegClassIDs[NumRegs - 2]);
738 auto RegSequence =
739 MIB.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {DesiredClass}, SrcOps: {});
740 for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
741 RegSequence.addUse(RegNo: Regs[I]);
742 RegSequence.addImm(Val: SubRegs[I]);
743 }
744 return RegSequence.getReg(Idx: 0);
745}
746
747/// Create a tuple of D-registers using the registers in \p Regs.
748static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
749 static const unsigned RegClassIDs[] = {
750 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
751 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
752 AArch64::dsub2, AArch64::dsub3};
753 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
754}
755
756/// Create a tuple of Q-registers using the registers in \p Regs.
757static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
758 static const unsigned RegClassIDs[] = {
759 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
760 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
761 AArch64::qsub2, AArch64::qsub3};
762 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
763}
764
765static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
766 auto &MI = *Root.getParent();
767 auto &MBB = *MI.getParent();
768 auto &MF = *MBB.getParent();
769 auto &MRI = MF.getRegInfo();
770 uint64_t Immed;
771 if (Root.isImm())
772 Immed = Root.getImm();
773 else if (Root.isCImm())
774 Immed = Root.getCImm()->getZExtValue();
775 else if (Root.isReg()) {
776 auto ValAndVReg =
777 getIConstantVRegValWithLookThrough(VReg: Root.getReg(), MRI, LookThroughInstrs: true);
778 if (!ValAndVReg)
779 return std::nullopt;
780 Immed = ValAndVReg->Value.getSExtValue();
781 } else
782 return std::nullopt;
783 return Immed;
784}
785
786/// Select the AArch64 opcode for the basic binary operation \p GenericOpc,
787/// appropriate for the register bank \p RegBankID and of size \p OpSize.
788/// \returns \p GenericOpc if the combination is unsupported.
789static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
790 unsigned OpSize) {
791 if (RegBankID == AArch64::GPRRegBankID) {
792 if (OpSize == 32) {
793 switch (GenericOpc) {
794 case TargetOpcode::G_SHL:
795 return AArch64::LSLVWr;
796 case TargetOpcode::G_LSHR:
797 return AArch64::LSRVWr;
798 case TargetOpcode::G_ASHR:
799 return AArch64::ASRVWr;
800 default:
801 return GenericOpc;
802 }
803 } else if (OpSize == 64) {
804 switch (GenericOpc) {
805 case TargetOpcode::G_SHL:
806 return AArch64::LSLVXr;
807 case TargetOpcode::G_LSHR:
808 return AArch64::LSRVXr;
809 case TargetOpcode::G_ASHR:
810 return AArch64::ASRVXr;
811 default:
812 return GenericOpc;
813 }
814 }
815 }
816 return GenericOpc;
817}
818
819/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
820/// appropriate for the (value) register bank \p RegBankID and of memory access
821/// size \p OpSize. This returns the variant with the base+unsigned-immediate
822/// addressing mode (e.g., LDRXui).
823/// \returns \p GenericOpc if the combination is unsupported.
824static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
825 unsigned OpSize) {
826 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
827 switch (RegBankID) {
828 case AArch64::GPRRegBankID:
829 switch (OpSize) {
830 case 8:
831 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
832 case 16:
833 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
834 case 32:
835 return isStore ? AArch64::STRWui : AArch64::LDRWui;
836 case 64:
837 return isStore ? AArch64::STRXui : AArch64::LDRXui;
838 }
839 break;
840 case AArch64::FPRRegBankID:
841 switch (OpSize) {
842 case 8:
843 return isStore ? AArch64::STRBui : AArch64::LDRBui;
844 case 16:
845 return isStore ? AArch64::STRHui : AArch64::LDRHui;
846 case 32:
847 return isStore ? AArch64::STRSui : AArch64::LDRSui;
848 case 64:
849 return isStore ? AArch64::STRDui : AArch64::LDRDui;
850 case 128:
851 return isStore ? AArch64::STRQui : AArch64::LDRQui;
852 }
853 break;
854 }
855 return GenericOpc;
856}
857
858/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
859/// to \p *To.
860///
861/// E.g "To = COPY SrcReg:SubReg"
862static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
863 const RegisterBankInfo &RBI, Register SrcReg,
864 const TargetRegisterClass *To, unsigned SubReg) {
865 assert(SrcReg.isValid() && "Expected a valid source register?");
866 assert(To && "Destination register class cannot be null");
867 assert(SubReg && "Expected a valid subregister");
868
869 MachineIRBuilder MIB(I);
870 auto SubRegCopy =
871 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {To}, SrcOps: {}).addReg(RegNo: SrcReg, Flags: {}, SubReg);
872 MachineOperand &RegOp = I.getOperand(i: 1);
873 RegOp.setReg(SubRegCopy.getReg(Idx: 0));
874
875 // It's possible that the destination register won't be constrained. Make
876 // sure that happens.
877 if (!I.getOperand(i: 0).getReg().isPhysical())
878 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: *To, MRI);
879
880 return true;
881}
882
883/// Helper function to get the source and destination register classes for a
884/// copy. Returns a std::pair containing the source register class for the
885/// copy, and the destination register class for the copy. If a register class
886/// cannot be determined, then it will be nullptr.
887static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
888getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
889 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
890 const RegisterBankInfo &RBI) {
891 Register DstReg = I.getOperand(i: 0).getReg();
892 Register SrcReg = I.getOperand(i: 1).getReg();
893 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
894 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
895
896 TypeSize DstSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
897 TypeSize SrcSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
898
899 // Special casing for cross-bank copies of s1s. We can technically represent
900 // a 1-bit value with any size of register. The minimum size for a GPR is 32
901 // bits. So, we need to put the FPR on 32 bits as well.
902 //
903 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
904 // then we can pull it into the helpers that get the appropriate class for a
905 // register bank. Or make a new helper that carries along some constraint
906 // information.
907 if (SrcRegBank != DstRegBank &&
908 (DstSize == TypeSize::getFixed(ExactSize: 1) && SrcSize == TypeSize::getFixed(ExactSize: 1)))
909 SrcSize = DstSize = TypeSize::getFixed(ExactSize: 32);
910
911 return {getMinClassForRegBank(RB: SrcRegBank, SizeInBits: SrcSize, GetAllRegSet: true),
912 getMinClassForRegBank(RB: DstRegBank, SizeInBits: DstSize, GetAllRegSet: true)};
913}
914
915// FIXME: We need some sort of API in RBI/TRI to allow generic code to
916// constrain operands of simple instructions given a TargetRegisterClass
917// and LLT
918static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI,
919 const RegisterBankInfo &RBI) {
920 for (MachineOperand &MO : I.operands()) {
921 if (!MO.isReg())
922 continue;
923 Register Reg = MO.getReg();
924 if (!Reg)
925 continue;
926 if (Reg.isPhysical())
927 continue;
928 LLT Ty = MRI.getType(Reg);
929 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
930 const TargetRegisterClass *RC =
931 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
932 if (!RC) {
933 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
934 RC = getRegClassForTypeOnBank(Ty, RB);
935 if (!RC) {
936 LLVM_DEBUG(
937 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
938 break;
939 }
940 }
941 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
942 }
943
944 return true;
945}
946
947static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
948 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
949 const RegisterBankInfo &RBI) {
950 Register DstReg = I.getOperand(i: 0).getReg();
951 Register SrcReg = I.getOperand(i: 1).getReg();
952 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
953 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
954
955 // Find the correct register classes for the source and destination registers.
956 const TargetRegisterClass *SrcRC;
957 const TargetRegisterClass *DstRC;
958 std::tie(args&: SrcRC, args&: DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);
959
960 if (!DstRC) {
961 LLVM_DEBUG(dbgs() << "Unexpected dest size "
962 << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n');
963 return false;
964 }
965
966 // Is this a copy? If so, then we may need to insert a subregister copy.
967 if (I.isCopy()) {
968 // Yes. Check if there's anything to fix up.
969 if (!SrcRC) {
970 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
971 return false;
972 }
973
974 const TypeSize SrcSize = TRI.getRegSizeInBits(RC: *SrcRC);
975 const TypeSize DstSize = TRI.getRegSizeInBits(RC: *DstRC);
976 unsigned SrcSubReg = I.getOperand(i: 1).getSubReg();
977 unsigned SubReg;
978
979 if (SrcSubReg)
980 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
981
982 // If the source bank doesn't support a subregister copy small enough,
983 // then we first need to copy to the destination bank.
984 if (getMinSizeForRegBank(RB: SrcRegBank) > DstSize) {
985 const TargetRegisterClass *DstTempRC =
986 getMinClassForRegBank(RB: DstRegBank, SizeInBits: SrcSize, /* GetAllRegSet */ true);
987 getSubRegForClass(RC: DstRC, TRI, SubReg);
988
989 MachineIRBuilder MIB(I);
990 auto Copy = MIB.buildCopy(Res: {DstTempRC}, Op: {SrcReg});
991 copySubReg(I, MRI, RBI, SrcReg: Copy.getReg(Idx: 0), To: DstRC, SubReg);
992 } else if (SrcSize > DstSize) {
993 // If the source register is bigger than the destination we need to
994 // perform a subregister copy.
995 const TargetRegisterClass *SubRegRC =
996 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
997 getSubRegForClass(RC: SubRegRC, TRI, SubReg);
998 copySubReg(I, MRI, RBI, SrcReg, To: DstRC, SubReg);
999 } else if (DstSize > SrcSize) {
1000 // If the destination register is bigger than the source we need to do
1001 // a promotion using SUBREG_TO_REG.
1002 const TargetRegisterClass *PromotionRC =
1003 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
1004 getSubRegForClass(RC: SrcRC, TRI, SubReg);
1005
1006 Register PromoteReg = MRI.createVirtualRegister(RegClass: PromotionRC);
1007 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1008 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG), DestReg: PromoteReg)
1009 .addUse(RegNo: SrcReg)
1010 .addImm(Val: SubReg);
1011 MachineOperand &RegOp = I.getOperand(i: 1);
1012 RegOp.setReg(PromoteReg);
1013 }
1014
1015 // If the destination is a physical register, then there's nothing to
1016 // change, so we're done.
1017 if (DstReg.isPhysical())
1018 return true;
1019 }
1020
1021 // No need to constrain SrcReg. It will get constrained when we hit another
1022 // of its use or its defs. Copies do not have constraints.
1023 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1024 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1025 << " operand\n");
1026 return false;
1027 }
1028
1029 // If this a GPR ZEXT that we want to just reduce down into a copy.
1030 // The sizes will be mismatched with the source < 32b but that's ok.
1031 if (I.getOpcode() == TargetOpcode::G_ZEXT) {
1032 I.setDesc(TII.get(Opcode: AArch64::COPY));
1033 assert(SrcRegBank.getID() == AArch64::GPRRegBankID);
1034 return selectCopy(I, TII, MRI, TRI, RBI);
1035 }
1036
1037 I.setDesc(TII.get(Opcode: AArch64::COPY));
1038 return true;
1039}
1040
1041MachineInstr *
1042AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1043 Register False, AArch64CC::CondCode CC,
1044 MachineIRBuilder &MIB) const {
1045 MachineRegisterInfo &MRI = *MIB.getMRI();
1046 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==
1047 RBI.getRegBank(True, MRI, TRI)->getID() &&
1048 "Expected both select operands to have the same regbank?");
1049 LLT Ty = MRI.getType(Reg: True);
1050 if (Ty.isVector())
1051 return nullptr;
1052 const unsigned Size = Ty.getSizeInBits();
1053 assert((Size == 32 || Size == 64) &&
1054 "Expected 32 bit or 64 bit select only?");
1055 const bool Is32Bit = Size == 32;
1056 if (RBI.getRegBank(Reg: True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1057 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1058 auto FCSel = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1059 constrainSelectedInstRegOperands(I&: *FCSel, TII, TRI, RBI);
1060 return &*FCSel;
1061 }
1062
1063 // By default, we'll try and emit a CSEL.
1064 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1065 bool Optimized = false;
1066 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1067 &Optimized](Register &Reg, Register &OtherReg,
1068 bool Invert) {
1069 if (Optimized)
1070 return false;
1071
1072 // Attempt to fold:
1073 //
1074 // %sub = G_SUB 0, %x
1075 // %select = G_SELECT cc, %reg, %sub
1076 //
1077 // Into:
1078 // %select = CSNEG %reg, %x, cc
1079 Register MatchReg;
1080 if (mi_match(R: Reg, MRI, P: m_Neg(Src: m_Reg(R&: MatchReg)))) {
1081 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1082 Reg = MatchReg;
1083 if (Invert) {
1084 CC = AArch64CC::getInvertedCondCode(Code: CC);
1085 std::swap(a&: Reg, b&: OtherReg);
1086 }
1087 return true;
1088 }
1089
1090 // Attempt to fold:
1091 //
1092 // %xor = G_XOR %x, -1
1093 // %select = G_SELECT cc, %reg, %xor
1094 //
1095 // Into:
1096 // %select = CSINV %reg, %x, cc
1097 if (mi_match(R: Reg, MRI, P: m_Not(Src: m_Reg(R&: MatchReg)))) {
1098 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1099 Reg = MatchReg;
1100 if (Invert) {
1101 CC = AArch64CC::getInvertedCondCode(Code: CC);
1102 std::swap(a&: Reg, b&: OtherReg);
1103 }
1104 return true;
1105 }
1106
1107 // Attempt to fold:
1108 //
1109 // %add = G_ADD %x, 1
1110 // %select = G_SELECT cc, %reg, %add
1111 //
1112 // Into:
1113 // %select = CSINC %reg, %x, cc
1114 if (mi_match(R: Reg, MRI,
1115 P: m_any_of(preds: m_GAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1)),
1116 preds: m_GPtrAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1))))) {
1117 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1118 Reg = MatchReg;
1119 if (Invert) {
1120 CC = AArch64CC::getInvertedCondCode(Code: CC);
1121 std::swap(a&: Reg, b&: OtherReg);
1122 }
1123 return true;
1124 }
1125
1126 return false;
1127 };
1128
1129 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1130 // true/false values are constants.
1131 // FIXME: All of these patterns already exist in tablegen. We should be
1132 // able to import these.
1133 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1134 &Optimized]() {
1135 if (Optimized)
1136 return false;
1137 auto TrueCst = getIConstantVRegValWithLookThrough(VReg: True, MRI);
1138 auto FalseCst = getIConstantVRegValWithLookThrough(VReg: False, MRI);
1139 if (!TrueCst && !FalseCst)
1140 return false;
1141
1142 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1143 if (TrueCst && FalseCst) {
1144 int64_t T = TrueCst->Value.getSExtValue();
1145 int64_t F = FalseCst->Value.getSExtValue();
1146
1147 if (T == 0 && F == 1) {
1148 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1149 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1150 True = ZReg;
1151 False = ZReg;
1152 return true;
1153 }
1154
1155 if (T == 0 && F == -1) {
1156 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1157 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1158 True = ZReg;
1159 False = ZReg;
1160 return true;
1161 }
1162 }
1163
1164 if (TrueCst) {
1165 int64_t T = TrueCst->Value.getSExtValue();
1166 if (T == 1) {
1167 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1168 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1169 True = False;
1170 False = ZReg;
1171 CC = AArch64CC::getInvertedCondCode(Code: CC);
1172 return true;
1173 }
1174
1175 if (T == -1) {
1176 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1177 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1178 True = False;
1179 False = ZReg;
1180 CC = AArch64CC::getInvertedCondCode(Code: CC);
1181 return true;
1182 }
1183 }
1184
1185 if (FalseCst) {
1186 int64_t F = FalseCst->Value.getSExtValue();
1187 if (F == 1) {
1188 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1189 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1190 False = ZReg;
1191 return true;
1192 }
1193
1194 if (F == -1) {
1195 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1196 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1197 False = ZReg;
1198 return true;
1199 }
1200 }
1201 return false;
1202 };
1203
1204 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1205 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1206 Optimized |= TryOptSelectCst();
1207 auto SelectInst = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1208 constrainSelectedInstRegOperands(I&: *SelectInst, TII, TRI, RBI);
1209 return &*SelectInst;
1210}
1211
1212static AArch64CC::CondCode
1213changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS = {},
1214 MachineRegisterInfo *MRI = nullptr) {
1215 switch (P) {
1216 default:
1217 llvm_unreachable("Unknown condition code!");
1218 case CmpInst::ICMP_NE:
1219 return AArch64CC::NE;
1220 case CmpInst::ICMP_EQ:
1221 return AArch64CC::EQ;
1222 case CmpInst::ICMP_SGT:
1223 return AArch64CC::GT;
1224 case CmpInst::ICMP_SGE:
1225 if (RHS && MRI) {
1226 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1227 if (ValAndVReg && ValAndVReg->Value == 0)
1228 return AArch64CC::PL;
1229 }
1230 return AArch64CC::GE;
1231 case CmpInst::ICMP_SLT:
1232 if (RHS && MRI) {
1233 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1234 if (ValAndVReg && ValAndVReg->Value == 0)
1235 return AArch64CC::MI;
1236 }
1237 return AArch64CC::LT;
1238 case CmpInst::ICMP_SLE:
1239 return AArch64CC::LE;
1240 case CmpInst::ICMP_UGT:
1241 return AArch64CC::HI;
1242 case CmpInst::ICMP_UGE:
1243 return AArch64CC::HS;
1244 case CmpInst::ICMP_ULT:
1245 return AArch64CC::LO;
1246 case CmpInst::ICMP_ULE:
1247 return AArch64CC::LS;
1248 }
1249}
1250
1251/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1252static void changeFPCCToORAArch64CC(CmpInst::Predicate CC,
1253 AArch64CC::CondCode &CondCode,
1254 AArch64CC::CondCode &CondCode2) {
1255 CondCode2 = AArch64CC::AL;
1256 switch (CC) {
1257 default:
1258 llvm_unreachable("Unknown FP condition!");
1259 case CmpInst::FCMP_OEQ:
1260 CondCode = AArch64CC::EQ;
1261 break;
1262 case CmpInst::FCMP_OGT:
1263 CondCode = AArch64CC::GT;
1264 break;
1265 case CmpInst::FCMP_OGE:
1266 CondCode = AArch64CC::GE;
1267 break;
1268 case CmpInst::FCMP_OLT:
1269 CondCode = AArch64CC::MI;
1270 break;
1271 case CmpInst::FCMP_OLE:
1272 CondCode = AArch64CC::LS;
1273 break;
1274 case CmpInst::FCMP_ONE:
1275 CondCode = AArch64CC::MI;
1276 CondCode2 = AArch64CC::GT;
1277 break;
1278 case CmpInst::FCMP_ORD:
1279 CondCode = AArch64CC::VC;
1280 break;
1281 case CmpInst::FCMP_UNO:
1282 CondCode = AArch64CC::VS;
1283 break;
1284 case CmpInst::FCMP_UEQ:
1285 CondCode = AArch64CC::EQ;
1286 CondCode2 = AArch64CC::VS;
1287 break;
1288 case CmpInst::FCMP_UGT:
1289 CondCode = AArch64CC::HI;
1290 break;
1291 case CmpInst::FCMP_UGE:
1292 CondCode = AArch64CC::PL;
1293 break;
1294 case CmpInst::FCMP_ULT:
1295 CondCode = AArch64CC::LT;
1296 break;
1297 case CmpInst::FCMP_ULE:
1298 CondCode = AArch64CC::LE;
1299 break;
1300 case CmpInst::FCMP_UNE:
1301 CondCode = AArch64CC::NE;
1302 break;
1303 }
1304}
1305
1306/// Convert an IR fp condition code to an AArch64 CC.
1307/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1308/// should be AND'ed instead of OR'ed.
1309static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC,
1310 AArch64CC::CondCode &CondCode,
1311 AArch64CC::CondCode &CondCode2) {
1312 CondCode2 = AArch64CC::AL;
1313 switch (CC) {
1314 default:
1315 changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
1316 assert(CondCode2 == AArch64CC::AL);
1317 break;
1318 case CmpInst::FCMP_ONE:
1319 // (a one b)
1320 // == ((a olt b) || (a ogt b))
1321 // == ((a ord b) && (a une b))
1322 CondCode = AArch64CC::VC;
1323 CondCode2 = AArch64CC::NE;
1324 break;
1325 case CmpInst::FCMP_UEQ:
1326 // (a ueq b)
1327 // == ((a uno b) || (a oeq b))
1328 // == ((a ule b) && (a uge b))
1329 CondCode = AArch64CC::PL;
1330 CondCode2 = AArch64CC::LE;
1331 break;
1332 }
1333}
1334
1335/// Return a register which can be used as a bit to test in a TB(N)Z.
1336static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1337 MachineRegisterInfo &MRI) {
1338 assert(Reg.isValid() && "Expected valid register!");
1339 bool HasZext = false;
1340 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1341 unsigned Opc = MI->getOpcode();
1342
1343 if (!MI->getOperand(i: 0).isReg() ||
1344 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1345 break;
1346
1347 // (tbz (any_ext x), b) -> (tbz x, b) and
1348 // (tbz (zext x), b) -> (tbz x, b) if we don't use the extended bits.
1349 //
1350 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1351 // on the truncated x is the same as the bit number on x.
1352 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1353 Opc == TargetOpcode::G_TRUNC) {
1354 if (Opc == TargetOpcode::G_ZEXT)
1355 HasZext = true;
1356
1357 Register NextReg = MI->getOperand(i: 1).getReg();
1358 // Did we find something worth folding?
1359 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(RegNo: NextReg))
1360 break;
1361 TypeSize InSize = MRI.getType(Reg: NextReg).getSizeInBits();
1362 if (Bit >= InSize)
1363 break;
1364
1365 // NextReg is worth folding. Keep looking.
1366 Reg = NextReg;
1367 continue;
1368 }
1369
1370 // Attempt to find a suitable operation with a constant on one side.
1371 std::optional<uint64_t> C;
1372 Register TestReg;
1373 switch (Opc) {
1374 default:
1375 break;
1376 case TargetOpcode::G_AND:
1377 case TargetOpcode::G_XOR: {
1378 TestReg = MI->getOperand(i: 1).getReg();
1379 Register ConstantReg = MI->getOperand(i: 2).getReg();
1380 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1381 if (!VRegAndVal) {
1382 // AND commutes, check the other side for a constant.
1383 // FIXME: Can we canonicalize the constant so that it's always on the
1384 // same side at some point earlier?
1385 std::swap(a&: ConstantReg, b&: TestReg);
1386 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1387 }
1388 if (VRegAndVal) {
1389 if (HasZext)
1390 C = VRegAndVal->Value.getZExtValue();
1391 else
1392 C = VRegAndVal->Value.getSExtValue();
1393 }
1394 break;
1395 }
1396 case TargetOpcode::G_ASHR:
1397 case TargetOpcode::G_LSHR:
1398 case TargetOpcode::G_SHL: {
1399 TestReg = MI->getOperand(i: 1).getReg();
1400 auto VRegAndVal =
1401 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
1402 if (VRegAndVal)
1403 C = VRegAndVal->Value.getSExtValue();
1404 break;
1405 }
1406 }
1407
1408 // Didn't find a constant or viable register. Bail out of the loop.
1409 if (!C || !TestReg.isValid())
1410 break;
1411
1412 // We found a suitable instruction with a constant. Check to see if we can
1413 // walk through the instruction.
1414 Register NextReg;
1415 unsigned TestRegSize = MRI.getType(Reg: TestReg).getSizeInBits();
1416 switch (Opc) {
1417 default:
1418 break;
1419 case TargetOpcode::G_AND:
1420 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1421 if ((*C >> Bit) & 1)
1422 NextReg = TestReg;
1423 break;
1424 case TargetOpcode::G_SHL:
1425 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1426 // the type of the register.
1427 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1428 NextReg = TestReg;
1429 Bit = Bit - *C;
1430 }
1431 break;
1432 case TargetOpcode::G_ASHR:
1433 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1434 // in x
1435 NextReg = TestReg;
1436 Bit = Bit + *C;
1437 if (Bit >= TestRegSize)
1438 Bit = TestRegSize - 1;
1439 break;
1440 case TargetOpcode::G_LSHR:
1441 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1442 if ((Bit + *C) < TestRegSize) {
1443 NextReg = TestReg;
1444 Bit = Bit + *C;
1445 }
1446 break;
1447 case TargetOpcode::G_XOR:
1448 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1449 // appropriate.
1450 //
1451 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1452 //
1453 // tbz x', b -> tbnz x, b
1454 //
1455 // Because x' only has the b-th bit set if x does not.
1456 if ((*C >> Bit) & 1)
1457 Invert = !Invert;
1458 NextReg = TestReg;
1459 break;
1460 }
1461
1462 // Check if we found anything worth folding.
1463 if (!NextReg.isValid())
1464 return Reg;
1465 Reg = NextReg;
1466 }
1467
1468 return Reg;
1469}
1470
1471MachineInstr *AArch64InstructionSelector::emitTestBit(
1472 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1473 MachineIRBuilder &MIB) const {
1474 assert(TestReg.isValid());
1475 assert(ProduceNonFlagSettingCondBr &&
1476 "Cannot emit TB(N)Z with speculation tracking!");
1477 MachineRegisterInfo &MRI = *MIB.getMRI();
1478
1479 // Attempt to optimize the test bit by walking over instructions.
1480 TestReg = getTestBitReg(Reg: TestReg, Bit, Invert&: IsNegative, MRI);
1481 LLT Ty = MRI.getType(Reg: TestReg);
1482 unsigned Size = Ty.getSizeInBits();
1483 assert(!Ty.isVector() && "Expected a scalar!");
1484 assert(Bit < 64 && "Bit is too large!");
1485
1486 // When the test register is a 64-bit register, we have to narrow to make
1487 // TBNZW work.
1488 bool UseWReg = Bit < 32;
1489 unsigned NecessarySize = UseWReg ? 32 : 64;
1490 if (Size != NecessarySize)
1491 TestReg = moveScalarRegClass(
1492 Reg: TestReg, RC: UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1493 MIB);
1494
1495 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1496 {AArch64::TBZW, AArch64::TBNZW}};
1497 unsigned Opc = OpcTable[UseWReg][IsNegative];
1498 auto TestBitMI =
1499 MIB.buildInstr(Opcode: Opc).addReg(RegNo: TestReg).addImm(Val: Bit).addMBB(MBB: DstMBB);
1500 constrainSelectedInstRegOperands(I&: *TestBitMI, TII, TRI, RBI);
1501 return &*TestBitMI;
1502}
1503
1504bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1505 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1506 MachineIRBuilder &MIB) const {
1507 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
1508 // Given something like this:
1509 //
1510 // %x = ...Something...
1511 // %one = G_CONSTANT i64 1
1512 // %zero = G_CONSTANT i64 0
1513 // %and = G_AND %x, %one
1514 // %cmp = G_ICMP intpred(ne), %and, %zero
1515 // %cmp_trunc = G_TRUNC %cmp
1516 // G_BRCOND %cmp_trunc, %bb.3
1517 //
1518 // We want to try and fold the AND into the G_BRCOND and produce either a
1519 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1520 //
1521 // In this case, we'd get
1522 //
1523 // TBNZ %x %bb.3
1524 //
1525
1526 // Check if the AND has a constant on its RHS which we can use as a mask.
1527 // If it's a power of 2, then it's the same as checking a specific bit.
1528 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1529 auto MaybeBit = getIConstantVRegValWithLookThrough(
1530 VReg: AndInst.getOperand(i: 2).getReg(), MRI: *MIB.getMRI());
1531 if (!MaybeBit)
1532 return false;
1533
1534 int32_t Bit = MaybeBit->Value.exactLogBase2();
1535 if (Bit < 0)
1536 return false;
1537
1538 Register TestReg = AndInst.getOperand(i: 1).getReg();
1539
1540 // Emit a TB(N)Z.
1541 emitTestBit(TestReg, Bit, IsNegative: Invert, DstMBB, MIB);
1542 return true;
1543}
1544
1545MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1546 bool IsNegative,
1547 MachineBasicBlock *DestMBB,
1548 MachineIRBuilder &MIB) const {
1549 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
1550 MachineRegisterInfo &MRI = *MIB.getMRI();
1551 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
1552 AArch64::GPRRegBankID &&
1553 "Expected GPRs only?");
1554 auto Ty = MRI.getType(Reg: CompareReg);
1555 unsigned Width = Ty.getSizeInBits();
1556 assert(!Ty.isVector() && "Expected scalar only?");
1557 assert(Width <= 64 && "Expected width to be at most 64?");
1558 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1559 {AArch64::CBNZW, AArch64::CBNZX}};
1560 unsigned Opc = OpcTable[IsNegative][Width == 64];
1561 auto BranchMI = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {CompareReg}).addMBB(MBB: DestMBB);
1562 constrainSelectedInstRegOperands(I&: *BranchMI, TII, TRI, RBI);
1563 return &*BranchMI;
1564}
1565
1566bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1567 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1568 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
1569 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1570 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1571 // totally clean. Some of them require two branches to implement.
1572 auto Pred = (CmpInst::Predicate)FCmp.getOperand(i: 1).getPredicate();
1573 emitFPCompare(LHS: FCmp.getOperand(i: 2).getReg(), RHS: FCmp.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
1574 Pred);
1575 AArch64CC::CondCode CC1, CC2;
1576 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
1577 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1578 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC1).addMBB(MBB: DestMBB);
1579 if (CC2 != AArch64CC::AL)
1580 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC2).addMBB(MBB: DestMBB);
1581 I.eraseFromParent();
1582 return true;
1583}
1584
1585bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1586 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1587 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1588 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1589 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1590 //
1591 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1592 // instructions will not be produced, as they are conditional branch
1593 // instructions that do not set flags.
1594 if (!ProduceNonFlagSettingCondBr)
1595 return false;
1596
1597 MachineRegisterInfo &MRI = *MIB.getMRI();
1598 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1599 auto Pred =
1600 static_cast<CmpInst::Predicate>(ICmp.getOperand(i: 1).getPredicate());
1601 Register LHS = ICmp.getOperand(i: 2).getReg();
1602 Register RHS = ICmp.getOperand(i: 3).getReg();
1603
1604 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1605 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1606 MachineInstr *AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1607
1608 // When we can emit a TB(N)Z, prefer that.
1609 //
1610 // Handle non-commutative condition codes first.
1611 // Note that we don't want to do this when we have a G_AND because it can
1612 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1613 if (VRegAndVal && !AndInst) {
1614 int64_t C = VRegAndVal->Value.getSExtValue();
1615
1616 // When we have a greater-than comparison, we can just test if the msb is
1617 // zero.
1618 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1619 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1620 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1621 I.eraseFromParent();
1622 return true;
1623 }
1624
1625 // When we have a less than comparison, we can just test if the msb is not
1626 // zero.
1627 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1628 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1629 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ true, DstMBB: DestMBB, MIB);
1630 I.eraseFromParent();
1631 return true;
1632 }
1633
1634 // Inversely, if we have a signed greater-than-or-equal comparison to zero,
1635 // we can test if the msb is zero.
1636 if (C == 0 && Pred == CmpInst::ICMP_SGE) {
1637 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1638 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1639 I.eraseFromParent();
1640 return true;
1641 }
1642 }
1643
1644 // Attempt to handle commutative condition codes. Right now, that's only
1645 // eq/ne.
1646 if (ICmpInst::isEquality(P: Pred)) {
1647 if (!VRegAndVal) {
1648 std::swap(a&: RHS, b&: LHS);
1649 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1650 AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1651 }
1652
1653 if (VRegAndVal && VRegAndVal->Value == 0) {
1654 // If there's a G_AND feeding into this branch, try to fold it away by
1655 // emitting a TB(N)Z instead.
1656 //
1657 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1658 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1659 // would be redundant.
1660 if (AndInst &&
1661 tryOptAndIntoCompareBranch(
1662 AndInst&: *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DstMBB: DestMBB, MIB)) {
1663 I.eraseFromParent();
1664 return true;
1665 }
1666
1667 // Otherwise, try to emit a CB(N)Z instead.
1668 auto LHSTy = MRI.getType(Reg: LHS);
1669 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1670 emitCBZ(CompareReg: LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1671 I.eraseFromParent();
1672 return true;
1673 }
1674 }
1675 }
1676
1677 return false;
1678}
1679
1680bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1681 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1682 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1683 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1684 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1685 return true;
1686
1687 // Couldn't optimize. Emit a compare + a Bcc.
1688 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1689 auto &PredOp = ICmp.getOperand(i: 1);
1690 emitIntegerCompare(LHS&: ICmp.getOperand(i: 2), RHS&: ICmp.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
1691 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
1692 P: static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
1693 RHS: ICmp.getOperand(i: 3).getReg(), MRI: MIB.getMRI());
1694 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC).addMBB(MBB: DestMBB);
1695 I.eraseFromParent();
1696 return true;
1697}
1698
1699bool AArch64InstructionSelector::selectCompareBranch(
1700 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
1701 Register CondReg = I.getOperand(i: 0).getReg();
1702 MachineInstr *CCMI = MRI.getVRegDef(Reg: CondReg);
1703 // Try to select the G_BRCOND using whatever is feeding the condition if
1704 // possible.
1705 unsigned CCMIOpc = CCMI->getOpcode();
1706 if (CCMIOpc == TargetOpcode::G_FCMP)
1707 return selectCompareBranchFedByFCmp(I, FCmp&: *CCMI, MIB);
1708 if (CCMIOpc == TargetOpcode::G_ICMP)
1709 return selectCompareBranchFedByICmp(I, ICmp&: *CCMI, MIB);
1710
1711 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1712 // instructions will not be produced, as they are conditional branch
1713 // instructions that do not set flags.
1714 if (ProduceNonFlagSettingCondBr) {
1715 emitTestBit(TestReg: CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1716 DstMBB: I.getOperand(i: 1).getMBB(), MIB);
1717 I.eraseFromParent();
1718 return true;
1719 }
1720
1721 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1722 auto TstMI =
1723 MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {LLT::scalar(SizeInBits: 32)}, SrcOps: {CondReg}).addImm(Val: 1);
1724 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
1725 auto Bcc = MIB.buildInstr(Opcode: AArch64::Bcc)
1726 .addImm(Val: AArch64CC::NE)
1727 .addMBB(MBB: I.getOperand(i: 1).getMBB());
1728 I.eraseFromParent();
1729 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1730 return true;
1731}
1732
1733/// Returns the element immediate value of a vector shift operand if found.
1734/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1735static std::optional<int64_t> getVectorShiftImm(Register Reg,
1736 MachineRegisterInfo &MRI) {
1737 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand");
1738 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1739 return getAArch64VectorSplatScalar(MI: *OpMI, MRI);
1740}
1741
1742/// Matches and returns the shift immediate value for a SHL instruction given
1743/// a shift operand.
1744static std::optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
1745 MachineRegisterInfo &MRI) {
1746 std::optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
1747 if (!ShiftImm)
1748 return std::nullopt;
1749 // Check the immediate is in range for a SHL.
1750 int64_t Imm = *ShiftImm;
1751 if (Imm < 0)
1752 return std::nullopt;
1753 switch (SrcTy.getElementType().getSizeInBits()) {
1754 default:
1755 LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift");
1756 return std::nullopt;
1757 case 8:
1758 if (Imm > 7)
1759 return std::nullopt;
1760 break;
1761 case 16:
1762 if (Imm > 15)
1763 return std::nullopt;
1764 break;
1765 case 32:
1766 if (Imm > 31)
1767 return std::nullopt;
1768 break;
1769 case 64:
1770 if (Imm > 63)
1771 return std::nullopt;
1772 break;
1773 }
1774 return Imm;
1775}
1776
1777bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
1778 MachineRegisterInfo &MRI) {
1779 assert(I.getOpcode() == TargetOpcode::G_SHL);
1780 Register DstReg = I.getOperand(i: 0).getReg();
1781 const LLT Ty = MRI.getType(Reg: DstReg);
1782 Register Src1Reg = I.getOperand(i: 1).getReg();
1783 Register Src2Reg = I.getOperand(i: 2).getReg();
1784
1785 if (!Ty.isVector())
1786 return false;
1787
1788 // Check if we have a vector of constants on RHS that we can select as the
1789 // immediate form.
1790 std::optional<int64_t> ImmVal = getVectorSHLImm(SrcTy: Ty, Reg: Src2Reg, MRI);
1791
1792 unsigned Opc = 0;
1793 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1794 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1795 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1796 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1797 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1798 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1799 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1800 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1801 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1802 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1803 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1804 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1805 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1806 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1807 } else {
1808 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type");
1809 return false;
1810 }
1811
1812 auto Shl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg});
1813 if (ImmVal)
1814 Shl.addImm(Val: *ImmVal);
1815 else
1816 Shl.addUse(RegNo: Src2Reg);
1817 constrainSelectedInstRegOperands(I&: *Shl, TII, TRI, RBI);
1818 I.eraseFromParent();
1819 return true;
1820}
1821
1822bool AArch64InstructionSelector::selectVectorAshrLshr(
1823 MachineInstr &I, MachineRegisterInfo &MRI) {
1824 assert(I.getOpcode() == TargetOpcode::G_ASHR ||
1825 I.getOpcode() == TargetOpcode::G_LSHR);
1826 Register DstReg = I.getOperand(i: 0).getReg();
1827 const LLT Ty = MRI.getType(Reg: DstReg);
1828 Register Src1Reg = I.getOperand(i: 1).getReg();
1829 Register Src2Reg = I.getOperand(i: 2).getReg();
1830
1831 if (!Ty.isVector())
1832 return false;
1833
1834 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1835
1836 // We expect the immediate case to be lowered in the PostLegalCombiner to
1837 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1838
1839 // There is not a shift right register instruction, but the shift left
1840 // register instruction takes a signed value, where negative numbers specify a
1841 // right shift.
1842
1843 unsigned Opc = 0;
1844 unsigned NegOpc = 0;
1845 const TargetRegisterClass *RC =
1846 getRegClassForTypeOnBank(Ty, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID));
1847 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1848 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1849 NegOpc = AArch64::NEGv2i64;
1850 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1851 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1852 NegOpc = AArch64::NEGv4i32;
1853 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1854 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1855 NegOpc = AArch64::NEGv2i32;
1856 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1857 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1858 NegOpc = AArch64::NEGv4i16;
1859 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1860 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1861 NegOpc = AArch64::NEGv8i16;
1862 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1863 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1864 NegOpc = AArch64::NEGv16i8;
1865 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1866 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1867 NegOpc = AArch64::NEGv8i8;
1868 } else {
1869 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type");
1870 return false;
1871 }
1872
1873 auto Neg = MIB.buildInstr(Opc: NegOpc, DstOps: {RC}, SrcOps: {Src2Reg});
1874 constrainSelectedInstRegOperands(I&: *Neg, TII, TRI, RBI);
1875 auto SShl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg, Neg});
1876 constrainSelectedInstRegOperands(I&: *SShl, TII, TRI, RBI);
1877 I.eraseFromParent();
1878 return true;
1879}
1880
1881bool AArch64InstructionSelector::selectVaStartAAPCS(
1882 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1883
1884 if (STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
1885 IsVarArg: MF.getFunction().isVarArg()))
1886 return false;
1887
1888 // The layout of the va_list struct is specified in the AArch64 Procedure Call
1889 // Standard, section 10.1.5.
1890
1891 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1892 const unsigned PtrSize = STI.isTargetILP32() ? 4 : 8;
1893 const auto *PtrRegClass =
1894 STI.isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1895
1896 const MCInstrDesc &MCIDAddAddr =
1897 TII.get(Opcode: STI.isTargetILP32() ? AArch64::ADDWri : AArch64::ADDXri);
1898 const MCInstrDesc &MCIDStoreAddr =
1899 TII.get(Opcode: STI.isTargetILP32() ? AArch64::STRWui : AArch64::STRXui);
1900
1901 /*
1902 * typedef struct va_list {
1903 * void * stack; // next stack param
1904 * void * gr_top; // end of GP arg reg save area
1905 * void * vr_top; // end of FP/SIMD arg reg save area
1906 * int gr_offs; // offset from gr_top to next GP register arg
1907 * int vr_offs; // offset from vr_top to next FP/SIMD register arg
1908 * } va_list;
1909 */
1910 const auto VAList = I.getOperand(i: 0).getReg();
1911
1912 // Our current offset in bytes from the va_list struct (VAList).
1913 unsigned OffsetBytes = 0;
1914
1915 // Helper function to store (FrameIndex + Imm) to VAList at offset OffsetBytes
1916 // and increment OffsetBytes by PtrSize.
1917 const auto PushAddress = [&](const int FrameIndex, const int64_t Imm) {
1918 const Register Top = MRI.createVirtualRegister(RegClass: PtrRegClass);
1919 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDAddAddr)
1920 .addDef(RegNo: Top)
1921 .addFrameIndex(Idx: FrameIndex)
1922 .addImm(Val: Imm)
1923 .addImm(Val: 0);
1924 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1925
1926 const auto *MMO = *I.memoperands_begin();
1927 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDStoreAddr)
1928 .addUse(RegNo: Top)
1929 .addUse(RegNo: VAList)
1930 .addImm(Val: OffsetBytes / PtrSize)
1931 .addMemOperand(MMO: MF.getMachineMemOperand(
1932 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1933 F: MachineMemOperand::MOStore, Size: PtrSize, BaseAlignment: MMO->getBaseAlign()));
1934 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1935
1936 OffsetBytes += PtrSize;
1937 };
1938
1939 // void* stack at offset 0
1940 PushAddress(FuncInfo->getVarArgsStackIndex(), 0);
1941
1942 // void* gr_top at offset 8 (4 on ILP32)
1943 const unsigned GPRSize = FuncInfo->getVarArgsGPRSize();
1944 PushAddress(FuncInfo->getVarArgsGPRIndex(), GPRSize);
1945
1946 // void* vr_top at offset 16 (8 on ILP32)
1947 const unsigned FPRSize = FuncInfo->getVarArgsFPRSize();
1948 PushAddress(FuncInfo->getVarArgsFPRIndex(), FPRSize);
1949
1950 // Helper function to store a 4-byte integer constant to VAList at offset
1951 // OffsetBytes, and increment OffsetBytes by 4.
1952 const auto PushIntConstant = [&](const int32_t Value) {
1953 constexpr int IntSize = 4;
1954 const Register Temp = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
1955 auto MIB =
1956 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVi32imm))
1957 .addDef(RegNo: Temp)
1958 .addImm(Val: Value);
1959 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1960
1961 const auto *MMO = *I.memoperands_begin();
1962 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRWui))
1963 .addUse(RegNo: Temp)
1964 .addUse(RegNo: VAList)
1965 .addImm(Val: OffsetBytes / IntSize)
1966 .addMemOperand(MMO: MF.getMachineMemOperand(
1967 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1968 F: MachineMemOperand::MOStore, Size: IntSize, BaseAlignment: MMO->getBaseAlign()));
1969 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1970 OffsetBytes += IntSize;
1971 };
1972
1973 // int gr_offs at offset 24 (12 on ILP32)
1974 PushIntConstant(-static_cast<int32_t>(GPRSize));
1975
1976 // int vr_offs at offset 28 (16 on ILP32)
1977 PushIntConstant(-static_cast<int32_t>(FPRSize));
1978
1979 assert(OffsetBytes == (STI.isTargetILP32() ? 20 : 32) && "Unexpected offset");
1980
1981 I.eraseFromParent();
1982 return true;
1983}
1984
1985bool AArch64InstructionSelector::selectVaStartDarwin(
1986 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1987 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1988 Register ListReg = I.getOperand(i: 0).getReg();
1989
1990 Register ArgsAddrReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1991
1992 int FrameIdx = FuncInfo->getVarArgsStackIndex();
1993 if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
1994 CC: MF.getFunction().getCallingConv(), IsVarArg: MF.getFunction().isVarArg())) {
1995 FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
1996 ? FuncInfo->getVarArgsGPRIndex()
1997 : FuncInfo->getVarArgsStackIndex();
1998 }
1999
2000 auto MIB =
2001 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
2002 .addDef(RegNo: ArgsAddrReg)
2003 .addFrameIndex(Idx: FrameIdx)
2004 .addImm(Val: 0)
2005 .addImm(Val: 0);
2006
2007 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2008
2009 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRXui))
2010 .addUse(RegNo: ArgsAddrReg)
2011 .addUse(RegNo: ListReg)
2012 .addImm(Val: 0)
2013 .addMemOperand(MMO: *I.memoperands_begin());
2014
2015 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2016 I.eraseFromParent();
2017 return true;
2018}
2019
2020void AArch64InstructionSelector::materializeLargeCMVal(
2021 MachineInstr &I, const Value *V, unsigned OpFlags) {
2022 MachineBasicBlock &MBB = *I.getParent();
2023 MachineFunction &MF = *MBB.getParent();
2024 MachineRegisterInfo &MRI = MF.getRegInfo();
2025
2026 auto MovZ = MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {});
2027 MovZ->addOperand(MF, Op: I.getOperand(i: 1));
2028 MovZ->getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
2029 AArch64II::MO_NC);
2030 MovZ->addOperand(MF, Op: MachineOperand::CreateImm(Val: 0));
2031 constrainSelectedInstRegOperands(I&: *MovZ, TII, TRI, RBI);
2032
2033 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
2034 Register ForceDstReg) {
2035 Register DstReg = ForceDstReg
2036 ? ForceDstReg
2037 : MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2038 auto MovI = MIB.buildInstr(Opcode: AArch64::MOVKXi).addDef(RegNo: DstReg).addUse(RegNo: SrcReg);
2039 if (auto *GV = dyn_cast<GlobalValue>(Val: V)) {
2040 MovI->addOperand(MF, Op: MachineOperand::CreateGA(
2041 GV, Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2042 } else {
2043 MovI->addOperand(
2044 MF, Op: MachineOperand::CreateBA(BA: cast<BlockAddress>(Val: V),
2045 Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2046 }
2047 MovI->addOperand(MF, Op: MachineOperand::CreateImm(Val: Offset));
2048 constrainSelectedInstRegOperands(I&: *MovI, TII, TRI, RBI);
2049 return DstReg;
2050 };
2051 Register DstReg = BuildMovK(MovZ.getReg(Idx: 0),
2052 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
2053 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
2054 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(i: 0).getReg());
2055}
2056
2057bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
2058 MachineBasicBlock &MBB = *I.getParent();
2059 MachineFunction &MF = *MBB.getParent();
2060 MachineRegisterInfo &MRI = MF.getRegInfo();
2061
2062 switch (I.getOpcode()) {
2063 case TargetOpcode::G_CONSTANT: {
2064 Register DefReg = I.getOperand(i: 0).getReg();
2065 const LLT DefTy = MRI.getType(Reg: DefReg);
2066 if (!DefTy.isPointer()) {
2067 if (DefTy.getSizeInBits() >= 32 ||
2068 RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
2069 return false;
2070 // Widen narrow GPR constants to s32 so imported patterns can match.
2071 APInt Val = I.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
2072 I.getOperand(i: 1).setCImm(
2073 ConstantInt::get(Context&: MF.getFunction().getContext(), V: Val));
2074
2075 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2076 MRI.setRegBank(Reg: WideReg, RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2077 I.getOperand(i: 0).setReg(WideReg);
2078
2079 MIB.setInsertPt(MBB, II: std::next(x: I.getIterator()));
2080 auto Copy = MIB.buildCopy(Res: DefReg, Op: WideReg);
2081 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
2082 MIB.setInstr(I);
2083 return true;
2084 }
2085 const unsigned PtrSize = DefTy.getSizeInBits();
2086 if (PtrSize != 32 && PtrSize != 64)
2087 return false;
2088 // Convert pointer typed constants to integers so TableGen can select.
2089 MRI.setType(VReg: DefReg, Ty: LLT::integer(SizeInBits: PtrSize));
2090 return true;
2091 }
2092 case TargetOpcode::G_STORE: {
2093 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
2094 MachineOperand &SrcOp = I.getOperand(i: 0);
2095 if (MRI.getType(Reg: SrcOp.getReg()).isPointer()) {
2096 // Allow matching with imported patterns for stores of pointers. Unlike
2097 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
2098 // and constrain.
2099 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: SrcOp);
2100 Register NewSrc = Copy.getReg(Idx: 0);
2101 SrcOp.setReg(NewSrc);
2102 RBI.constrainGenericRegister(Reg: NewSrc, RC: AArch64::GPR64RegClass, MRI);
2103 Changed = true;
2104 }
2105 return Changed;
2106 }
2107 case TargetOpcode::G_PTR_ADD: {
2108 // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
2109 // arithmetic semantics instead of falling back to regular arithmetic.
2110 const auto &TL = STI.getTargetLowering();
2111 if (TL->shouldPreservePtrArith(F: MF.getFunction(), PtrVT: EVT()))
2112 return false;
2113 return convertPtrAddToAdd(I, MRI);
2114 }
2115 case TargetOpcode::G_LOAD: {
2116 // For scalar loads of pointers, we try to convert the dest type from p0
2117 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
2118 // conversion, this should be ok because all users should have been
2119 // selected already, so the type doesn't matter for them.
2120 Register DstReg = I.getOperand(i: 0).getReg();
2121 const LLT DstTy = MRI.getType(Reg: DstReg);
2122 if (!DstTy.isPointer())
2123 return false;
2124 MRI.setType(VReg: DstReg, Ty: LLT::scalar(SizeInBits: 64));
2125 return true;
2126 }
2127 case TargetOpcode::G_VECREDUCE_ADD:
2128 case TargetOpcode::G_VECREDUCE_SMAX:
2129 case TargetOpcode::G_VECREDUCE_SMIN:
2130 case TargetOpcode::G_VECREDUCE_UMAX:
2131 case TargetOpcode::G_VECREDUCE_UMIN: {
2132 // Imported patterns require an FPR result. For a GPR, use a temporary FPR
2133 // and insert a cross-bank copy.
2134 Register DstReg = I.getOperand(i: 0).getReg();
2135 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2136 if (DstRB.getID() != AArch64::GPRRegBankID)
2137 return false;
2138
2139 LLT DstTy = MRI.getType(Reg: DstReg);
2140 const TargetRegisterClass *DstRC =
2141 getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB, /*GetAllRegSet=*/true);
2142 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI))
2143 return false;
2144
2145 Register FPRDst = MRI.createGenericVirtualRegister(Ty: DstTy);
2146 MRI.setRegBank(Reg: FPRDst, RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2147 I.getOperand(i: 0).setReg(FPRDst);
2148
2149 BuildMI(BB&: MBB, I: std::next(x: I.getIterator()), MIMD: MIMetadata(I),
2150 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
2151 .addReg(RegNo: FPRDst);
2152 return true;
2153 }
2154 case AArch64::G_DUP: {
2155 // Convert the type from p0 to s64 to help selection.
2156 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2157 if (!DstTy.isPointerVector())
2158 return false;
2159 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 1).getReg());
2160 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2161 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2162 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2163 I.getOperand(i: 1).setReg(NewSrc.getReg(Idx: 0));
2164 return true;
2165 }
2166 case AArch64::G_INSERT_VECTOR_ELT: {
2167 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2168 LLT SrcVecTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
2169 if (SrcVecTy.isPointerVector()) {
2170 // Convert the type from p0 to s64 to help selection.
2171 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 2).getReg());
2172 MRI.setType(VReg: I.getOperand(i: 1).getReg(),
2173 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2174 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2175 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2176 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2177 I.getOperand(i: 2).setReg(NewSrc.getReg(Idx: 0));
2178 return true;
2179 }
2180
2181 Register EltReg = I.getOperand(i: 2).getReg();
2182 LLT EltTy = MRI.getType(Reg: EltReg);
2183 if (EltTy.isScalar() &&
2184 (EltTy.getSizeInBits() == 8 || EltTy.getSizeInBits() == 16) &&
2185 RBI.getRegBank(Reg: EltReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) {
2186 // Convert the type from s8/s16 to s32 to help selection.
2187 auto NewElt = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 32), Op: EltReg);
2188 MRI.setRegClass(Reg: NewElt.getReg(Idx: 0), RC: &AArch64::GPR32RegClass);
2189 I.getOperand(i: 2).setReg(NewElt.getReg(Idx: 0));
2190 return true;
2191 }
2192 return false;
2193 }
2194 case TargetOpcode::G_UITOFP:
2195 case TargetOpcode::G_SITOFP: {
2196 // If both source and destination regbanks are FPR, then convert the opcode
2197 // to G_SITOF so that the importer can select it to an fpr variant.
2198 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
2199 // copy.
2200 Register SrcReg = I.getOperand(i: 1).getReg();
2201 LLT SrcTy = MRI.getType(Reg: SrcReg);
2202 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2203 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
2204 return false;
2205
2206 if (RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
2207 // Need to add a copy to change the type so that the existing patterns can
2208 // match when there is an integer on an FPR bank.
2209 if (SrcTy.getScalarType().isInteger()) {
2210 auto Copy = MIB.buildCopy(Res: DstTy, Op: SrcReg);
2211 I.getOperand(i: 1).setReg(Copy.getReg(Idx: 0));
2212 MRI.setRegClass(Reg: Copy.getReg(Idx: 0),
2213 RC: getRegClassForTypeOnBank(
2214 Ty: SrcTy, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID)));
2215 }
2216 if (I.getOpcode() == TargetOpcode::G_SITOFP)
2217 I.setDesc(TII.get(Opcode: AArch64::G_SITOF));
2218 else
2219 I.setDesc(TII.get(Opcode: AArch64::G_UITOF));
2220 return true;
2221 }
2222 return false;
2223 }
2224 default:
2225 return false;
2226 }
2227}
2228
2229/// This lowering tries to look for G_PTR_ADD instructions and then converts
2230/// them to a standard G_ADD with a COPY on the source.
2231///
2232/// The motivation behind this is to expose the add semantics to the imported
2233/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2234/// because the selector works bottom up, uses before defs. By the time we
2235/// end up trying to select a G_PTR_ADD, we should have already attempted to
2236/// fold this into addressing modes and were therefore unsuccessful.
2237bool AArch64InstructionSelector::convertPtrAddToAdd(
2238 MachineInstr &I, MachineRegisterInfo &MRI) {
2239 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
2240 Register DstReg = I.getOperand(i: 0).getReg();
2241 Register AddOp1Reg = I.getOperand(i: 1).getReg();
2242 const LLT PtrTy = MRI.getType(Reg: DstReg);
2243 if (PtrTy.getAddressSpace() != 0)
2244 return false;
2245
2246 const LLT CastPtrTy = PtrTy.isVector()
2247 ? LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 64))
2248 : LLT::integer(SizeInBits: 64);
2249 auto PtrToInt = MIB.buildPtrToInt(Dst: CastPtrTy, Src: AddOp1Reg);
2250 // Set regbanks on the registers.
2251 if (PtrTy.isVector())
2252 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2253 else
2254 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2255
2256 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2257 // %dst(intty) = G_ADD %intbase, off
2258 I.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
2259 MRI.setType(VReg: DstReg, Ty: CastPtrTy);
2260 I.getOperand(i: 1).setReg(PtrToInt.getReg(Idx: 0));
2261 if (!select(I&: *PtrToInt)) {
2262 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd");
2263 return false;
2264 }
2265
2266 // Also take the opportunity here to try to do some optimization.
2267 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2268 Register NegatedReg;
2269 if (!mi_match(R: I.getOperand(i: 2).getReg(), MRI, P: m_Neg(Src: m_Reg(R&: NegatedReg))))
2270 return true;
2271 I.getOperand(i: 2).setReg(NegatedReg);
2272 I.setDesc(TII.get(Opcode: TargetOpcode::G_SUB));
2273 return true;
2274}
2275
2276bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
2277 MachineRegisterInfo &MRI) {
2278 // We try to match the immediate variant of LSL, which is actually an alias
2279 // for a special case of UBFM. Otherwise, we fall back to the imported
2280 // selector which will match the register variant.
2281 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op");
2282 const auto &MO = I.getOperand(i: 2);
2283 auto VRegAndVal = getIConstantVRegVal(VReg: MO.getReg(), MRI);
2284 if (!VRegAndVal)
2285 return false;
2286
2287 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2288 if (DstTy.isVector())
2289 return false;
2290 bool Is64Bit = DstTy.getSizeInBits() == 64;
2291 auto Imm1Fn = Is64Bit ? selectShiftA_64(Root: MO) : selectShiftA_32(Root: MO);
2292 auto Imm2Fn = Is64Bit ? selectShiftB_64(Root: MO) : selectShiftB_32(Root: MO);
2293
2294 if (!Imm1Fn || !Imm2Fn)
2295 return false;
2296
2297 auto NewI =
2298 MIB.buildInstr(Opc: Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2299 DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {I.getOperand(i: 1).getReg()});
2300
2301 for (auto &RenderFn : *Imm1Fn)
2302 RenderFn(NewI);
2303 for (auto &RenderFn : *Imm2Fn)
2304 RenderFn(NewI);
2305
2306 I.eraseFromParent();
2307 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
2308 return true;
2309}
2310
2311bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2312 MachineInstr &I, MachineRegisterInfo &MRI) {
2313 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE");
2314 // If we're storing a scalar, it doesn't matter what register bank that
2315 // scalar is on. All that matters is the size.
2316 //
2317 // So, if we see something like this (with a 32-bit scalar as an example):
2318 //
2319 // %x:gpr(s32) = ... something ...
2320 // %y:fpr(s32) = COPY %x:gpr(s32)
2321 // G_STORE %y:fpr(s32)
2322 //
2323 // We can fix this up into something like this:
2324 //
2325 // G_STORE %x:gpr(s32)
2326 //
2327 // And then continue the selection process normally.
2328 Register DefDstReg = getSrcRegIgnoringCopies(Reg: I.getOperand(i: 0).getReg(), MRI);
2329 if (!DefDstReg.isValid())
2330 return false;
2331 LLT DefDstTy = MRI.getType(Reg: DefDstReg);
2332 Register StoreSrcReg = I.getOperand(i: 0).getReg();
2333 LLT StoreSrcTy = MRI.getType(Reg: StoreSrcReg);
2334
2335 // If we get something strange like a physical register, then we shouldn't
2336 // go any further.
2337 if (!DefDstTy.isValid())
2338 return false;
2339
2340 // Are the source and dst types the same size?
2341 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2342 return false;
2343
2344 if (RBI.getRegBank(Reg: StoreSrcReg, MRI, TRI) ==
2345 RBI.getRegBank(Reg: DefDstReg, MRI, TRI))
2346 return false;
2347
2348 // We have a cross-bank copy, which is entering a store. Let's fold it.
2349 I.getOperand(i: 0).setReg(DefDstReg);
2350 return true;
2351}
2352
2353bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
2354 assert(I.getParent() && "Instruction should be in a basic block!");
2355 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2356
2357 MachineBasicBlock &MBB = *I.getParent();
2358 MachineFunction &MF = *MBB.getParent();
2359 MachineRegisterInfo &MRI = MF.getRegInfo();
2360
2361 switch (I.getOpcode()) {
2362 case AArch64::G_DUP: {
2363 // Before selecting a DUP instruction, check if it is better selected as a
2364 // MOV or load from a constant pool.
2365 Register Src = I.getOperand(i: 1).getReg();
2366 auto ValAndVReg = getAnyConstantVRegValWithLookThrough(
2367 VReg: Src, MRI, /*LookThroughInstrs=*/true, /*LookThroughAnyExt=*/true);
2368 if (!ValAndVReg)
2369 return false;
2370 LLVMContext &Ctx = MF.getFunction().getContext();
2371 Register Dst = I.getOperand(i: 0).getReg();
2372 auto *CV = ConstantDataVector::getSplat(
2373 NumElts: MRI.getType(Reg: Dst).getNumElements(),
2374 Elt: ConstantInt::get(
2375 Ty: Type::getIntNTy(C&: Ctx, N: MRI.getType(Reg: Dst).getScalarSizeInBits()),
2376 V: ValAndVReg->Value.trunc(width: MRI.getType(Reg: Dst).getScalarSizeInBits())));
2377 if (!emitConstantVector(Dst, CV, MIRBuilder&: MIB, MRI))
2378 return false;
2379 I.eraseFromParent();
2380 return true;
2381 }
2382 case TargetOpcode::G_SEXT:
2383 // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
2384 // over a normal extend.
2385 if (selectUSMovFromExtend(I, MRI))
2386 return true;
2387 return false;
2388 case TargetOpcode::G_BR:
2389 return false;
2390 case TargetOpcode::G_SHL:
2391 return earlySelectSHL(I, MRI);
2392 case TargetOpcode::G_CONSTANT: {
2393 bool IsZero = false;
2394 if (I.getOperand(i: 1).isCImm())
2395 IsZero = I.getOperand(i: 1).getCImm()->isZero();
2396 else if (I.getOperand(i: 1).isImm())
2397 IsZero = I.getOperand(i: 1).getImm() == 0;
2398
2399 if (!IsZero)
2400 return false;
2401
2402 Register DefReg = I.getOperand(i: 0).getReg();
2403 LLT Ty = MRI.getType(Reg: DefReg);
2404 if (Ty.getSizeInBits() == 64) {
2405 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::XZR, isDef: false);
2406 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
2407 } else if (Ty.getSizeInBits() <= 32) {
2408 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::WZR, isDef: false);
2409 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR32RegClass, MRI);
2410 } else
2411 return false;
2412
2413 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2414 return true;
2415 }
2416
2417 case TargetOpcode::G_ADD: {
2418 // Check if this is being fed by a G_ICMP on either side.
2419 //
2420 // (cmp pred, x, y) + z
2421 //
2422 // In the above case, when the cmp is true, we increment z by 1. So, we can
2423 // fold the add into the cset for the cmp by using cinc.
2424 //
2425 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2426 Register AddDst = I.getOperand(i: 0).getReg();
2427 Register AddLHS = I.getOperand(i: 1).getReg();
2428 Register AddRHS = I.getOperand(i: 2).getReg();
2429 // Only handle scalars.
2430 LLT Ty = MRI.getType(Reg: AddLHS);
2431 if (Ty.isVector())
2432 return false;
2433 // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
2434 // bits.
2435 unsigned Size = Ty.getSizeInBits();
2436 if (Size != 32 && Size != 64)
2437 return false;
2438 auto MatchCmp = [&](Register Reg) -> MachineInstr * {
2439 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2440 return nullptr;
2441 // If the LHS of the add is 32 bits, then we want to fold a 32-bit
2442 // compare.
2443 if (Size == 32)
2444 return getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg, MRI);
2445 // We model scalar compares using 32-bit destinations right now.
2446 // If it's a 64-bit compare, it'll have 64-bit sources.
2447 Register ZExt;
2448 if (!mi_match(R: Reg, MRI,
2449 P: m_OneNonDBGUse(SP: m_GZExt(Src: m_OneNonDBGUse(SP: m_Reg(R&: ZExt))))))
2450 return nullptr;
2451 auto *Cmp = getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg: ZExt, MRI);
2452 if (!Cmp ||
2453 MRI.getType(Reg: Cmp->getOperand(i: 2).getReg()).getSizeInBits() != 64)
2454 return nullptr;
2455 return Cmp;
2456 };
2457 // Try to match
2458 // z + (cmp pred, x, y)
2459 MachineInstr *Cmp = MatchCmp(AddRHS);
2460 if (!Cmp) {
2461 // (cmp pred, x, y) + z
2462 std::swap(a&: AddLHS, b&: AddRHS);
2463 Cmp = MatchCmp(AddRHS);
2464 if (!Cmp)
2465 return false;
2466 }
2467 auto &PredOp = Cmp->getOperand(i: 1);
2468 MIB.setInstrAndDebugLoc(I);
2469 emitIntegerCompare(/*LHS=*/Cmp->getOperand(i: 2),
2470 /*RHS=*/Cmp->getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
2471 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
2472 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
2473 P: CmpInst::getInversePredicate(pred: Pred), RHS: Cmp->getOperand(i: 3).getReg(), MRI: &MRI);
2474 emitCSINC(/*Dst=*/AddDst, /*Src =*/Src1: AddLHS, /*Src2=*/AddLHS, Pred: InvCC, MIRBuilder&: MIB);
2475 I.eraseFromParent();
2476 return true;
2477 }
2478 case TargetOpcode::G_OR: {
2479 // Look for operations that take the lower `Width=Size-ShiftImm` bits of
2480 // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
2481 // shifting and masking that we can replace with a BFI (encoded as a BFM).
2482 Register Dst = I.getOperand(i: 0).getReg();
2483 LLT Ty = MRI.getType(Reg: Dst);
2484
2485 if (!Ty.isScalar())
2486 return false;
2487
2488 unsigned Size = Ty.getSizeInBits();
2489 if (Size != 32 && Size != 64)
2490 return false;
2491
2492 Register ShiftSrc;
2493 int64_t ShiftImm;
2494 Register MaskSrc;
2495 int64_t MaskImm;
2496 if (!mi_match(
2497 R: Dst, MRI,
2498 P: m_GOr(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))),
2499 R: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: MaskSrc), R: m_ICst(Cst&: MaskImm))))))
2500 return false;
2501
2502 if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
2503 return false;
2504
2505 int64_t Immr = Size - ShiftImm;
2506 int64_t Imms = Size - ShiftImm - 1;
2507 unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2508 emitInstr(Opcode: Opc, DstOps: {Dst}, SrcOps: {MaskSrc, ShiftSrc, Immr, Imms}, MIRBuilder&: MIB);
2509 I.eraseFromParent();
2510 return true;
2511 }
2512 case TargetOpcode::G_FENCE: {
2513 if (I.getOperand(i: 1).getImm() == 0)
2514 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2515 else
2516 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: AArch64::DMB))
2517 .addImm(Val: I.getOperand(i: 0).getImm() == 4 ? 0x9 : 0xb);
2518 I.eraseFromParent();
2519 return true;
2520 }
2521 default:
2522 return false;
2523 }
2524}
2525
2526bool AArch64InstructionSelector::select(MachineInstr &I) {
2527 assert(I.getParent() && "Instruction should be in a basic block!");
2528 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2529
2530 MachineBasicBlock &MBB = *I.getParent();
2531 MachineFunction &MF = *MBB.getParent();
2532 MachineRegisterInfo &MRI = MF.getRegInfo();
2533
2534 const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
2535 if (Subtarget->requiresStrictAlign()) {
2536 // We don't support this feature yet.
2537 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
2538 return false;
2539 }
2540
2541 MIB.setInstrAndDebugLoc(I);
2542
2543 unsigned Opcode = I.getOpcode();
2544 // G_PHI requires same handling as PHI
2545 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2546 // Certain non-generic instructions also need some special handling.
2547
2548 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2549 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2550 return true;
2551 }
2552
2553 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2554 const Register DefReg = I.getOperand(i: 0).getReg();
2555 const LLT DefTy = MRI.getType(Reg: DefReg);
2556
2557 const RegClassOrRegBank &RegClassOrBank =
2558 MRI.getRegClassOrRegBank(Reg: DefReg);
2559
2560 const TargetRegisterClass *DefRC =
2561 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
2562 if (!DefRC) {
2563 if (!DefTy.isValid()) {
2564 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
2565 return false;
2566 }
2567 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
2568 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
2569 if (!DefRC) {
2570 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
2571 return false;
2572 }
2573 }
2574
2575 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
2576
2577 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI);
2578 }
2579
2580 if (I.isCopy())
2581 return selectCopy(I, TII, MRI, TRI, RBI);
2582
2583 if (I.isDebugInstr())
2584 return selectDebugInstr(I, MRI, RBI);
2585
2586 return true;
2587 }
2588
2589
2590 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2591 LLVM_DEBUG(
2592 dbgs() << "Generic instruction has unexpected implicit operands\n");
2593 return false;
2594 }
2595
2596 // Try to do some lowering before we start instruction selecting. These
2597 // lowerings are purely transformations on the input G_MIR and so selection
2598 // must continue after any modification of the instruction.
2599 if (preISelLower(I)) {
2600 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2601 }
2602
2603 // There may be patterns where the importer can't deal with them optimally,
2604 // but does select it to a suboptimal sequence so our custom C++ selection
2605 // code later never has a chance to work on it. Therefore, we have an early
2606 // selection attempt here to give priority to certain selection routines
2607 // over the imported ones.
2608 if (earlySelect(I))
2609 return true;
2610
2611 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2612 return true;
2613
2614 LLT Ty =
2615 I.getOperand(i: 0).isReg() ? MRI.getType(Reg: I.getOperand(i: 0).getReg()) : LLT{};
2616
2617 switch (Opcode) {
2618 case TargetOpcode::G_SBFX:
2619 case TargetOpcode::G_UBFX: {
2620 static const unsigned OpcTable[2][2] = {
2621 {AArch64::UBFMWri, AArch64::UBFMXri},
2622 {AArch64::SBFMWri, AArch64::SBFMXri}};
2623 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2624 unsigned Size = Ty.getSizeInBits();
2625 unsigned Opc = OpcTable[IsSigned][Size == 64];
2626 auto Cst1 =
2627 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 2).getReg(), MRI);
2628 assert(Cst1 && "Should have gotten a constant for src 1?");
2629 auto Cst2 =
2630 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 3).getReg(), MRI);
2631 assert(Cst2 && "Should have gotten a constant for src 2?");
2632 auto LSB = Cst1->Value.getZExtValue();
2633 auto Width = Cst2->Value.getZExtValue();
2634 auto BitfieldInst =
2635 MIB.buildInstr(Opc, DstOps: {I.getOperand(i: 0)}, SrcOps: {I.getOperand(i: 1)})
2636 .addImm(Val: LSB)
2637 .addImm(Val: LSB + Width - 1);
2638 I.eraseFromParent();
2639 constrainSelectedInstRegOperands(I&: *BitfieldInst, TII, TRI, RBI);
2640 return true;
2641 }
2642 case TargetOpcode::G_BRCOND:
2643 return selectCompareBranch(I, MF, MRI);
2644
2645 case TargetOpcode::G_BRINDIRECT: {
2646 const Function &Fn = MF.getFunction();
2647 if (std::optional<uint16_t> BADisc =
2648 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn)) {
2649 auto MI = MIB.buildInstr(Opc: AArch64::BRA, DstOps: {}, SrcOps: {I.getOperand(i: 0).getReg()});
2650 MI.addImm(Val: AArch64PACKey::IA);
2651 MI.addImm(Val: *BADisc);
2652 MI.addReg(/*AddrDisc=*/RegNo: AArch64::XZR);
2653 I.eraseFromParent();
2654 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
2655 return true;
2656 }
2657 I.setDesc(TII.get(Opcode: AArch64::BR));
2658 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2659 return true;
2660 }
2661
2662 case TargetOpcode::G_BRJT:
2663 return selectBrJT(I, MRI);
2664
2665 case AArch64::G_ADD_LOW: {
2666 // This op may have been separated from it's ADRP companion by the localizer
2667 // or some other code motion pass. Given that many CPUs will try to
2668 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2669 // which will later be expanded into an ADRP+ADD pair after scheduling.
2670 MachineInstr *BaseMI = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
2671 if (BaseMI->getOpcode() != AArch64::ADRP) {
2672 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2673 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2674 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2675 return true;
2676 }
2677 assert(TM.getCodeModel() == CodeModel::Small &&
2678 "Expected small code model");
2679 auto Op1 = BaseMI->getOperand(i: 1);
2680 auto Op2 = I.getOperand(i: 2);
2681 auto MovAddr = MIB.buildInstr(Opc: AArch64::MOVaddr, DstOps: {I.getOperand(i: 0)}, SrcOps: {})
2682 .addGlobalAddress(GV: Op1.getGlobal(), Offset: Op1.getOffset(),
2683 TargetFlags: Op1.getTargetFlags())
2684 .addGlobalAddress(GV: Op2.getGlobal(), Offset: Op2.getOffset(),
2685 TargetFlags: Op2.getTargetFlags());
2686 I.eraseFromParent();
2687 constrainSelectedInstRegOperands(I&: *MovAddr, TII, TRI, RBI);
2688 return true;
2689 }
2690
2691 case TargetOpcode::G_FCONSTANT: {
2692 const Register DefReg = I.getOperand(i: 0).getReg();
2693 const LLT DefTy = MRI.getType(Reg: DefReg);
2694 const unsigned DefSize = DefTy.getSizeInBits();
2695 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
2696
2697 const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(Ty: DefTy, RB);
2698 // For 16, 64, and 128b values, emit a constant pool load.
2699 switch (DefSize) {
2700 default:
2701 llvm_unreachable("Unexpected destination size for G_FCONSTANT?");
2702 case 32:
2703 case 64: {
2704 bool OptForSize = shouldOptForSize(MF: &MF);
2705 const auto &TLI = MF.getSubtarget().getTargetLowering();
2706 // If TLI says that this fpimm is illegal, then we'll expand to a
2707 // constant pool load.
2708 if (TLI->isFPImmLegal(I.getOperand(i: 1).getFPImm()->getValueAPF(),
2709 EVT::getFloatingPointVT(BitWidth: DefSize), ForCodeSize: OptForSize))
2710 break;
2711 [[fallthrough]];
2712 }
2713 case 16:
2714 case 128: {
2715 auto *FPImm = I.getOperand(i: 1).getFPImm();
2716 auto *LoadMI = emitLoadFromConstantPool(CPVal: FPImm, MIRBuilder&: MIB);
2717 if (!LoadMI) {
2718 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n");
2719 return false;
2720 }
2721 MIB.buildCopy(Res: {DefReg}, Op: {LoadMI->getOperand(i: 0).getReg()});
2722 I.eraseFromParent();
2723 return RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2724 }
2725 }
2726
2727 assert((DefSize == 32 || DefSize == 64) && "Unexpected const def size");
2728 // Either emit a FMOV, or emit a copy to emit a normal mov.
2729 const Register DefGPRReg = MRI.createVirtualRegister(
2730 RegClass: DefSize == 32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2731 MachineOperand &RegOp = I.getOperand(i: 0);
2732 RegOp.setReg(DefGPRReg);
2733 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2734 MIB.buildCopy(Res: {DefReg}, Op: {DefGPRReg});
2735
2736 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI)) {
2737 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
2738 return false;
2739 }
2740
2741 MachineOperand &ImmOp = I.getOperand(i: 1);
2742 ImmOp.ChangeToImmediate(
2743 ImmVal: ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2744
2745 const unsigned MovOpc =
2746 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2747 I.setDesc(TII.get(Opcode: MovOpc));
2748 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2749 return true;
2750 }
2751 case TargetOpcode::G_EXTRACT: {
2752 Register DstReg = I.getOperand(i: 0).getReg();
2753 Register SrcReg = I.getOperand(i: 1).getReg();
2754 LLT SrcTy = MRI.getType(Reg: SrcReg);
2755 LLT DstTy = MRI.getType(Reg: DstReg);
2756 (void)DstTy;
2757 unsigned SrcSize = SrcTy.getSizeInBits();
2758
2759 if (SrcTy.getSizeInBits() > 64) {
2760 // This should be an extract of an s128, which is like a vector extract.
2761 if (SrcTy.getSizeInBits() != 128)
2762 return false;
2763 // Only support extracting 64 bits from an s128 at the moment.
2764 if (DstTy.getSizeInBits() != 64)
2765 return false;
2766
2767 unsigned Offset = I.getOperand(i: 2).getImm();
2768 if (Offset % 64 != 0)
2769 return false;
2770
2771 // Check we have the right regbank always.
2772 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
2773 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2774 assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!");
2775
2776 if (SrcRB.getID() == AArch64::GPRRegBankID) {
2777 auto NewI =
2778 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
2779 .addUse(RegNo: SrcReg, Flags: {},
2780 SubReg: Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2781 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: *NewI,
2782 RegClass: AArch64::GPR64RegClass, RegMO&: NewI->getOperand(i: 0));
2783 I.eraseFromParent();
2784 return true;
2785 }
2786
2787 // Emit the same code as a vector extract.
2788 // Offset must be a multiple of 64.
2789 unsigned LaneIdx = Offset / 64;
2790 MachineInstr *Extract = emitExtractVectorElt(
2791 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: 64), VecReg: SrcReg, LaneIdx, MIRBuilder&: MIB);
2792 if (!Extract)
2793 return false;
2794 I.eraseFromParent();
2795 return true;
2796 }
2797
2798 I.setDesc(TII.get(Opcode: SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2799 MachineInstrBuilder(MF, I).addImm(Val: I.getOperand(i: 2).getImm() +
2800 Ty.getSizeInBits() - 1);
2801
2802 if (SrcSize < 64) {
2803 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
2804 "unexpected G_EXTRACT types");
2805 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2806 return true;
2807 }
2808
2809 DstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2810 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2811 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
2812 .addReg(RegNo: DstReg, Flags: {}, SubReg: AArch64::sub_32);
2813 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
2814 RC: AArch64::GPR32RegClass, MRI);
2815 I.getOperand(i: 0).setReg(DstReg);
2816
2817 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2818 return true;
2819 }
2820
2821 case TargetOpcode::G_INSERT: {
2822 LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 2).getReg());
2823 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2824 unsigned DstSize = DstTy.getSizeInBits();
2825 // Larger inserts are vectors, same-size ones should be something else by
2826 // now (split up or turned into COPYs).
2827 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2828 return false;
2829
2830 I.setDesc(TII.get(Opcode: DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2831 unsigned LSB = I.getOperand(i: 3).getImm();
2832 unsigned Width = MRI.getType(Reg: I.getOperand(i: 2).getReg()).getSizeInBits();
2833 I.getOperand(i: 3).setImm((DstSize - LSB) % DstSize);
2834 MachineInstrBuilder(MF, I).addImm(Val: Width - 1);
2835
2836 if (DstSize < 64) {
2837 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
2838 "unexpected G_INSERT types");
2839 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2840 return true;
2841 }
2842
2843 Register SrcReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2844 BuildMI(BB&: MBB, I: I.getIterator(), MIMD: I.getDebugLoc(),
2845 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
2846 .addDef(RegNo: SrcReg)
2847 .addUse(RegNo: I.getOperand(i: 2).getReg())
2848 .addImm(Val: AArch64::sub_32);
2849 RBI.constrainGenericRegister(Reg: I.getOperand(i: 2).getReg(),
2850 RC: AArch64::GPR32RegClass, MRI);
2851 I.getOperand(i: 2).setReg(SrcReg);
2852
2853 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2854 return true;
2855 }
2856 case TargetOpcode::G_FRAME_INDEX: {
2857 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2858 if (Ty != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2859 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
2860 << ", expected: " << LLT::pointer(0, 64) << '\n');
2861 return false;
2862 }
2863 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2864
2865 // MOs for a #0 shifted immediate.
2866 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2867 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2868
2869 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2870 return true;
2871 }
2872
2873 case TargetOpcode::G_GLOBAL_VALUE: {
2874 const GlobalValue *GV = nullptr;
2875 unsigned OpFlags;
2876 if (I.getOperand(i: 1).isSymbol()) {
2877 OpFlags = I.getOperand(i: 1).getTargetFlags();
2878 // Currently only used by "RtLibUseGOT".
2879 assert(OpFlags == AArch64II::MO_GOT);
2880 } else {
2881 GV = I.getOperand(i: 1).getGlobal();
2882 if (GV->isThreadLocal()) {
2883 // We don't support instructions with emulated TLS variables yet
2884 if (TM.useEmulatedTLS())
2885 return false;
2886 return selectTLSGlobalValue(I, MRI);
2887 }
2888 OpFlags = STI.ClassifyGlobalReference(GV, TM);
2889 }
2890
2891 if (OpFlags & AArch64II::MO_GOT) {
2892 bool IsGOTSigned = MF.getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2893 I.setDesc(TII.get(Opcode: IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2894 I.getOperand(i: 1).setTargetFlags(OpFlags);
2895 I.addImplicitDefUseOperands(MF);
2896 } else if (TM.getCodeModel() == CodeModel::Large &&
2897 !TM.isPositionIndependent()) {
2898 // Materialize the global using movz/movk instructions.
2899 materializeLargeCMVal(I, V: GV, OpFlags);
2900 I.eraseFromParent();
2901 return true;
2902 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2903 I.setDesc(TII.get(Opcode: AArch64::ADR));
2904 I.getOperand(i: 1).setTargetFlags(OpFlags);
2905 } else {
2906 I.setDesc(TII.get(Opcode: AArch64::MOVaddr));
2907 I.getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2908 MachineInstrBuilder MIB(MF, I);
2909 MIB.addGlobalAddress(GV, Offset: I.getOperand(i: 1).getOffset(),
2910 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2911 }
2912 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2913 return true;
2914 }
2915
2916 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2917 return selectPtrAuthGlobalValue(I, MRI);
2918
2919 case TargetOpcode::G_ZEXTLOAD:
2920 case TargetOpcode::G_LOAD:
2921 case TargetOpcode::G_STORE: {
2922 GLoadStore &LdSt = cast<GLoadStore>(Val&: I);
2923 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2924 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
2925
2926 // Can only handle AddressSpace 0, 64-bit pointers.
2927 if (PtrTy != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2928 return false;
2929 }
2930
2931 uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
2932 unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
2933 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
2934
2935 // Need special instructions for atomics that affect ordering.
2936 if (isStrongerThanMonotonic(AO: Order)) {
2937 assert(!isa<GZExtLoad>(LdSt));
2938 assert(MemSizeInBytes <= 8 &&
2939 "128-bit atomics should already be custom-legalized");
2940
2941 if (isa<GLoad>(Val: LdSt)) {
2942 static constexpr unsigned LDAPROpcodes[] = {
2943 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2944 static constexpr unsigned LDAROpcodes[] = {
2945 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2946 ArrayRef<unsigned> Opcodes =
2947 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2948 ? LDAPROpcodes
2949 : LDAROpcodes;
2950 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2951 } else {
2952 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2953 AArch64::STLRW, AArch64::STLRX};
2954 Register ValReg = LdSt.getReg(Idx: 0);
2955 if (MRI.getType(Reg: ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2956 // Emit a subreg copy of 32 bits.
2957 Register NewVal = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
2958 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {NewVal}, SrcOps: {})
2959 .addReg(RegNo: I.getOperand(i: 0).getReg(), Flags: {}, SubReg: AArch64::sub_32);
2960 I.getOperand(i: 0).setReg(NewVal);
2961 }
2962 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2963 }
2964 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2965 return true;
2966 }
2967
2968#ifndef NDEBUG
2969 const Register PtrReg = LdSt.getPointerReg();
2970 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2971 // Check that the pointer register is valid.
2972 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
2973 "Load/Store pointer operand isn't a GPR");
2974 assert(MRI.getType(PtrReg).isPointer() &&
2975 "Load/Store pointer operand isn't a pointer");
2976#endif
2977
2978 const Register ValReg = LdSt.getReg(Idx: 0);
2979 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI, TRI);
2980 LLT ValTy = MRI.getType(Reg: ValReg);
2981
2982 // The code below doesn't support truncating stores, so we need to split it
2983 // again.
2984 if (isa<GStore>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits &&
2985 RB.getID() == AArch64::FPRRegBankID) {
2986 unsigned SubReg;
2987 LLT MemTy = LdSt.getMMO().getMemoryType();
2988 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
2989 if (!getSubRegForClass(RC, TRI, SubReg))
2990 return false;
2991
2992 // Generate a subreg copy.
2993 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {MemTy}, SrcOps: {})
2994 .addReg(RegNo: ValReg, Flags: {}, SubReg)
2995 .getReg(Idx: 0);
2996 RBI.constrainGenericRegister(Reg: Copy, RC: *RC, MRI);
2997 LdSt.getOperand(i: 0).setReg(Copy);
2998 } else if (isa<GLoad>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
2999 // If this is an any-extending load from the FPR bank, split it into a regular
3000 // load + extend.
3001 if (RB.getID() == AArch64::FPRRegBankID) {
3002 unsigned SubReg;
3003 LLT MemTy = LdSt.getMMO().getMemoryType();
3004 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
3005 if (!getSubRegForClass(RC, TRI, SubReg))
3006 return false;
3007 Register OldDst = LdSt.getReg(Idx: 0);
3008 Register NewDst =
3009 MRI.createGenericVirtualRegister(Ty: LdSt.getMMO().getMemoryType());
3010 LdSt.getOperand(i: 0).setReg(NewDst);
3011 MRI.setRegBank(Reg: NewDst, RegBank: RB);
3012 // Generate a SUBREG_TO_REG to extend it.
3013 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LdSt.getIterator()));
3014 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {OldDst}, SrcOps: {})
3015 .addUse(RegNo: NewDst)
3016 .addImm(Val: SubReg);
3017 auto SubRegRC = getRegClassForTypeOnBank(Ty: MRI.getType(Reg: OldDst), RB);
3018 RBI.constrainGenericRegister(Reg: OldDst, RC: *SubRegRC, MRI);
3019 MIB.setInstr(LdSt);
3020 ValTy = MemTy; // This is no longer an extending load.
3021 }
3022 }
3023
3024 // Helper lambda for partially selecting I. Either returns the original
3025 // instruction with an updated opcode, or a new instruction.
3026 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3027 bool IsStore = isa<GStore>(Val: I);
3028 const unsigned NewOpc =
3029 selectLoadStoreUIOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize: MemSizeInBits);
3030 if (NewOpc == I.getOpcode())
3031 return nullptr;
3032 // Check if we can fold anything into the addressing mode.
3033 auto AddrModeFns =
3034 selectAddrModeIndexed(Root&: I.getOperand(i: 1), Size: MemSizeInBytes);
3035 if (!AddrModeFns) {
3036 // Can't fold anything. Use the original instruction.
3037 I.setDesc(TII.get(Opcode: NewOpc));
3038 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
3039 return &I;
3040 }
3041
3042 // Folded something. Create a new instruction and return it.
3043 auto NewInst = MIB.buildInstr(Opc: NewOpc, DstOps: {}, SrcOps: {}, Flags: I.getFlags());
3044 Register CurValReg = I.getOperand(i: 0).getReg();
3045 IsStore ? NewInst.addUse(RegNo: CurValReg) : NewInst.addDef(RegNo: CurValReg);
3046 NewInst.cloneMemRefs(OtherMI: I);
3047 for (auto &Fn : *AddrModeFns)
3048 Fn(NewInst);
3049 I.eraseFromParent();
3050 return &*NewInst;
3051 };
3052
3053 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
3054 if (!LoadStore)
3055 return false;
3056
3057 // If we're storing a 0, use WZR/XZR.
3058 if (Opcode == TargetOpcode::G_STORE) {
3059 auto CVal = getIConstantVRegValWithLookThrough(
3060 VReg: LoadStore->getOperand(i: 0).getReg(), MRI);
3061 if (CVal && CVal->Value == 0) {
3062 switch (LoadStore->getOpcode()) {
3063 case AArch64::STRWui:
3064 case AArch64::STRHHui:
3065 case AArch64::STRBBui:
3066 LoadStore->getOperand(i: 0).setReg(AArch64::WZR);
3067 break;
3068 case AArch64::STRXui:
3069 LoadStore->getOperand(i: 0).setReg(AArch64::XZR);
3070 break;
3071 }
3072 }
3073 }
3074
3075 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3076 ValTy == LLT::scalar(SizeInBits: 64) && MemSizeInBits == 32)) {
3077 // The any/zextload from a smaller type to i32 should be handled by the
3078 // importer.
3079 if (MRI.getType(Reg: LoadStore->getOperand(i: 0).getReg()).getSizeInBits() != 64)
3080 return false;
3081 // If we have an extending load then change the load's type to be a
3082 // narrower reg and zero_extend with SUBREG_TO_REG.
3083 Register LdReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3084 Register DstReg = LoadStore->getOperand(i: 0).getReg();
3085 LoadStore->getOperand(i: 0).setReg(LdReg);
3086
3087 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LoadStore->getIterator()));
3088 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DstReg}, SrcOps: {})
3089 .addUse(RegNo: LdReg)
3090 .addImm(Val: AArch64::sub_32);
3091 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3092 return RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64allRegClass,
3093 MRI);
3094 }
3095 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3096 return true;
3097 }
3098
3099 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3100 case TargetOpcode::G_INDEXED_SEXTLOAD:
3101 return selectIndexedExtLoad(I, MRI);
3102 case TargetOpcode::G_INDEXED_LOAD:
3103 return selectIndexedLoad(I, MRI);
3104 case TargetOpcode::G_INDEXED_STORE:
3105 return selectIndexedStore(I&: cast<GIndexedStore>(Val&: I), MRI);
3106
3107 case TargetOpcode::G_LSHR:
3108 case TargetOpcode::G_ASHR:
3109 if (MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3110 return selectVectorAshrLshr(I, MRI);
3111 [[fallthrough]];
3112 case TargetOpcode::G_SHL: {
3113 if (Opcode == TargetOpcode::G_SHL &&
3114 MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3115 return selectVectorSHL(I, MRI);
3116
3117 // These shifts were legalized to have 64 bit shift amounts because we
3118 // want to take advantage of the selection patterns that assume the
3119 // immediates are s64s, however, selectBinaryOp will assume both operands
3120 // will have the same bit size.
3121 {
3122 Register SrcReg = I.getOperand(i: 1).getReg();
3123 Register ShiftReg = I.getOperand(i: 2).getReg();
3124 const LLT ShiftTy = MRI.getType(Reg: ShiftReg);
3125 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3126 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
3127 ShiftTy.getSizeInBits() == 64) {
3128 assert(!ShiftTy.isVector() && "unexpected vector shift ty");
3129 // Insert a subregister copy to implement a 64->32 trunc
3130 auto Trunc = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {SrcTy}, SrcOps: {})
3131 .addReg(RegNo: ShiftReg, Flags: {}, SubReg: AArch64::sub_32);
3132 MRI.setRegBank(Reg: Trunc.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
3133 I.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
3134 }
3135 }
3136
3137 const unsigned OpSize = Ty.getSizeInBits();
3138 const Register DefReg = I.getOperand(i: 0).getReg();
3139 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
3140
3141 const unsigned NewOpc = selectBinaryOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize);
3142 if (NewOpc == I.getOpcode())
3143 return false;
3144
3145 I.setDesc(TII.get(Opcode: NewOpc));
3146 // FIXME: Should the type be always reset in setDesc?
3147
3148 // Now that we selected an opcode, we need to constrain the register
3149 // operands to use appropriate classes.
3150 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3151 return true;
3152 }
3153 case TargetOpcode::G_PTR_ADD: {
3154 emitADD(DefReg: I.getOperand(i: 0).getReg(), LHS&: I.getOperand(i: 1), RHS&: I.getOperand(i: 2), MIRBuilder&: MIB);
3155 I.eraseFromParent();
3156 return true;
3157 }
3158
3159 case TargetOpcode::G_SADDE:
3160 case TargetOpcode::G_UADDE:
3161 case TargetOpcode::G_SSUBE:
3162 case TargetOpcode::G_USUBE:
3163 case TargetOpcode::G_SADDO:
3164 case TargetOpcode::G_UADDO:
3165 case TargetOpcode::G_SSUBO:
3166 case TargetOpcode::G_USUBO:
3167 return selectOverflowOp(I, MRI);
3168
3169 case TargetOpcode::G_PTRMASK: {
3170 Register MaskReg = I.getOperand(i: 2).getReg();
3171 std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(VReg: MaskReg, MRI);
3172 // TODO: Implement arbitrary cases
3173 if (!MaskVal || !isShiftedMask_64(Value: *MaskVal))
3174 return false;
3175
3176 uint64_t Mask = *MaskVal;
3177 I.setDesc(TII.get(Opcode: AArch64::ANDXri));
3178 I.getOperand(i: 2).ChangeToImmediate(
3179 ImmVal: AArch64_AM::encodeLogicalImmediate(imm: Mask, regSize: 64));
3180
3181 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3182 return true;
3183 }
3184 case TargetOpcode::G_PTRTOINT:
3185 case TargetOpcode::G_TRUNC: {
3186 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3187 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3188
3189 const Register DstReg = I.getOperand(i: 0).getReg();
3190 const Register SrcReg = I.getOperand(i: 1).getReg();
3191
3192 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3193 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3194
3195 if (DstRB.getID() != SrcRB.getID()) {
3196 LLVM_DEBUG(
3197 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
3198 return false;
3199 }
3200
3201 if (DstRB.getID() == AArch64::GPRRegBankID) {
3202 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3203 if (!DstRC)
3204 return false;
3205
3206 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(Ty: SrcTy, RB: SrcRB);
3207 if (!SrcRC)
3208 return false;
3209
3210 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
3211 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
3212 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
3213 return false;
3214 }
3215
3216 if (DstRC == SrcRC) {
3217 // Nothing to be done
3218 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(SizeInBits: 32) &&
3219 SrcTy == LLT::scalar(SizeInBits: 64)) {
3220 llvm_unreachable("TableGen can import this case");
3221 return false;
3222 } else if (DstRC == &AArch64::GPR32RegClass &&
3223 SrcRC == &AArch64::GPR64RegClass) {
3224 I.getOperand(i: 1).setSubReg(AArch64::sub_32);
3225 } else {
3226 LLVM_DEBUG(
3227 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3228 return false;
3229 }
3230
3231 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3232 return true;
3233 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
3234 if (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
3235 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
3236 I.setDesc(TII.get(Opcode: AArch64::XTNv4i16));
3237 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3238 return true;
3239 }
3240
3241 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
3242 MachineInstr *Extract = emitExtractVectorElt(
3243 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: DstTy.getSizeInBits()), VecReg: SrcReg, LaneIdx: 0, MIRBuilder&: MIB);
3244 if (!Extract)
3245 return false;
3246 I.eraseFromParent();
3247 return true;
3248 }
3249
3250 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
3251 if (Opcode == TargetOpcode::G_PTRTOINT) {
3252 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector");
3253 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3254 return selectCopy(I, TII, MRI, TRI, RBI);
3255 }
3256 }
3257
3258 return false;
3259 }
3260
3261 case TargetOpcode::G_ANYEXT: {
3262 if (selectUSMovFromExtend(I, MRI))
3263 return true;
3264
3265 const Register DstReg = I.getOperand(i: 0).getReg();
3266 const Register SrcReg = I.getOperand(i: 1).getReg();
3267
3268 const RegisterBank &RBDst = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3269 if (RBDst.getID() != AArch64::GPRRegBankID) {
3270 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
3271 << ", expected: GPR\n");
3272 return false;
3273 }
3274
3275 const RegisterBank &RBSrc = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3276 if (RBSrc.getID() != AArch64::GPRRegBankID) {
3277 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
3278 << ", expected: GPR\n");
3279 return false;
3280 }
3281
3282 const unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
3283
3284 if (DstSize == 0) {
3285 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
3286 return false;
3287 }
3288
3289 if (DstSize != 64 && DstSize > 32) {
3290 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
3291 << ", expected: 32 or 64\n");
3292 return false;
3293 }
3294 // At this point G_ANYEXT is just like a plain COPY, but we need
3295 // to explicitly form the 64-bit value if any.
3296 if (DstSize > 32) {
3297 Register ExtSrc = MRI.createVirtualRegister(RegClass: &AArch64::GPR64allRegClass);
3298 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
3299 .addDef(RegNo: ExtSrc)
3300 .addUse(RegNo: SrcReg)
3301 .addImm(Val: AArch64::sub_32);
3302 I.getOperand(i: 1).setReg(ExtSrc);
3303 }
3304 return selectCopy(I, TII, MRI, TRI, RBI);
3305 }
3306
3307 case TargetOpcode::G_ZEXT:
3308 case TargetOpcode::G_SEXT_INREG:
3309 case TargetOpcode::G_SEXT: {
3310 if (selectUSMovFromExtend(I, MRI))
3311 return true;
3312
3313 unsigned Opcode = I.getOpcode();
3314 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3315 const Register DefReg = I.getOperand(i: 0).getReg();
3316 Register SrcReg = I.getOperand(i: 1).getReg();
3317 const LLT DstTy = MRI.getType(Reg: DefReg);
3318 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3319 unsigned DstSize = DstTy.getSizeInBits();
3320 unsigned SrcSize = SrcTy.getSizeInBits();
3321
3322 // SEXT_INREG has the same src reg size as dst, the size of the value to be
3323 // extended is encoded in the imm.
3324 if (Opcode == TargetOpcode::G_SEXT_INREG)
3325 SrcSize = I.getOperand(i: 2).getImm();
3326
3327 if (DstTy.isVector())
3328 return false; // Should be handled by imported patterns.
3329
3330 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==
3331 AArch64::GPRRegBankID &&
3332 "Unexpected ext regbank");
3333
3334 MachineInstr *ExtI;
3335
3336 // First check if we're extending the result of a load which has a dest type
3337 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
3338 // GPR register on AArch64 and all loads which are smaller automatically
3339 // zero-extend the upper bits. E.g.
3340 // %v(s8) = G_LOAD %p, :: (load 1)
3341 // %v2(s32) = G_ZEXT %v(s8)
3342 if (!IsSigned) {
3343 auto *LoadMI = getOpcodeDef(Opcode: TargetOpcode::G_LOAD, Reg: SrcReg, MRI);
3344 bool IsGPR =
3345 RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
3346 if (LoadMI && IsGPR) {
3347 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3348 unsigned BytesLoaded = MemOp->getSize().getValue();
3349 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3350 return selectCopy(I, TII, MRI, TRI, RBI);
3351 }
3352
3353 // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
3354 // + SUBREG_TO_REG.
3355 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3356 Register SubregToRegSrc =
3357 MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3358 const Register ZReg = AArch64::WZR;
3359 MIB.buildInstr(Opc: AArch64::ORRWrs, DstOps: {SubregToRegSrc}, SrcOps: {ZReg, SrcReg})
3360 .addImm(Val: 0);
3361
3362 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
3363 .addUse(RegNo: SubregToRegSrc)
3364 .addImm(Val: AArch64::sub_32);
3365
3366 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass,
3367 MRI)) {
3368 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n");
3369 return false;
3370 }
3371
3372 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3373 MRI)) {
3374 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n");
3375 return false;
3376 }
3377
3378 I.eraseFromParent();
3379 return true;
3380 }
3381 }
3382
3383 if (DstSize == 64) {
3384 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3385 // FIXME: Can we avoid manually doing this?
3386 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3387 MRI)) {
3388 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
3389 << " operand\n");
3390 return false;
3391 }
3392 SrcReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG,
3393 DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
3394 .addUse(RegNo: SrcReg)
3395 .addImm(Val: AArch64::sub_32)
3396 .getReg(Idx: 0);
3397 }
3398
3399 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3400 DstOps: {DefReg}, SrcOps: {SrcReg})
3401 .addImm(Val: 0)
3402 .addImm(Val: SrcSize - 1);
3403 } else if (DstSize <= 32) {
3404 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3405 DstOps: {DefReg}, SrcOps: {SrcReg})
3406 .addImm(Val: 0)
3407 .addImm(Val: SrcSize - 1);
3408 } else {
3409 return false;
3410 }
3411
3412 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
3413 I.eraseFromParent();
3414 return true;
3415 }
3416
3417 case TargetOpcode::G_FREEZE:
3418 return selectCopy(I, TII, MRI, TRI, RBI);
3419
3420 case TargetOpcode::G_INTTOPTR:
3421 // The importer is currently unable to import pointer types since they
3422 // didn't exist in SelectionDAG.
3423 return selectCopy(I, TII, MRI, TRI, RBI);
3424
3425 case TargetOpcode::G_BITCAST:
3426 // Imported SelectionDAG rules can handle every bitcast except those that
3427 // bitcast from a type to the same type. Ideally, these shouldn't occur
3428 // but we might not run an optimizer that deletes them. The other exception
3429 // is bitcasts involving pointer types, as SelectionDAG has no knowledge
3430 // of them.
3431 return selectCopy(I, TII, MRI, TRI, RBI);
3432
3433 case TargetOpcode::G_SELECT: {
3434 auto &Sel = cast<GSelect>(Val&: I);
3435 const Register CondReg = Sel.getCondReg();
3436 const Register TReg = Sel.getTrueReg();
3437 const Register FReg = Sel.getFalseReg();
3438
3439 if (tryOptSelect(Sel))
3440 return true;
3441
3442 // Make sure to use an unused vreg instead of wzr, so that the peephole
3443 // optimizations will be able to optimize these.
3444 Register DeadVReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3445 auto TstMI = MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {DeadVReg}, SrcOps: {CondReg})
3446 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 1, regSize: 32));
3447 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
3448 if (!emitSelect(Dst: Sel.getReg(Idx: 0), True: TReg, False: FReg, CC: AArch64CC::NE, MIB))
3449 return false;
3450 Sel.eraseFromParent();
3451 return true;
3452 }
3453 case TargetOpcode::G_ICMP: {
3454 if (Ty.isVector())
3455 return false;
3456
3457 if (Ty != LLT::scalar(SizeInBits: 32)) {
3458 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
3459 << ", expected: " << LLT::scalar(32) << '\n');
3460 return false;
3461 }
3462
3463 auto &PredOp = I.getOperand(i: 1);
3464 emitIntegerCompare(LHS&: I.getOperand(i: 2), RHS&: I.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
3465 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
3466 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
3467 P: CmpInst::getInversePredicate(pred: Pred), RHS: I.getOperand(i: 3).getReg(), MRI: &MRI);
3468 emitCSINC(/*Dst=*/I.getOperand(i: 0).getReg(), /*Src1=*/AArch64::WZR,
3469 /*Src2=*/AArch64::WZR, Pred: InvCC, MIRBuilder&: MIB);
3470 I.eraseFromParent();
3471 return true;
3472 }
3473
3474 case TargetOpcode::G_FCMP: {
3475 CmpInst::Predicate Pred =
3476 static_cast<CmpInst::Predicate>(I.getOperand(i: 1).getPredicate());
3477 if (!emitFPCompare(LHS: I.getOperand(i: 2).getReg(), RHS: I.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
3478 Pred) ||
3479 !emitCSetForFCmp(Dst: I.getOperand(i: 0).getReg(), Pred, MIRBuilder&: MIB))
3480 return false;
3481 I.eraseFromParent();
3482 return true;
3483 }
3484 case TargetOpcode::G_VASTART:
3485 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3486 : selectVaStartAAPCS(I, MF, MRI);
3487 case TargetOpcode::G_INTRINSIC:
3488 return selectIntrinsic(I, MRI);
3489 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3490 return selectIntrinsicWithSideEffects(I, MRI);
3491 case TargetOpcode::G_IMPLICIT_DEF: {
3492 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
3493 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3494 const Register DstReg = I.getOperand(i: 0).getReg();
3495 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3496 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3497 RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
3498 return true;
3499 }
3500 case TargetOpcode::G_BLOCK_ADDR: {
3501 Function *BAFn = I.getOperand(i: 1).getBlockAddress()->getFunction();
3502 if (std::optional<uint16_t> BADisc =
3503 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: *BAFn)) {
3504 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X16}, SrcOps: {});
3505 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
3506 MIB.buildInstr(Opcode: AArch64::MOVaddrPAC)
3507 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress())
3508 .addImm(Val: AArch64PACKey::IA)
3509 .addReg(/*AddrDisc=*/RegNo: AArch64::XZR)
3510 .addImm(Val: *BADisc)
3511 .constrainAllUses(TII, TRI, RBI);
3512 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X16));
3513 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
3514 RC: AArch64::GPR64RegClass, MRI);
3515 I.eraseFromParent();
3516 return true;
3517 }
3518 if (TM.getCodeModel() == CodeModel::Large && !TM.isPositionIndependent()) {
3519 materializeLargeCMVal(I, V: I.getOperand(i: 1).getBlockAddress(), OpFlags: 0);
3520 I.eraseFromParent();
3521 return true;
3522 } else {
3523 I.setDesc(TII.get(Opcode: AArch64::MOVaddrBA));
3524 auto MovMI = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVaddrBA),
3525 DestReg: I.getOperand(i: 0).getReg())
3526 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress(),
3527 /* Offset */ 0, TargetFlags: AArch64II::MO_PAGE)
3528 .addBlockAddress(
3529 BA: I.getOperand(i: 1).getBlockAddress(), /* Offset */ 0,
3530 TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3531 I.eraseFromParent();
3532 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3533 return true;
3534 }
3535 }
3536 case AArch64::G_DUP: {
3537 // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
3538 // imported patterns. Do it manually here. Avoiding generating s16 gpr is
3539 // difficult because at RBS we may end up pessimizing the fpr case if we
3540 // decided to add an anyextend to fix this. Manual selection is the most
3541 // robust solution for now.
3542 if (RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI)->getID() !=
3543 AArch64::GPRRegBankID)
3544 return false; // We expect the fpr regbank case to be imported.
3545 LLT VecTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3546 if (VecTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8))
3547 I.setDesc(TII.get(Opcode: AArch64::DUPv8i8gpr));
3548 else if (VecTy == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8))
3549 I.setDesc(TII.get(Opcode: AArch64::DUPv16i8gpr));
3550 else if (VecTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16))
3551 I.setDesc(TII.get(Opcode: AArch64::DUPv4i16gpr));
3552 else if (VecTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16))
3553 I.setDesc(TII.get(Opcode: AArch64::DUPv8i16gpr));
3554 else
3555 return false;
3556 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3557 return true;
3558 }
3559 case TargetOpcode::G_BUILD_VECTOR:
3560 return selectBuildVector(I, MRI);
3561 case TargetOpcode::G_MERGE_VALUES:
3562 return selectMergeValues(I, MRI);
3563 case TargetOpcode::G_UNMERGE_VALUES:
3564 return selectUnmergeValues(I, MRI);
3565 case TargetOpcode::G_SHUFFLE_VECTOR:
3566 return selectShuffleVector(I, MRI);
3567 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3568 return selectExtractElt(I, MRI);
3569 case TargetOpcode::G_CONCAT_VECTORS:
3570 return selectConcatVectors(I, MRI);
3571 case TargetOpcode::G_JUMP_TABLE:
3572 return selectJumpTable(I, MRI);
3573 case TargetOpcode::G_MEMCPY:
3574 case TargetOpcode::G_MEMCPY_INLINE:
3575 case TargetOpcode::G_MEMMOVE:
3576 case TargetOpcode::G_MEMSET:
3577 case TargetOpcode::G_MEMSET_INLINE:
3578 assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
3579 return selectMOPS(I, MRI);
3580 }
3581
3582 return false;
3583}
3584
3585bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
3586 MachineIRBuilderState OldMIBState = MIB.getState();
3587 bool Success = select(I);
3588 MIB.setState(OldMIBState);
3589 return Success;
3590}
3591
3592bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3593 MachineRegisterInfo &MRI) {
3594 unsigned Mopcode;
3595 switch (GI.getOpcode()) {
3596 case TargetOpcode::G_MEMCPY:
3597 case TargetOpcode::G_MEMCPY_INLINE:
3598 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3599 break;
3600 case TargetOpcode::G_MEMMOVE:
3601 Mopcode = AArch64::MOPSMemoryMovePseudo;
3602 break;
3603 case TargetOpcode::G_MEMSET:
3604 case TargetOpcode::G_MEMSET_INLINE:
3605 // For tagged memset see llvm.aarch64.mops.memset.tag
3606 Mopcode = AArch64::MOPSMemorySetPseudo;
3607 break;
3608 }
3609
3610 auto &DstPtr = GI.getOperand(i: 0);
3611 auto &SrcOrVal = GI.getOperand(i: 1);
3612 auto &Size = GI.getOperand(i: 2);
3613
3614 // Create copies of the registers that can be clobbered.
3615 const Register DstPtrCopy = MRI.cloneVirtualRegister(VReg: DstPtr.getReg());
3616 const Register SrcValCopy = MRI.cloneVirtualRegister(VReg: SrcOrVal.getReg());
3617 const Register SizeCopy = MRI.cloneVirtualRegister(VReg: Size.getReg());
3618
3619 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3620 const auto &SrcValRegClass =
3621 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3622
3623 // Constrain to specific registers
3624 RBI.constrainGenericRegister(Reg: DstPtrCopy, RC: AArch64::GPR64commonRegClass, MRI);
3625 RBI.constrainGenericRegister(Reg: SrcValCopy, RC: SrcValRegClass, MRI);
3626 RBI.constrainGenericRegister(Reg: SizeCopy, RC: AArch64::GPR64RegClass, MRI);
3627
3628 MIB.buildCopy(Res: DstPtrCopy, Op: DstPtr);
3629 MIB.buildCopy(Res: SrcValCopy, Op: SrcOrVal);
3630 MIB.buildCopy(Res: SizeCopy, Op: Size);
3631
3632 // New instruction uses the copied registers because it must update them.
3633 // The defs are not used since they don't exist in G_MEM*. They are still
3634 // tied.
3635 // Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
3636 Register DefDstPtr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3637 Register DefSize = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3638 if (IsSet) {
3639 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSize},
3640 SrcOps: {DstPtrCopy, SizeCopy, SrcValCopy});
3641 } else {
3642 Register DefSrcPtr = MRI.createVirtualRegister(RegClass: &SrcValRegClass);
3643 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSrcPtr, DefSize},
3644 SrcOps: {DstPtrCopy, SrcValCopy, SizeCopy});
3645 }
3646
3647 GI.eraseFromParent();
3648 return true;
3649}
3650
3651bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3652 MachineRegisterInfo &MRI) {
3653 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT");
3654 Register JTAddr = I.getOperand(i: 0).getReg();
3655 unsigned JTI = I.getOperand(i: 1).getIndex();
3656 Register Index = I.getOperand(i: 2).getReg();
3657
3658 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: nullptr);
3659
3660 // With aarch64-jump-table-hardening, we only expand the jump table dispatch
3661 // sequence later, to guarantee the integrity of the intermediate values.
3662 if (MF->getFunction().hasFnAttribute(Kind: "aarch64-jump-table-hardening")) {
3663 CodeModel::Model CM = TM.getCodeModel();
3664 if (STI.isTargetMachO()) {
3665 if (CM != CodeModel::Small && CM != CodeModel::Large)
3666 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3667 } else {
3668 // Note that COFF support would likely also need JUMP_TABLE_DEBUG_INFO.
3669 assert(STI.isTargetELF() &&
3670 "jump table hardening only supported on MachO/ELF");
3671 if (CM != CodeModel::Small)
3672 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3673 }
3674
3675 MIB.buildCopy(Res: {AArch64::X16}, Op: I.getOperand(i: 2).getReg());
3676 MIB.buildInstr(Opcode: AArch64::BR_JumpTable)
3677 .addJumpTableIndex(Idx: I.getOperand(i: 1).getIndex());
3678 I.eraseFromParent();
3679 return true;
3680 }
3681
3682 Register TargetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3683 Register ScratchReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
3684
3685 auto JumpTableInst = MIB.buildInstr(Opc: AArch64::JumpTableDest32,
3686 DstOps: {TargetReg, ScratchReg}, SrcOps: {JTAddr, Index})
3687 .addJumpTableIndex(Idx: JTI);
3688 // Save the jump table info.
3689 MIB.buildInstr(Opc: TargetOpcode::JUMP_TABLE_DEBUG_INFO, DstOps: {},
3690 SrcOps: {static_cast<int64_t>(JTI)});
3691 // Build the indirect branch.
3692 MIB.buildInstr(Opc: AArch64::BR, DstOps: {}, SrcOps: {TargetReg});
3693 I.eraseFromParent();
3694 constrainSelectedInstRegOperands(I&: *JumpTableInst, TII, TRI, RBI);
3695 return true;
3696}
3697
3698bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
3699 MachineRegisterInfo &MRI) {
3700 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table");
3701 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!");
3702
3703 Register DstReg = I.getOperand(i: 0).getReg();
3704 unsigned JTI = I.getOperand(i: 1).getIndex();
3705 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3706 auto MovMI =
3707 MIB.buildInstr(Opc: AArch64::MOVaddrJT, DstOps: {DstReg}, SrcOps: {})
3708 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_PAGE)
3709 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3710 I.eraseFromParent();
3711 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3712 return true;
3713}
3714
3715bool AArch64InstructionSelector::selectTLSGlobalValue(
3716 MachineInstr &I, MachineRegisterInfo &MRI) {
3717 if (!STI.isTargetMachO())
3718 return false;
3719 MachineFunction &MF = *I.getParent()->getParent();
3720 MF.getFrameInfo().setAdjustsStack(true);
3721
3722 const auto &GlobalOp = I.getOperand(i: 1);
3723 assert(GlobalOp.getOffset() == 0 &&
3724 "Shouldn't have an offset on TLS globals!");
3725 const GlobalValue &GV = *GlobalOp.getGlobal();
3726
3727 auto LoadGOT =
3728 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {&AArch64::GPR64commonRegClass}, SrcOps: {})
3729 .addGlobalAddress(GV: &GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3730
3731 auto Load = MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {&AArch64::GPR64commonRegClass},
3732 SrcOps: {LoadGOT.getReg(Idx: 0)})
3733 .addImm(Val: 0);
3734
3735 MIB.buildCopy(Res: Register(AArch64::X0), Op: LoadGOT.getReg(Idx: 0));
3736 // TLS calls preserve all registers except those that absolutely must be
3737 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3738 // silly).
3739 unsigned Opcode = getBLRCallOpcode(MF);
3740
3741 // With ptrauth-calls, the tlv access thunk pointer is authenticated (IA, 0).
3742 if (MF.getFunction().hasFnAttribute(Kind: "ptrauth-calls")) {
3743 assert(Opcode == AArch64::BLR);
3744 Opcode = AArch64::BLRAAZ;
3745 }
3746
3747 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {Load})
3748 .addUse(RegNo: AArch64::X0, Flags: RegState::Implicit)
3749 .addDef(RegNo: AArch64::X0, Flags: RegState::Implicit)
3750 .addRegMask(Mask: TRI.getTLSCallPreservedMask());
3751
3752 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X0));
3753 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: AArch64::GPR64RegClass,
3754 MRI);
3755 I.eraseFromParent();
3756 return true;
3757}
3758
3759MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3760 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3761 MachineIRBuilder &MIRBuilder) const {
3762 auto Undef = MIRBuilder.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstRC}, SrcOps: {});
3763
3764 auto BuildFn = [&](unsigned SubregIndex) {
3765 auto Ins =
3766 MIRBuilder
3767 .buildInstr(Opc: TargetOpcode::INSERT_SUBREG, DstOps: {DstRC}, SrcOps: {Undef, Scalar})
3768 .addImm(Val: SubregIndex);
3769 constrainSelectedInstRegOperands(I&: *Undef, TII, TRI, RBI);
3770 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
3771 return &*Ins;
3772 };
3773
3774 switch (EltSize) {
3775 case 8:
3776 return BuildFn(AArch64::bsub);
3777 case 16:
3778 return BuildFn(AArch64::hsub);
3779 case 32:
3780 return BuildFn(AArch64::ssub);
3781 case 64:
3782 return BuildFn(AArch64::dsub);
3783 default:
3784 return nullptr;
3785 }
3786}
3787
3788MachineInstr *
3789AArch64InstructionSelector::emitNarrowVector(Register DstReg, Register SrcReg,
3790 MachineIRBuilder &MIB,
3791 MachineRegisterInfo &MRI) const {
3792 LLT DstTy = MRI.getType(Reg: DstReg);
3793 const TargetRegisterClass *RC =
3794 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
3795 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3796 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
3797 return nullptr;
3798 }
3799 unsigned SubReg = 0;
3800 if (!getSubRegForClass(RC, TRI, SubReg))
3801 return nullptr;
3802 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3803 LLVM_DEBUG(dbgs() << "Unsupported destination size! ("
3804 << DstTy.getSizeInBits() << "\n");
3805 return nullptr;
3806 }
3807 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
3808 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
3809 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
3810 return Copy;
3811}
3812
3813bool AArch64InstructionSelector::selectMergeValues(
3814 MachineInstr &I, MachineRegisterInfo &MRI) {
3815 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
3816 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3817 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3818 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
3819 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
3820
3821 if (I.getNumOperands() != 3)
3822 return false;
3823
3824 // Merging 2 s64s into an s128.
3825 if (DstTy == LLT::scalar(SizeInBits: 128)) {
3826 if (SrcTy.getSizeInBits() != 64)
3827 return false;
3828 Register DstReg = I.getOperand(i: 0).getReg();
3829 Register Src1Reg = I.getOperand(i: 1).getReg();
3830 Register Src2Reg = I.getOperand(i: 2).getReg();
3831 auto Tmp = MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstTy}, SrcOps: {});
3832 MachineInstr *InsMI = emitLaneInsert(DstReg: std::nullopt, SrcReg: Tmp.getReg(Idx: 0), EltReg: Src1Reg,
3833 /* LaneIdx */ 0, RB, MIRBuilder&: MIB);
3834 if (!InsMI)
3835 return false;
3836 MachineInstr *Ins2MI = emitLaneInsert(DstReg, SrcReg: InsMI->getOperand(i: 0).getReg(),
3837 EltReg: Src2Reg, /* LaneIdx */ 1, RB, MIRBuilder&: MIB);
3838 if (!Ins2MI)
3839 return false;
3840 constrainSelectedInstRegOperands(I&: *InsMI, TII, TRI, RBI);
3841 constrainSelectedInstRegOperands(I&: *Ins2MI, TII, TRI, RBI);
3842 I.eraseFromParent();
3843 return true;
3844 }
3845
3846 if (RB.getID() != AArch64::GPRRegBankID)
3847 return false;
3848
3849 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
3850 return false;
3851
3852 auto *DstRC = &AArch64::GPR64RegClass;
3853 Register SubToRegDef = MRI.createVirtualRegister(RegClass: DstRC);
3854 MachineInstr &SubRegMI = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3855 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
3856 .addDef(RegNo: SubToRegDef)
3857 .addUse(RegNo: I.getOperand(i: 1).getReg())
3858 .addImm(Val: AArch64::sub_32);
3859 Register SubToRegDef2 = MRI.createVirtualRegister(RegClass: DstRC);
3860 // Need to anyext the second scalar before we can use bfm
3861 MachineInstr &SubRegMI2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3862 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
3863 .addDef(RegNo: SubToRegDef2)
3864 .addUse(RegNo: I.getOperand(i: 2).getReg())
3865 .addImm(Val: AArch64::sub_32);
3866 MachineInstr &BFM =
3867 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::BFMXri))
3868 .addDef(RegNo: I.getOperand(i: 0).getReg())
3869 .addUse(RegNo: SubToRegDef)
3870 .addUse(RegNo: SubToRegDef2)
3871 .addImm(Val: 32)
3872 .addImm(Val: 31);
3873 constrainSelectedInstRegOperands(I&: SubRegMI, TII, TRI, RBI);
3874 constrainSelectedInstRegOperands(I&: SubRegMI2, TII, TRI, RBI);
3875 constrainSelectedInstRegOperands(I&: BFM, TII, TRI, RBI);
3876 I.eraseFromParent();
3877 return true;
3878}
3879
3880static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
3881 const unsigned EltSize) {
3882 // Choose a lane copy opcode and subregister based off of the size of the
3883 // vector's elements.
3884 switch (EltSize) {
3885 case 8:
3886 CopyOpc = AArch64::DUPi8;
3887 ExtractSubReg = AArch64::bsub;
3888 break;
3889 case 16:
3890 CopyOpc = AArch64::DUPi16;
3891 ExtractSubReg = AArch64::hsub;
3892 break;
3893 case 32:
3894 CopyOpc = AArch64::DUPi32;
3895 ExtractSubReg = AArch64::ssub;
3896 break;
3897 case 64:
3898 CopyOpc = AArch64::DUPi64;
3899 ExtractSubReg = AArch64::dsub;
3900 break;
3901 default:
3902 // Unknown size, bail out.
3903 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n");
3904 return false;
3905 }
3906 return true;
3907}
3908
3909MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
3910 std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
3911 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
3912 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3913 unsigned CopyOpc = 0;
3914 unsigned ExtractSubReg = 0;
3915 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: ScalarTy.getSizeInBits())) {
3916 LLVM_DEBUG(
3917 dbgs() << "Couldn't determine lane copy opcode for instruction.\n");
3918 return nullptr;
3919 }
3920
3921 const TargetRegisterClass *DstRC =
3922 getRegClassForTypeOnBank(Ty: ScalarTy, RB: DstRB, GetAllRegSet: true);
3923 if (!DstRC) {
3924 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n");
3925 return nullptr;
3926 }
3927
3928 const RegisterBank &VecRB = *RBI.getRegBank(Reg: VecReg, MRI, TRI);
3929 const LLT &VecTy = MRI.getType(Reg: VecReg);
3930 const TargetRegisterClass *VecRC =
3931 getRegClassForTypeOnBank(Ty: VecTy, RB: VecRB, GetAllRegSet: true);
3932 if (!VecRC) {
3933 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
3934 return nullptr;
3935 }
3936
3937 // The register that we're going to copy into.
3938 Register InsertReg = VecReg;
3939 if (!DstReg)
3940 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
3941 // If the lane index is 0, we just use a subregister COPY.
3942 if (LaneIdx == 0) {
3943 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {*DstReg}, SrcOps: {})
3944 .addReg(RegNo: VecReg, Flags: {}, SubReg: ExtractSubReg);
3945 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
3946 return &*Copy;
3947 }
3948
3949 // Lane copies require 128-bit wide registers. If we're dealing with an
3950 // unpacked vector, then we need to move up to that width. Insert an implicit
3951 // def and a subregister insert to get us there.
3952 if (VecTy.getSizeInBits() != 128) {
3953 MachineInstr *ScalarToVector = emitScalarToVector(
3954 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: VecReg, MIRBuilder);
3955 if (!ScalarToVector)
3956 return nullptr;
3957 InsertReg = ScalarToVector->getOperand(i: 0).getReg();
3958 }
3959
3960 MachineInstr *LaneCopyMI =
3961 MIRBuilder.buildInstr(Opc: CopyOpc, DstOps: {*DstReg}, SrcOps: {InsertReg}).addImm(Val: LaneIdx);
3962 constrainSelectedInstRegOperands(I&: *LaneCopyMI, TII, TRI, RBI);
3963
3964 // Make sure that we actually constrain the initial copy.
3965 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
3966 return LaneCopyMI;
3967}
3968
3969bool AArch64InstructionSelector::selectExtractElt(
3970 MachineInstr &I, MachineRegisterInfo &MRI) {
3971 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
3972 "unexpected opcode!");
3973 Register DstReg = I.getOperand(i: 0).getReg();
3974 const LLT NarrowTy = MRI.getType(Reg: DstReg);
3975 const Register SrcReg = I.getOperand(i: 1).getReg();
3976 const LLT WideTy = MRI.getType(Reg: SrcReg);
3977 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
3978 "source register size too small!");
3979 assert(!NarrowTy.isVector() && "cannot extract vector into vector!");
3980
3981 // Need the lane index to determine the correct copy opcode.
3982 MachineOperand &LaneIdxOp = I.getOperand(i: 2);
3983 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?");
3984
3985 // Find the index to extract from.
3986 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: LaneIdxOp.getReg(), MRI);
3987 if (!VRegAndVal)
3988 return false;
3989 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
3990
3991 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3992 if (DstRB.getID() == AArch64::GPRRegBankID) {
3993 unsigned Opcode;
3994 switch (WideTy.getScalarSizeInBits()) {
3995 case 8:
3996 Opcode = AArch64::UMOVvi8;
3997 break;
3998 case 16:
3999 Opcode = AArch64::UMOVvi16;
4000 break;
4001 case 32:
4002 Opcode = AArch64::UMOVvi32;
4003 break;
4004 default:
4005 return false;
4006 }
4007
4008 if (WideTy.getSizeInBits() != 128) {
4009 MachineInstr *ScalarToVector = emitScalarToVector(
4010 EltSize: WideTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: SrcReg, MIRBuilder&: MIB);
4011 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
4012 I.getOperand(i: 1).setReg(ScalarToVector->getOperand(i: 0).getReg());
4013 }
4014
4015 I.setDesc(TII.get(Opcode));
4016 I.getOperand(i: 2).ChangeToImmediate(ImmVal: LaneIdx);
4017 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
4018 return true;
4019 }
4020
4021 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg,
4022 LaneIdx, MIRBuilder&: MIB);
4023 if (!Extract)
4024 return false;
4025
4026 I.eraseFromParent();
4027 return true;
4028}
4029
4030bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4031 MachineInstr &I, MachineRegisterInfo &MRI) {
4032 unsigned NumElts = I.getNumOperands() - 1;
4033 Register SrcReg = I.getOperand(i: NumElts).getReg();
4034 const LLT NarrowTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
4035 const LLT SrcTy = MRI.getType(Reg: SrcReg);
4036
4037 assert(NarrowTy.isVector() && "Expected an unmerge into vectors");
4038 if (SrcTy.getSizeInBits() > 128) {
4039 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge");
4040 return false;
4041 }
4042
4043 // We implement a split vector operation by treating the sub-vectors as
4044 // scalars and extracting them.
4045 const RegisterBank &DstRB =
4046 *RBI.getRegBank(Reg: I.getOperand(i: 0).getReg(), MRI, TRI);
4047 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4048 Register Dst = I.getOperand(i: OpIdx).getReg();
4049 MachineInstr *Extract =
4050 emitExtractVectorElt(DstReg: Dst, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg, LaneIdx: OpIdx, MIRBuilder&: MIB);
4051 if (!Extract)
4052 return false;
4053 }
4054 I.eraseFromParent();
4055 return true;
4056}
4057
4058bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
4059 MachineRegisterInfo &MRI) {
4060 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4061 "unexpected opcode");
4062
4063 // The last operand is the vector source register, and every other operand is
4064 // a register to unpack into.
4065 unsigned NumElts = I.getNumOperands() - 1;
4066 Register SrcReg = I.getOperand(i: NumElts).getReg();
4067 Register LoReg = I.getOperand(i: 0).getReg();
4068 Register HiReg = I.getOperand(i: 1).getReg();
4069 const LLT NarrowTy = MRI.getType(Reg: LoReg);
4070 const LLT WideTy = MRI.getType(Reg: SrcReg);
4071 const RegisterBank &LoRB = *RBI.getRegBank(Reg: LoReg, MRI, TRI);
4072 const RegisterBank &HiRB = *RBI.getRegBank(Reg: HiReg, MRI, TRI);
4073 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
4074
4075 // Handle unmerging a 128-bit FPR value into two 64-bit GPR values.
4076 if (NarrowTy == LLT::scalar(SizeInBits: 64) && WideTy == LLT::scalar(SizeInBits: 128) &&
4077 LoRB.getID() == AArch64::GPRRegBankID &&
4078 HiRB.getID() == AArch64::GPRRegBankID &&
4079 SrcRB.getID() == AArch64::FPRRegBankID) {
4080 MachineInstr &Lo = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4081 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: LoReg)
4082 .addUse(RegNo: SrcReg)
4083 .addImm(Val: 0);
4084 MachineInstr &Hi = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4085 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: HiReg)
4086 .addUse(RegNo: SrcReg)
4087 .addImm(Val: 1);
4088 constrainSelectedInstRegOperands(I&: Lo, TII, TRI, RBI);
4089 constrainSelectedInstRegOperands(I&: Hi, TII, TRI, RBI);
4090 I.eraseFromParent();
4091 return true;
4092 }
4093
4094 // TODO: Handle other unmerges into GPRs and from scalars to scalars.
4095 if (LoRB.getID() != AArch64::FPRRegBankID ||
4096 HiRB.getID() != AArch64::FPRRegBankID) {
4097 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
4098 "currently unsupported.\n");
4099 return false;
4100 }
4101
4102 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
4103 "source register size too small!");
4104
4105 if (!NarrowTy.isScalar())
4106 return selectSplitVectorUnmerge(I, MRI);
4107
4108 // Choose a lane copy opcode and subregister based off of the size of the
4109 // vector's elements.
4110 unsigned CopyOpc = 0;
4111 unsigned ExtractSubReg = 0;
4112 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: NarrowTy.getSizeInBits()))
4113 return false;
4114
4115 // Set up for the lane copies.
4116 MachineBasicBlock &MBB = *I.getParent();
4117
4118 // Stores the registers we'll be copying from.
4119 SmallVector<Register, 4> InsertRegs;
4120
4121 // We'll use the first register twice, so we only need NumElts-1 registers.
4122 unsigned NumInsertRegs = NumElts - 1;
4123
4124 // If our elements fit into exactly 128 bits, then we can copy from the source
4125 // directly. Otherwise, we need to do a bit of setup with some subregister
4126 // inserts.
4127 if (NarrowTy.getSizeInBits() * NumElts == 128) {
4128 InsertRegs.assign(NumElts: NumInsertRegs, Elt: SrcReg);
4129 } else {
4130 // No. We have to perform subregister inserts. For each insert, create an
4131 // implicit def and a subregister insert, and save the register we create.
4132 // For scalar sources, treat as a pseudo-vector of NarrowTy elements.
4133 unsigned EltSize = WideTy.isVector() ? WideTy.getScalarSizeInBits()
4134 : NarrowTy.getSizeInBits();
4135 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
4136 Ty: LLT::fixed_vector(NumElements: NumElts, ScalarSizeInBits: EltSize), RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
4137 unsigned SubReg = 0;
4138 bool Found = getSubRegForClass(RC, TRI, SubReg);
4139 (void)Found;
4140 assert(Found && "expected to find last operand's subeg idx");
4141 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
4142 Register ImpDefReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4143 MachineInstr &ImpDefMI =
4144 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF),
4145 DestReg: ImpDefReg);
4146
4147 // Now, create the subregister insert from SrcReg.
4148 Register InsertReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4149 MachineInstr &InsMI =
4150 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(),
4151 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: InsertReg)
4152 .addUse(RegNo: ImpDefReg)
4153 .addUse(RegNo: SrcReg)
4154 .addImm(Val: SubReg);
4155
4156 constrainSelectedInstRegOperands(I&: ImpDefMI, TII, TRI, RBI);
4157 constrainSelectedInstRegOperands(I&: InsMI, TII, TRI, RBI);
4158
4159 // Save the register so that we can copy from it after.
4160 InsertRegs.push_back(Elt: InsertReg);
4161 }
4162 }
4163
4164 // Now that we've created any necessary subregister inserts, we can
4165 // create the copies.
4166 //
4167 // Perform the first copy separately as a subregister copy.
4168 Register CopyTo = I.getOperand(i: 0).getReg();
4169 auto FirstCopy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {CopyTo}, SrcOps: {})
4170 .addReg(RegNo: InsertRegs[0], Flags: {}, SubReg: ExtractSubReg);
4171 constrainSelectedInstRegOperands(I&: *FirstCopy, TII, TRI, RBI);
4172
4173 // Now, perform the remaining copies as vector lane copies.
4174 unsigned LaneIdx = 1;
4175 for (Register InsReg : InsertRegs) {
4176 Register CopyTo = I.getOperand(i: LaneIdx).getReg();
4177 MachineInstr &CopyInst =
4178 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CopyOpc), DestReg: CopyTo)
4179 .addUse(RegNo: InsReg)
4180 .addImm(Val: LaneIdx);
4181 constrainSelectedInstRegOperands(I&: CopyInst, TII, TRI, RBI);
4182 ++LaneIdx;
4183 }
4184
4185 // Separately constrain the first copy's destination. Because of the
4186 // limitation in constrainOperandRegClass, we can't guarantee that this will
4187 // actually be constrained. So, do it ourselves using the second operand.
4188 const TargetRegisterClass *RC =
4189 MRI.getRegClassOrNull(Reg: I.getOperand(i: 1).getReg());
4190 if (!RC) {
4191 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
4192 return false;
4193 }
4194
4195 RBI.constrainGenericRegister(Reg: CopyTo, RC: *RC, MRI);
4196 I.eraseFromParent();
4197 return true;
4198}
4199
4200bool AArch64InstructionSelector::selectConcatVectors(
4201 MachineInstr &I, MachineRegisterInfo &MRI) {
4202 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4203 "Unexpected opcode");
4204 Register Dst = I.getOperand(i: 0).getReg();
4205 Register Op1 = I.getOperand(i: 1).getReg();
4206 Register Op2 = I.getOperand(i: 2).getReg();
4207 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder&: MIB);
4208 if (!ConcatMI)
4209 return false;
4210 I.eraseFromParent();
4211 return true;
4212}
4213
4214unsigned
4215AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4216 MachineFunction &MF) const {
4217 Type *CPTy = CPVal->getType();
4218 Align Alignment = MF.getDataLayout().getPrefTypeAlign(Ty: CPTy);
4219
4220 MachineConstantPool *MCP = MF.getConstantPool();
4221 return MCP->getConstantPoolIndex(C: CPVal, Alignment);
4222}
4223
4224MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4225 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4226 const TargetRegisterClass *RC;
4227 unsigned Opc;
4228 bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
4229 unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(Ty: CPVal->getType());
4230 switch (Size) {
4231 case 16:
4232 RC = &AArch64::FPR128RegClass;
4233 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4234 break;
4235 case 8:
4236 RC = &AArch64::FPR64RegClass;
4237 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4238 break;
4239 case 4:
4240 RC = &AArch64::FPR32RegClass;
4241 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4242 break;
4243 case 2:
4244 RC = &AArch64::FPR16RegClass;
4245 Opc = AArch64::LDRHui;
4246 break;
4247 default:
4248 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "
4249 << *CPVal->getType());
4250 return nullptr;
4251 }
4252
4253 MachineInstr *LoadMI = nullptr;
4254 auto &MF = MIRBuilder.getMF();
4255 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4256 if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
4257 // Use load(literal) for tiny code model.
4258 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {}).addConstantPoolIndex(Idx: CPIdx);
4259 } else {
4260 auto Adrp =
4261 MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
4262 .addConstantPoolIndex(Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
4263
4264 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {Adrp})
4265 .addConstantPoolIndex(
4266 Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4267
4268 constrainSelectedInstRegOperands(I&: *Adrp, TII, TRI, RBI);
4269 }
4270
4271 MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
4272 LoadMI->addMemOperand(MF, MO: MF.getMachineMemOperand(PtrInfo,
4273 F: MachineMemOperand::MOLoad,
4274 Size, BaseAlignment: Align(Size)));
4275 constrainSelectedInstRegOperands(I&: *LoadMI, TII, TRI, RBI);
4276 return LoadMI;
4277}
4278
4279/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4280/// size and RB.
4281static std::pair<unsigned, unsigned>
4282getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4283 unsigned Opc, SubregIdx;
4284 if (RB.getID() == AArch64::GPRRegBankID) {
4285 if (EltSize == 8) {
4286 Opc = AArch64::INSvi8gpr;
4287 SubregIdx = AArch64::bsub;
4288 } else if (EltSize == 16) {
4289 Opc = AArch64::INSvi16gpr;
4290 SubregIdx = AArch64::ssub;
4291 } else if (EltSize == 32) {
4292 Opc = AArch64::INSvi32gpr;
4293 SubregIdx = AArch64::ssub;
4294 } else if (EltSize == 64) {
4295 Opc = AArch64::INSvi64gpr;
4296 SubregIdx = AArch64::dsub;
4297 } else {
4298 llvm_unreachable("invalid elt size!");
4299 }
4300 } else {
4301 if (EltSize == 8) {
4302 Opc = AArch64::INSvi8lane;
4303 SubregIdx = AArch64::bsub;
4304 } else if (EltSize == 16) {
4305 Opc = AArch64::INSvi16lane;
4306 SubregIdx = AArch64::hsub;
4307 } else if (EltSize == 32) {
4308 Opc = AArch64::INSvi32lane;
4309 SubregIdx = AArch64::ssub;
4310 } else if (EltSize == 64) {
4311 Opc = AArch64::INSvi64lane;
4312 SubregIdx = AArch64::dsub;
4313 } else {
4314 llvm_unreachable("invalid elt size!");
4315 }
4316 }
4317 return std::make_pair(x&: Opc, y&: SubregIdx);
4318}
4319
4320MachineInstr *AArch64InstructionSelector::emitInstr(
4321 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4322 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4323 const ComplexRendererFns &RenderFns) const {
4324 assert(Opcode && "Expected an opcode?");
4325 assert(!isPreISelGenericOpcode(Opcode) &&
4326 "Function should only be used to produce selected instructions!");
4327 auto MI = MIRBuilder.buildInstr(Opc: Opcode, DstOps, SrcOps);
4328 if (RenderFns)
4329 for (auto &Fn : *RenderFns)
4330 Fn(MI);
4331 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
4332 return &*MI;
4333}
4334
4335MachineInstr *AArch64InstructionSelector::emitAddSub(
4336 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4337 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4338 MachineIRBuilder &MIRBuilder) const {
4339 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4340 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4341 auto Ty = MRI.getType(Reg: LHS.getReg());
4342 assert(!Ty.isVector() && "Expected a scalar or pointer?");
4343 unsigned Size = Ty.getSizeInBits();
4344 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
4345 bool Is32Bit = Size == 32;
4346
4347 // INSTRri form with positive arithmetic immediate.
4348 if (auto Fns = selectArithImmed(Root&: RHS))
4349 return emitInstr(Opcode: AddrModeAndSizeToOpcode[0][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4350 MIRBuilder, RenderFns: Fns);
4351
4352 // INSTRri form with negative arithmetic immediate.
4353 if (auto Fns = selectNegArithImmed(Root&: RHS))
4354 return emitInstr(Opcode: AddrModeAndSizeToOpcode[3][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4355 MIRBuilder, RenderFns: Fns);
4356
4357 // INSTRrx form.
4358 if (auto Fns = selectArithExtendedRegister(Root&: RHS))
4359 return emitInstr(Opcode: AddrModeAndSizeToOpcode[4][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4360 MIRBuilder, RenderFns: Fns);
4361
4362 // INSTRrs form.
4363 if (auto Fns = selectShiftedRegister(Root&: RHS))
4364 return emitInstr(Opcode: AddrModeAndSizeToOpcode[1][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4365 MIRBuilder, RenderFns: Fns);
4366 return emitInstr(Opcode: AddrModeAndSizeToOpcode[2][Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS},
4367 MIRBuilder);
4368}
4369
4370MachineInstr *
4371AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4372 MachineOperand &RHS,
4373 MachineIRBuilder &MIRBuilder) const {
4374 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4375 ._M_elems: {{AArch64::ADDXri, AArch64::ADDWri},
4376 {AArch64::ADDXrs, AArch64::ADDWrs},
4377 {AArch64::ADDXrr, AArch64::ADDWrr},
4378 {AArch64::SUBXri, AArch64::SUBWri},
4379 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4380 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst: DefReg, LHS, RHS, MIRBuilder);
4381}
4382
4383MachineInstr *
4384AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4385 MachineOperand &RHS,
4386 MachineIRBuilder &MIRBuilder) const {
4387 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4388 ._M_elems: {{AArch64::ADDSXri, AArch64::ADDSWri},
4389 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4390 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4391 {AArch64::SUBSXri, AArch64::SUBSWri},
4392 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4393 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4394}
4395
4396MachineInstr *
4397AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4398 MachineOperand &RHS,
4399 MachineIRBuilder &MIRBuilder) const {
4400 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4401 ._M_elems: {{AArch64::SUBSXri, AArch64::SUBSWri},
4402 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4403 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4404 {AArch64::ADDSXri, AArch64::ADDSWri},
4405 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4406 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4407}
4408
4409MachineInstr *
4410AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
4411 MachineOperand &RHS,
4412 MachineIRBuilder &MIRBuilder) const {
4413 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4414 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4415 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4416 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4417 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4418}
4419
4420MachineInstr *
4421AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
4422 MachineOperand &RHS,
4423 MachineIRBuilder &MIRBuilder) const {
4424 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4425 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4426 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4427 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4428 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4429}
4430
4431MachineInstr *
4432AArch64InstructionSelector::emitCMP(MachineOperand &LHS, MachineOperand &RHS,
4433 MachineIRBuilder &MIRBuilder) const {
4434 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4435 bool Is32Bit = MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32;
4436 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4437 return emitSUBS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4438}
4439
4440MachineInstr *
4441AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4442 MachineIRBuilder &MIRBuilder) const {
4443 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4444 bool Is32Bit = (MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4445 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4446 return emitADDS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4447}
4448
4449MachineInstr *
4450AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4451 MachineIRBuilder &MIRBuilder) const {
4452 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4453 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4454 LLT Ty = MRI.getType(Reg: LHS.getReg());
4455 unsigned RegSize = Ty.getSizeInBits();
4456 bool Is32Bit = (RegSize == 32);
4457 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4458 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4459 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4460 // ANDS needs a logical immediate for its immediate form. Check if we can
4461 // fold one in.
4462 if (auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI)) {
4463 int64_t Imm = ValAndVReg->Value.getSExtValue();
4464
4465 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize)) {
4466 auto TstMI = MIRBuilder.buildInstr(Opc: OpcTable[0][Is32Bit], DstOps: {Ty}, SrcOps: {LHS});
4467 TstMI.addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: Imm, regSize: RegSize));
4468 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
4469 return &*TstMI;
4470 }
4471 }
4472
4473 if (auto Fns = selectLogicalShiftedRegister(Root&: RHS))
4474 return emitInstr(Opcode: OpcTable[1][Is32Bit], DstOps: {Ty}, SrcOps: {LHS}, MIRBuilder, RenderFns: Fns);
4475 return emitInstr(Opcode: OpcTable[2][Is32Bit], DstOps: {Ty}, SrcOps: {LHS, RHS}, MIRBuilder);
4476}
4477
4478MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4479 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4480 MachineIRBuilder &MIRBuilder) const {
4481 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!");
4482 assert(Predicate.isPredicate() && "Expected predicate?");
4483 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4484 LLT CmpTy = MRI.getType(Reg: LHS.getReg());
4485 assert(!CmpTy.isVector() && "Expected scalar or pointer");
4486 unsigned Size = CmpTy.getSizeInBits();
4487 (void)Size;
4488 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?");
4489 // Fold the compare into a cmn or tst if possible.
4490 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4491 return FoldCmp;
4492 return emitCMP(LHS, RHS, MIRBuilder);
4493}
4494
4495MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4496 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4497 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4498#ifndef NDEBUG
4499 LLT Ty = MRI.getType(Dst);
4500 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&
4501 "Expected a 32-bit scalar register?");
4502#endif
4503 const Register ZReg = AArch64::WZR;
4504 AArch64CC::CondCode CC1, CC2;
4505 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
4506 auto InvCC1 = AArch64CC::getInvertedCondCode(Code: CC1);
4507 if (CC2 == AArch64CC::AL)
4508 return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1,
4509 MIRBuilder);
4510 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4511 Register Def1Reg = MRI.createVirtualRegister(RegClass: RC);
4512 Register Def2Reg = MRI.createVirtualRegister(RegClass: RC);
4513 auto InvCC2 = AArch64CC::getInvertedCondCode(Code: CC2);
4514 emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1, MIRBuilder);
4515 emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC2, MIRBuilder);
4516 auto OrMI = MIRBuilder.buildInstr(Opc: AArch64::ORRWrr, DstOps: {Dst}, SrcOps: {Def1Reg, Def2Reg});
4517 constrainSelectedInstRegOperands(I&: *OrMI, TII, TRI, RBI);
4518 return &*OrMI;
4519}
4520
4521MachineInstr *AArch64InstructionSelector::emitFPCompare(
4522 Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
4523 std::optional<CmpInst::Predicate> Pred) const {
4524 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4525 LLT Ty = MRI.getType(Reg: LHS);
4526 if (Ty.isVector())
4527 return nullptr;
4528 unsigned OpSize = Ty.getSizeInBits();
4529 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4530
4531 // If this is a compare against +0.0, then we don't have
4532 // to explicitly materialize a constant.
4533 bool ShouldUseImm = mi_match(R: RHS, MRI, P: m_PosZeroFP());
4534
4535 auto IsEqualityPred = [](CmpInst::Predicate P) {
4536 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4537 P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
4538 };
4539 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4540 // Try commuting the operands.
4541 if (mi_match(R: LHS, MRI, P: m_PosZeroFP())) {
4542 ShouldUseImm = true;
4543 std::swap(a&: LHS, b&: RHS);
4544 }
4545 }
4546 unsigned CmpOpcTbl[2][3] = {
4547 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4548 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4549 unsigned CmpOpc =
4550 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4551
4552 // Partially build the compare. Decide if we need to add a use for the
4553 // third operand based off whether or not we're comparing against 0.0.
4554 auto CmpMI = MIRBuilder.buildInstr(Opcode: CmpOpc).addUse(RegNo: LHS);
4555 CmpMI.setMIFlags(MachineInstr::NoFPExcept);
4556 if (!ShouldUseImm)
4557 CmpMI.addUse(RegNo: RHS);
4558 constrainSelectedInstRegOperands(I&: *CmpMI, TII, TRI, RBI);
4559 return &*CmpMI;
4560}
4561
4562MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4563 std::optional<Register> Dst, Register Op1, Register Op2,
4564 MachineIRBuilder &MIRBuilder) const {
4565 // We implement a vector concat by:
4566 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4567 // 2. Insert the upper vector into the destination's upper element
4568 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4569 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4570
4571 const LLT Op1Ty = MRI.getType(Reg: Op1);
4572 const LLT Op2Ty = MRI.getType(Reg: Op2);
4573
4574 if (Op1Ty != Op2Ty) {
4575 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys");
4576 return nullptr;
4577 }
4578 assert(Op1Ty.isVector() && "Expected a vector for vector concat");
4579
4580 if (Op1Ty.getSizeInBits() >= 128) {
4581 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors");
4582 return nullptr;
4583 }
4584
4585 // At the moment we just support 64 bit vector concats.
4586 if (Op1Ty.getSizeInBits() != 64) {
4587 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors");
4588 return nullptr;
4589 }
4590
4591 const LLT ScalarTy = LLT::scalar(SizeInBits: Op1Ty.getSizeInBits());
4592 const RegisterBank &FPRBank = *RBI.getRegBank(Reg: Op1, MRI, TRI);
4593 const TargetRegisterClass *DstRC =
4594 getRegClassForTypeOnBank(Ty: Op1Ty.multiplyElements(Factor: 2), RB: FPRBank);
4595
4596 MachineInstr *WidenedOp1 =
4597 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op1, MIRBuilder);
4598 MachineInstr *WidenedOp2 =
4599 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op2, MIRBuilder);
4600 if (!WidenedOp1 || !WidenedOp2) {
4601 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value");
4602 return nullptr;
4603 }
4604
4605 // Now do the insert of the upper element.
4606 unsigned InsertOpc, InsSubRegIdx;
4607 std::tie(args&: InsertOpc, args&: InsSubRegIdx) =
4608 getInsertVecEltOpInfo(RB: FPRBank, EltSize: ScalarTy.getSizeInBits());
4609
4610 if (!Dst)
4611 Dst = MRI.createVirtualRegister(RegClass: DstRC);
4612 auto InsElt =
4613 MIRBuilder
4614 .buildInstr(Opc: InsertOpc, DstOps: {*Dst}, SrcOps: {WidenedOp1->getOperand(i: 0).getReg()})
4615 .addImm(Val: 1) /* Lane index */
4616 .addUse(RegNo: WidenedOp2->getOperand(i: 0).getReg())
4617 .addImm(Val: 0);
4618 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
4619 return &*InsElt;
4620}
4621
4622MachineInstr *
4623AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
4624 Register Src2, AArch64CC::CondCode Pred,
4625 MachineIRBuilder &MIRBuilder) const {
4626 auto &MRI = *MIRBuilder.getMRI();
4627 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg: Dst);
4628 // If we used a register class, then this won't necessarily have an LLT.
4629 // Compute the size based off whether or not we have a class or bank.
4630 unsigned Size;
4631 if (const auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
4632 Size = TRI.getRegSizeInBits(RC: *RC);
4633 else
4634 Size = MRI.getType(Reg: Dst).getSizeInBits();
4635 // Some opcodes use s1.
4636 assert(Size <= 64 && "Expected 64 bits or less only!");
4637 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4638 unsigned Opc = OpcTable[Size == 64];
4639 auto CSINC = MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Src1, Src2}).addImm(Val: Pred);
4640 constrainSelectedInstRegOperands(I&: *CSINC, TII, TRI, RBI);
4641 return &*CSINC;
4642}
4643
4644MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
4645 Register CarryReg) {
4646 MachineRegisterInfo *MRI = MIB.getMRI();
4647 unsigned Opcode = I.getOpcode();
4648
4649 // If the instruction is a SUB, we need to negate the carry,
4650 // because borrowing is indicated by carry-flag == 0.
4651 bool NeedsNegatedCarry =
4652 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4653
4654 // If the previous instruction will already produce the correct carry, do not
4655 // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
4656 // generated during legalization of wide add/sub. This optimization depends on
4657 // these sequences not being interrupted by other instructions.
4658 // We have to select the previous instruction before the carry-using
4659 // instruction is deleted by the calling function, otherwise the previous
4660 // instruction might become dead and would get deleted.
4661 MachineInstr *SrcMI = MRI->getVRegDef(Reg: CarryReg);
4662 if (SrcMI == I.getPrevNode()) {
4663 if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(Val: SrcMI)) {
4664 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4665 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4666 CarrySrcMI->isUnsigned() &&
4667 CarrySrcMI->getCarryOutReg() == CarryReg &&
4668 selectAndRestoreState(I&: *SrcMI))
4669 return nullptr;
4670 }
4671 }
4672
4673 Register DeadReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4674
4675 if (NeedsNegatedCarry) {
4676 // (0 - Carry) sets !C in NZCV when Carry == 1
4677 Register ZReg = AArch64::WZR;
4678 return emitInstr(Opcode: AArch64::SUBSWrr, DstOps: {DeadReg}, SrcOps: {ZReg, CarryReg}, MIRBuilder&: MIB);
4679 }
4680
4681 // (Carry - 1) sets !C in NZCV when Carry == 0
4682 auto Fns = select12BitValueWithLeftShift(Immed: 1);
4683 return emitInstr(Opcode: AArch64::SUBSWri, DstOps: {DeadReg}, SrcOps: {CarryReg}, MIRBuilder&: MIB, RenderFns: Fns);
4684}
4685
4686bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &I,
4687 MachineRegisterInfo &MRI) {
4688 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
4689
4690 if (auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
4691 // Set NZCV carry according to carry-in VReg
4692 emitCarryIn(I, CarryReg: CarryInMI->getCarryInReg());
4693 }
4694
4695 // Emit the operation and get the correct condition code.
4696 auto OpAndCC = emitOverflowOp(Opcode: I.getOpcode(), Dst: CarryMI.getDstReg(),
4697 LHS&: CarryMI.getLHS(), RHS&: CarryMI.getRHS(), MIRBuilder&: MIB);
4698
4699 Register CarryOutReg = CarryMI.getCarryOutReg();
4700
4701 // Don't convert carry-out to VReg if it is never used
4702 if (!MRI.use_nodbg_empty(RegNo: CarryOutReg)) {
4703 // Now, put the overflow result in the register given by the first operand
4704 // to the overflow op. CSINC increments the result when the predicate is
4705 // false, so to get the increment when it's true, we need to use the
4706 // inverse. In this case, we want to increment when carry is set.
4707 Register ZReg = AArch64::WZR;
4708 emitCSINC(/*Dst=*/CarryOutReg, /*Src1=*/ZReg, /*Src2=*/ZReg,
4709 Pred: getInvertedCondCode(Code: OpAndCC.second), MIRBuilder&: MIB);
4710 }
4711
4712 I.eraseFromParent();
4713 return true;
4714}
4715
4716std::pair<MachineInstr *, AArch64CC::CondCode>
4717AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4718 MachineOperand &LHS,
4719 MachineOperand &RHS,
4720 MachineIRBuilder &MIRBuilder) const {
4721 switch (Opcode) {
4722 default:
4723 llvm_unreachable("Unexpected opcode!");
4724 case TargetOpcode::G_SADDO:
4725 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4726 case TargetOpcode::G_UADDO:
4727 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4728 case TargetOpcode::G_SSUBO:
4729 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4730 case TargetOpcode::G_USUBO:
4731 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4732 case TargetOpcode::G_SADDE:
4733 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4734 case TargetOpcode::G_UADDE:
4735 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4736 case TargetOpcode::G_SSUBE:
4737 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4738 case TargetOpcode::G_USUBE:
4739 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4740 }
4741}
4742
4743/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4744/// expressed as a conjunction.
4745/// \param CanNegate Set to true if we can negate the whole sub-tree just by
4746/// changing the conditions on the CMP tests.
4747/// (this means we can call emitConjunctionRec() with
4748/// Negate==true on this sub-tree)
4749/// \param MustBeFirst Set to true if this subtree needs to be negated and we
4750/// cannot do the negation naturally. We are required to
4751/// emit the subtree first in this case.
4752/// \param WillNegate Is true if are called when the result of this
4753/// subexpression must be negated. This happens when the
4754/// outer expression is an OR. We can use this fact to know
4755/// that we have a double negation (or (or ...) ...) that
4756/// can be implemented for free.
4757static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
4758 bool WillNegate, MachineRegisterInfo &MRI,
4759 unsigned Depth = 0) {
4760 if (!MRI.hasOneNonDBGUse(RegNo: Val))
4761 return false;
4762 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4763 unsigned Opcode = ValDef->getOpcode();
4764 if (isa<GAnyCmp>(Val: ValDef)) {
4765 CanNegate = true;
4766 MustBeFirst = false;
4767 return true;
4768 }
4769 // Protect against exponential runtime and stack overflow.
4770 if (Depth > 6)
4771 return false;
4772 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4773 bool IsOR = Opcode == TargetOpcode::G_OR;
4774 Register O0 = ValDef->getOperand(i: 1).getReg();
4775 Register O1 = ValDef->getOperand(i: 2).getReg();
4776 bool CanNegateL;
4777 bool MustBeFirstL;
4778 if (!canEmitConjunction(Val: O0, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI, Depth: Depth + 1))
4779 return false;
4780 bool CanNegateR;
4781 bool MustBeFirstR;
4782 if (!canEmitConjunction(Val: O1, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI, Depth: Depth + 1))
4783 return false;
4784
4785 if (MustBeFirstL && MustBeFirstR)
4786 return false;
4787
4788 if (IsOR) {
4789 // For an OR expression we need to be able to naturally negate at least
4790 // one side or we cannot do the transformation at all.
4791 if (!CanNegateL && !CanNegateR)
4792 return false;
4793 // If we the result of the OR will be negated and we can naturally negate
4794 // the leaves, then this sub-tree as a whole negates naturally.
4795 CanNegate = WillNegate && CanNegateL && CanNegateR;
4796 // If we cannot naturally negate the whole sub-tree, then this must be
4797 // emitted first.
4798 MustBeFirst = !CanNegate;
4799 } else {
4800 assert(Opcode == TargetOpcode::G_AND && "Must be G_AND");
4801 // We cannot naturally negate an AND operation.
4802 CanNegate = false;
4803 MustBeFirst = MustBeFirstL || MustBeFirstR;
4804 }
4805 return true;
4806 }
4807 return false;
4808}
4809
4810MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4811 Register LHS, Register RHS, CmpInst::Predicate CC,
4812 AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC,
4813 MachineIRBuilder &MIB) const {
4814 auto &MRI = *MIB.getMRI();
4815 LLT OpTy = MRI.getType(Reg: LHS);
4816 unsigned CCmpOpc;
4817 std::optional<ValueAndVReg> C;
4818 if (CmpInst::isIntPredicate(P: CC)) {
4819 assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64);
4820 C = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
4821 if (!C || C->Value.sgt(RHS: 31) || C->Value.slt(RHS: -31))
4822 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4823 else if (C->Value.ule(RHS: 31))
4824 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4825 else
4826 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4827 } else {
4828 assert(OpTy.getSizeInBits() == 16 || OpTy.getSizeInBits() == 32 ||
4829 OpTy.getSizeInBits() == 64);
4830 switch (OpTy.getSizeInBits()) {
4831 case 16:
4832 assert(STI.hasFullFP16() && "Expected Full FP16 for fp16 comparisons");
4833 CCmpOpc = AArch64::FCCMPHrr;
4834 break;
4835 case 32:
4836 CCmpOpc = AArch64::FCCMPSrr;
4837 break;
4838 case 64:
4839 CCmpOpc = AArch64::FCCMPDrr;
4840 break;
4841 default:
4842 return nullptr;
4843 }
4844 }
4845 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
4846 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(Code: InvOutCC);
4847 auto CCmp =
4848 MIB.buildInstr(Opc: CCmpOpc, DstOps: {}, SrcOps: {LHS});
4849 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
4850 CCmp.addImm(Val: C->Value.getZExtValue());
4851 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
4852 CCmp.addImm(Val: C->Value.abs().getZExtValue());
4853 else
4854 CCmp.addReg(RegNo: RHS);
4855 CCmp.addImm(Val: NZCV).addImm(Val: Predicate);
4856 constrainSelectedInstRegOperands(I&: *CCmp, TII, TRI, RBI);
4857 return &*CCmp;
4858}
4859
4860MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
4861 Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
4862 AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
4863 // We're at a tree leaf, produce a conditional comparison operation.
4864 auto &MRI = *MIB.getMRI();
4865 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4866 unsigned Opcode = ValDef->getOpcode();
4867 if (auto *Cmp = dyn_cast<GAnyCmp>(Val: ValDef)) {
4868 Register LHS = Cmp->getLHSReg();
4869 Register RHS = Cmp->getRHSReg();
4870 CmpInst::Predicate CC = Cmp->getCond();
4871 if (Negate)
4872 CC = CmpInst::getInversePredicate(pred: CC);
4873 if (isa<GICmp>(Val: Cmp)) {
4874 OutCC = changeICMPPredToAArch64CC(P: CC, RHS, MRI: MIB.getMRI());
4875 } else {
4876 // Handle special FP cases.
4877 AArch64CC::CondCode ExtraCC;
4878 changeFPCCToANDAArch64CC(CC, CondCode&: OutCC, CondCode2&: ExtraCC);
4879 // Some floating point conditions can't be tested with a single condition
4880 // code. Construct an additional comparison in this case.
4881 if (ExtraCC != AArch64CC::AL) {
4882 MachineInstr *ExtraCmp;
4883 if (!CCOp)
4884 ExtraCmp = emitFPCompare(LHS, RHS, MIRBuilder&: MIB, Pred: CC);
4885 else
4886 ExtraCmp =
4887 emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC: ExtraCC, MIB);
4888 CCOp = ExtraCmp->getOperand(i: 0).getReg();
4889 Predicate = ExtraCC;
4890 }
4891 }
4892
4893 // Produce a normal comparison if we are first in the chain
4894 if (!CCOp) {
4895 if (isa<GICmp>(Val: Cmp))
4896 return emitCMP(LHS&: Cmp->getOperand(i: 2), RHS&: Cmp->getOperand(i: 3), MIRBuilder&: MIB);
4897 return emitFPCompare(LHS: Cmp->getOperand(i: 2).getReg(),
4898 RHS: Cmp->getOperand(i: 3).getReg(), MIRBuilder&: MIB);
4899 }
4900 // Otherwise produce a ccmp.
4901 return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
4902 }
4903 assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree");
4904
4905 bool IsOR = Opcode == TargetOpcode::G_OR;
4906
4907 Register LHS = ValDef->getOperand(i: 1).getReg();
4908 bool CanNegateL;
4909 bool MustBeFirstL;
4910 bool ValidL = canEmitConjunction(Val: LHS, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI);
4911 assert(ValidL && "Valid conjunction/disjunction tree");
4912 (void)ValidL;
4913
4914 Register RHS = ValDef->getOperand(i: 2).getReg();
4915 bool CanNegateR;
4916 bool MustBeFirstR;
4917 bool ValidR = canEmitConjunction(Val: RHS, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI);
4918 assert(ValidR && "Valid conjunction/disjunction tree");
4919 (void)ValidR;
4920
4921 // Swap sub-tree that must come first to the right side.
4922 if (MustBeFirstL) {
4923 assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
4924 std::swap(a&: LHS, b&: RHS);
4925 std::swap(a&: CanNegateL, b&: CanNegateR);
4926 std::swap(a&: MustBeFirstL, b&: MustBeFirstR);
4927 }
4928
4929 bool NegateR;
4930 bool NegateAfterR;
4931 bool NegateL;
4932 bool NegateAfterAll;
4933 if (Opcode == TargetOpcode::G_OR) {
4934 // Swap the sub-tree that we can negate naturally to the left.
4935 if (!CanNegateL) {
4936 assert(CanNegateR && "at least one side must be negatable");
4937 assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
4938 assert(!Negate);
4939 std::swap(a&: LHS, b&: RHS);
4940 NegateR = false;
4941 NegateAfterR = true;
4942 } else {
4943 // Negate the left sub-tree if possible, otherwise negate the result.
4944 NegateR = CanNegateR;
4945 NegateAfterR = !CanNegateR;
4946 }
4947 NegateL = true;
4948 NegateAfterAll = !Negate;
4949 } else {
4950 assert(Opcode == TargetOpcode::G_AND &&
4951 "Valid conjunction/disjunction tree");
4952 assert(!Negate && "Valid conjunction/disjunction tree");
4953
4954 NegateL = false;
4955 NegateR = false;
4956 NegateAfterR = false;
4957 NegateAfterAll = false;
4958 }
4959
4960 // Emit sub-trees.
4961 AArch64CC::CondCode RHSCC;
4962 MachineInstr *CmpR =
4963 emitConjunctionRec(Val: RHS, OutCC&: RHSCC, Negate: NegateR, CCOp, Predicate, MIB);
4964 if (NegateAfterR)
4965 RHSCC = AArch64CC::getInvertedCondCode(Code: RHSCC);
4966 MachineInstr *CmpL = emitConjunctionRec(
4967 Val: LHS, OutCC, Negate: NegateL, CCOp: CmpR->getOperand(i: 0).getReg(), Predicate: RHSCC, MIB);
4968 if (NegateAfterAll)
4969 OutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
4970 return CmpL;
4971}
4972
4973MachineInstr *AArch64InstructionSelector::emitConjunction(
4974 Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
4975 bool DummyCanNegate;
4976 bool DummyMustBeFirst;
4977 if (!canEmitConjunction(Val, CanNegate&: DummyCanNegate, MustBeFirst&: DummyMustBeFirst, WillNegate: false,
4978 MRI&: *MIB.getMRI()))
4979 return nullptr;
4980 return emitConjunctionRec(Val, OutCC, Negate: false, CCOp: Register(), Predicate: AArch64CC::AL, MIB);
4981}
4982
4983bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
4984 MachineInstr &CondMI) {
4985 AArch64CC::CondCode AArch64CC;
4986 MachineInstr *ConjMI = emitConjunction(Val: SelI.getCondReg(), OutCC&: AArch64CC, MIB);
4987 if (!ConjMI)
4988 return false;
4989
4990 emitSelect(Dst: SelI.getReg(Idx: 0), True: SelI.getTrueReg(), False: SelI.getFalseReg(), CC: AArch64CC, MIB);
4991 SelI.eraseFromParent();
4992 return true;
4993}
4994
4995bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
4996 MachineRegisterInfo &MRI = *MIB.getMRI();
4997 // We want to recognize this pattern:
4998 //
4999 // $z = G_FCMP pred, $x, $y
5000 // ...
5001 // $w = G_SELECT $z, $a, $b
5002 //
5003 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
5004 // some copies/truncs in between.)
5005 //
5006 // If we see this, then we can emit something like this:
5007 //
5008 // fcmp $x, $y
5009 // fcsel $w, $a, $b, pred
5010 //
5011 // Rather than emitting both of the rather long sequences in the standard
5012 // G_FCMP/G_SELECT select methods.
5013
5014 // First, check if the condition is defined by a compare.
5015 MachineInstr *CondDef = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
5016
5017 // We can only fold if all of the defs have one use.
5018 Register CondDefReg = CondDef->getOperand(i: 0).getReg();
5019 if (!MRI.hasOneNonDBGUse(RegNo: CondDefReg)) {
5020 // Unless it's another select.
5021 for (const MachineInstr &UI : MRI.use_nodbg_instructions(Reg: CondDefReg)) {
5022 if (CondDef == &UI)
5023 continue;
5024 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5025 return false;
5026 }
5027 }
5028
5029 // Is the condition defined by a compare?
5030 unsigned CondOpc = CondDef->getOpcode();
5031 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5032 if (tryOptSelectConjunction(SelI&: I, CondMI&: *CondDef))
5033 return true;
5034 return false;
5035 }
5036
5037 AArch64CC::CondCode CondCode;
5038 if (CondOpc == TargetOpcode::G_ICMP) {
5039 auto &PredOp = CondDef->getOperand(i: 1);
5040 emitIntegerCompare(LHS&: CondDef->getOperand(i: 2), RHS&: CondDef->getOperand(i: 3), Predicate&: PredOp,
5041 MIRBuilder&: MIB);
5042 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
5043 CondCode =
5044 changeICMPPredToAArch64CC(P: Pred, RHS: CondDef->getOperand(i: 3).getReg(), MRI: &MRI);
5045 } else {
5046 // Get the condition code for the select.
5047 auto Pred =
5048 static_cast<CmpInst::Predicate>(CondDef->getOperand(i: 1).getPredicate());
5049 AArch64CC::CondCode CondCode2;
5050 changeFCMPPredToAArch64CC(P: Pred, CondCode, CondCode2);
5051
5052 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
5053 // instructions to emit the comparison.
5054 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
5055 // unnecessary.
5056 if (CondCode2 != AArch64CC::AL)
5057 return false;
5058
5059 if (!emitFPCompare(LHS: CondDef->getOperand(i: 2).getReg(),
5060 RHS: CondDef->getOperand(i: 3).getReg(), MIRBuilder&: MIB)) {
5061 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n");
5062 return false;
5063 }
5064 }
5065
5066 // Emit the select.
5067 emitSelect(Dst: I.getOperand(i: 0).getReg(), True: I.getOperand(i: 2).getReg(),
5068 False: I.getOperand(i: 3).getReg(), CC: CondCode, MIB);
5069 I.eraseFromParent();
5070 return true;
5071}
5072
5073MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5074 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
5075 MachineIRBuilder &MIRBuilder) const {
5076 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&
5077 "Unexpected MachineOperand");
5078 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5079 // We want to find this sort of thing:
5080 // x = G_SUB 0, y
5081 // G_ICMP z, x
5082 //
5083 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
5084 // e.g:
5085 //
5086 // cmn z, y
5087
5088 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
5089 MachineInstr *LHSDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
5090 MachineInstr *RHSDef = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
5091 auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
5092
5093 // Given this:
5094 //
5095 // x = G_SUB 0, y
5096 // G_ICMP z, x
5097 //
5098 // Produce this:
5099 //
5100 // cmn z, y
5101 if (isCMN(MaybeSub: RHSDef, Pred: P, MRI))
5102 return emitCMN(LHS, RHS&: RHSDef->getOperand(i: 2), MIRBuilder);
5103
5104 // Same idea here, but with the LHS of the compare instead:
5105 //
5106 // Given this:
5107 //
5108 // x = G_SUB 0, y
5109 // G_ICMP x, z
5110 //
5111 // Produce this:
5112 //
5113 // cmn y, z
5114 //
5115 // But be careful! We need to swap the predicate!
5116 if (isCMN(MaybeSub: LHSDef, Pred: P, MRI)) {
5117 if (!CmpInst::isEquality(pred: P)) {
5118 P = CmpInst::getSwappedPredicate(pred: P);
5119 Predicate = MachineOperand::CreatePredicate(Pred: P);
5120 }
5121 return emitCMN(LHS&: LHSDef->getOperand(i: 2), RHS, MIRBuilder);
5122 }
5123
5124 // Given this:
5125 //
5126 // z = G_AND x, y
5127 // G_ICMP z, 0
5128 //
5129 // Produce this if the compare is signed:
5130 //
5131 // tst x, y
5132 if (!CmpInst::isUnsigned(Pred: P) && LHSDef &&
5133 LHSDef->getOpcode() == TargetOpcode::G_AND) {
5134 // Make sure that the RHS is 0.
5135 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
5136 if (!ValAndVReg || ValAndVReg->Value != 0)
5137 return nullptr;
5138
5139 return emitTST(LHS&: LHSDef->getOperand(i: 1),
5140 RHS&: LHSDef->getOperand(i: 2), MIRBuilder);
5141 }
5142
5143 return nullptr;
5144}
5145
5146bool AArch64InstructionSelector::selectShuffleVector(
5147 MachineInstr &I, MachineRegisterInfo &MRI) {
5148 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5149 Register Src1Reg = I.getOperand(i: 1).getReg();
5150 Register Src2Reg = I.getOperand(i: 2).getReg();
5151 ArrayRef<int> Mask = I.getOperand(i: 3).getShuffleMask();
5152 assert(DstTy == MRI.getType(Src1Reg) &&
5153 "Expected equal shuffle types during selection");
5154
5155 MachineBasicBlock &MBB = *I.getParent();
5156 MachineFunction &MF = *MBB.getParent();
5157 LLVMContext &Ctx = MF.getFunction().getContext();
5158
5159 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
5160 int NumElts = DstTy.getNumElements();
5161
5162 SmallVector<int> NewMask;
5163 bool FirstUsed = false;
5164 bool SecondUsed = false;
5165 for (int M : Mask) {
5166 // Map any undef or zero lanes to 255.
5167 if (M < 0 || VT->getKnownBits(R: M < NumElts ? Src1Reg : Src2Reg,
5168 DemandedElts: APInt::getOneBitSet(numBits: NumElts, BitNo: M % NumElts))
5169 .isZero()) {
5170 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte)
5171 NewMask.push_back(Elt: 255);
5172 continue;
5173 }
5174
5175 FirstUsed |= M < NumElts;
5176 SecondUsed |= M >= NumElts;
5177 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
5178 unsigned Offset = Byte + M * BytesPerElt;
5179 NewMask.push_back(Elt: Offset);
5180 }
5181 }
5182
5183 // If the first is unused or all zeros, use the second src in a tbl1.
5184 if (!FirstUsed) {
5185 int ByteLanes = DstTy.getSizeInBits() == 128 ? 16 : 8;
5186 for (int &M : NewMask) {
5187 if (M != 255) {
5188 assert(M >= ByteLanes && M < 2 * ByteLanes);
5189 M -= ByteLanes;
5190 }
5191 }
5192 std::swap(a&: Src1Reg, b&: Src2Reg);
5193 std::swap(a&: FirstUsed, b&: SecondUsed);
5194 }
5195
5196 // Use a constant pool to load the index vector for TBL.
5197 SmallVector<Constant *> CstIdxs;
5198 transform(Range&: NewMask, d_first: std::back_inserter(x&: CstIdxs), F: [&Ctx](int M) {
5199 return ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: M);
5200 });
5201 Constant *CPVal = ConstantVector::get(V: CstIdxs);
5202 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder&: MIB);
5203 if (!IndexLoad) {
5204 LLVM_DEBUG(dbgs() << "Could not load from a constant pool");
5205 return false;
5206 }
5207
5208 if (DstTy.getSizeInBits() != 128) {
5209 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty");
5210 // This case can be done with TBL1.
5211 MachineInstr *Concat =
5212 emitVectorConcat(Dst: std::nullopt, Op1: Src1Reg, Op2: Src2Reg, MIRBuilder&: MIB);
5213 if (!Concat) {
5214 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1");
5215 return false;
5216 }
5217
5218 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
5219 IndexLoad = emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass,
5220 Scalar: IndexLoad->getOperand(i: 0).getReg(), MIRBuilder&: MIB);
5221
5222 auto TBL1 = MIB.buildInstr(
5223 Opc: AArch64::TBLv16i8One, DstOps: {&AArch64::FPR128RegClass},
5224 SrcOps: {Concat->getOperand(i: 0).getReg(), IndexLoad->getOperand(i: 0).getReg()});
5225 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5226
5227 auto Copy =
5228 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
5229 .addReg(RegNo: TBL1.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5230 RBI.constrainGenericRegister(Reg: Copy.getReg(Idx: 0), RC: AArch64::FPR64RegClass, MRI);
5231 I.eraseFromParent();
5232 return true;
5233 }
5234
5235 if (!SecondUsed) {
5236 auto TBL1 = MIB.buildInstr(Opc: AArch64::TBLv16i8One, DstOps: {I.getOperand(i: 0)},
5237 SrcOps: {Src1Reg, IndexLoad->getOperand(i: 0)});
5238 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5239 I.eraseFromParent();
5240 return true;
5241 }
5242
5243 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
5244 // Q registers for regalloc.
5245 SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
5246 auto RegSeq = createQTuple(Regs, MIB);
5247 auto TBL2 = MIB.buildInstr(Opc: AArch64::TBLv16i8Two, DstOps: {I.getOperand(i: 0)},
5248 SrcOps: {RegSeq, IndexLoad->getOperand(i: 0)});
5249 constrainSelectedInstRegOperands(I&: *TBL2, TII, TRI, RBI);
5250 I.eraseFromParent();
5251 return true;
5252}
5253
5254MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5255 std::optional<Register> DstReg, Register SrcReg, Register EltReg,
5256 unsigned LaneIdx, const RegisterBank &RB,
5257 MachineIRBuilder &MIRBuilder) const {
5258 MachineInstr *InsElt = nullptr;
5259 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5260 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5261
5262 // Create a register to define with the insert if one wasn't passed in.
5263 if (!DstReg)
5264 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
5265
5266 unsigned EltSize = MRI.getType(Reg: EltReg).getSizeInBits();
5267 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
5268
5269 if (RB.getID() == AArch64::FPRRegBankID) {
5270 auto InsSub = emitScalarToVector(EltSize, DstRC, Scalar: EltReg, MIRBuilder);
5271 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5272 .addImm(Val: LaneIdx)
5273 .addUse(RegNo: InsSub->getOperand(i: 0).getReg())
5274 .addImm(Val: 0);
5275 } else {
5276 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5277 .addImm(Val: LaneIdx)
5278 .addUse(RegNo: EltReg);
5279 }
5280
5281 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
5282 return InsElt;
5283}
5284
5285bool AArch64InstructionSelector::selectUSMovFromExtend(
5286 MachineInstr &MI, MachineRegisterInfo &MRI) {
5287 if (MI.getOpcode() != TargetOpcode::G_SEXT &&
5288 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5289 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5290 return false;
5291 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
5292 const Register DefReg = MI.getOperand(i: 0).getReg();
5293 const LLT DstTy = MRI.getType(Reg: DefReg);
5294 unsigned DstSize = DstTy.getSizeInBits();
5295
5296 if (DstSize != 32 && DstSize != 64)
5297 return false;
5298
5299 MachineInstr *Extract = getOpcodeDef(Opcode: TargetOpcode::G_EXTRACT_VECTOR_ELT,
5300 Reg: MI.getOperand(i: 1).getReg(), MRI);
5301 int64_t Lane;
5302 if (!Extract || !mi_match(R: Extract->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Lane)))
5303 return false;
5304 Register Src0 = Extract->getOperand(i: 1).getReg();
5305
5306 const LLT VecTy = MRI.getType(Reg: Src0);
5307 if (VecTy.isScalableVector())
5308 return false;
5309
5310 if (VecTy.getSizeInBits() != 128) {
5311 const MachineInstr *ScalarToVector = emitScalarToVector(
5312 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: Src0, MIRBuilder&: MIB);
5313 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
5314 Src0 = ScalarToVector->getOperand(i: 0).getReg();
5315 }
5316
5317 unsigned Opcode;
5318 if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
5319 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5320 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
5321 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5322 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
5323 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5324 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
5325 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5326 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
5327 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5328 else
5329 llvm_unreachable("Unexpected type combo for S/UMov!");
5330
5331 // We may need to generate one of these, depending on the type and sign of the
5332 // input:
5333 // DstReg = SMOV Src0, Lane;
5334 // NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
5335 MachineInstr *ExtI = nullptr;
5336 if (DstSize == 64 && !IsSigned) {
5337 Register NewReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
5338 MIB.buildInstr(Opc: Opcode, DstOps: {NewReg}, SrcOps: {Src0}).addImm(Val: Lane);
5339 ExtI = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
5340 .addUse(RegNo: NewReg)
5341 .addImm(Val: AArch64::sub_32);
5342 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
5343 } else
5344 ExtI = MIB.buildInstr(Opc: Opcode, DstOps: {DefReg}, SrcOps: {Src0}).addImm(Val: Lane);
5345
5346 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
5347 MI.eraseFromParent();
5348 return true;
5349}
5350
5351MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm8(
5352 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5353 unsigned int Op;
5354 if (DstSize == 128) {
5355 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5356 return nullptr;
5357 Op = AArch64::MOVIv16b_ns;
5358 } else {
5359 Op = AArch64::MOVIv8b_ns;
5360 }
5361
5362 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5363
5364 if (AArch64_AM::isAdvSIMDModImmType9(Imm: Val)) {
5365 Val = AArch64_AM::encodeAdvSIMDModImmType9(Imm: Val);
5366 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5367 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5368 return &*Mov;
5369 }
5370 return nullptr;
5371}
5372
5373MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm16(
5374 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5375 bool Inv) {
5376
5377 unsigned int Op;
5378 if (DstSize == 128) {
5379 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5380 return nullptr;
5381 Op = Inv ? AArch64::MVNIv8i16 : AArch64::MOVIv8i16;
5382 } else {
5383 Op = Inv ? AArch64::MVNIv4i16 : AArch64::MOVIv4i16;
5384 }
5385
5386 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5387 uint64_t Shift;
5388
5389 if (AArch64_AM::isAdvSIMDModImmType5(Imm: Val)) {
5390 Val = AArch64_AM::encodeAdvSIMDModImmType5(Imm: Val);
5391 Shift = 0;
5392 } else if (AArch64_AM::isAdvSIMDModImmType6(Imm: Val)) {
5393 Val = AArch64_AM::encodeAdvSIMDModImmType6(Imm: Val);
5394 Shift = 8;
5395 } else
5396 return nullptr;
5397
5398 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5399 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5400 return &*Mov;
5401}
5402
5403MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm32(
5404 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5405 bool Inv) {
5406
5407 unsigned int Op;
5408 if (DstSize == 128) {
5409 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5410 return nullptr;
5411 Op = Inv ? AArch64::MVNIv4i32 : AArch64::MOVIv4i32;
5412 } else {
5413 Op = Inv ? AArch64::MVNIv2i32 : AArch64::MOVIv2i32;
5414 }
5415
5416 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5417 uint64_t Shift;
5418
5419 if ((AArch64_AM::isAdvSIMDModImmType1(Imm: Val))) {
5420 Val = AArch64_AM::encodeAdvSIMDModImmType1(Imm: Val);
5421 Shift = 0;
5422 } else if ((AArch64_AM::isAdvSIMDModImmType2(Imm: Val))) {
5423 Val = AArch64_AM::encodeAdvSIMDModImmType2(Imm: Val);
5424 Shift = 8;
5425 } else if ((AArch64_AM::isAdvSIMDModImmType3(Imm: Val))) {
5426 Val = AArch64_AM::encodeAdvSIMDModImmType3(Imm: Val);
5427 Shift = 16;
5428 } else if ((AArch64_AM::isAdvSIMDModImmType4(Imm: Val))) {
5429 Val = AArch64_AM::encodeAdvSIMDModImmType4(Imm: Val);
5430 Shift = 24;
5431 } else
5432 return nullptr;
5433
5434 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5435 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5436 return &*Mov;
5437}
5438
5439MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm64(
5440 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5441
5442 unsigned int Op;
5443 if (DstSize == 128) {
5444 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5445 return nullptr;
5446 Op = AArch64::MOVIv2d_ns;
5447 } else {
5448 Op = AArch64::MOVID;
5449 }
5450
5451 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5452 if (AArch64_AM::isAdvSIMDModImmType10(Imm: Val)) {
5453 Val = AArch64_AM::encodeAdvSIMDModImmType10(Imm: Val);
5454 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5455 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5456 return &*Mov;
5457 }
5458 return nullptr;
5459}
5460
5461MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm321s(
5462 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5463 bool Inv) {
5464
5465 unsigned int Op;
5466 if (DstSize == 128) {
5467 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5468 return nullptr;
5469 Op = Inv ? AArch64::MVNIv4s_msl : AArch64::MOVIv4s_msl;
5470 } else {
5471 Op = Inv ? AArch64::MVNIv2s_msl : AArch64::MOVIv2s_msl;
5472 }
5473
5474 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5475 uint64_t Shift;
5476
5477 if (AArch64_AM::isAdvSIMDModImmType7(Imm: Val)) {
5478 Val = AArch64_AM::encodeAdvSIMDModImmType7(Imm: Val);
5479 Shift = 264;
5480 } else if (AArch64_AM::isAdvSIMDModImmType8(Imm: Val)) {
5481 Val = AArch64_AM::encodeAdvSIMDModImmType8(Imm: Val);
5482 Shift = 272;
5483 } else
5484 return nullptr;
5485
5486 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5487 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5488 return &*Mov;
5489}
5490
5491MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImmFP(
5492 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5493
5494 unsigned int Op;
5495 bool IsWide = false;
5496 if (DstSize == 128) {
5497 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5498 return nullptr;
5499 Op = AArch64::FMOVv4f32_ns;
5500 IsWide = true;
5501 } else {
5502 Op = AArch64::FMOVv2f32_ns;
5503 }
5504
5505 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5506
5507 if (AArch64_AM::isAdvSIMDModImmType11(Imm: Val)) {
5508 Val = AArch64_AM::encodeAdvSIMDModImmType11(Imm: Val);
5509 } else if (IsWide && AArch64_AM::isAdvSIMDModImmType12(Imm: Val)) {
5510 Val = AArch64_AM::encodeAdvSIMDModImmType12(Imm: Val);
5511 Op = AArch64::FMOVv2f64_ns;
5512 } else
5513 return nullptr;
5514
5515 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5516 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5517 return &*Mov;
5518}
5519
5520bool AArch64InstructionSelector::selectIndexedExtLoad(
5521 MachineInstr &MI, MachineRegisterInfo &MRI) {
5522 auto &ExtLd = cast<GIndexedAnyExtLoad>(Val&: MI);
5523 Register Dst = ExtLd.getDstReg();
5524 Register WriteBack = ExtLd.getWritebackReg();
5525 Register Base = ExtLd.getBaseReg();
5526 Register Offset = ExtLd.getOffsetReg();
5527 LLT Ty = MRI.getType(Reg: Dst);
5528 assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
5529 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5530 bool IsPre = ExtLd.isPre();
5531 bool IsSExt = isa<GIndexedSExtLoad>(Val: ExtLd);
5532 unsigned InsertIntoSubReg = 0;
5533 bool IsDst64 = Ty.getSizeInBits() == 64;
5534
5535 // ZExt/SExt should be on gpr but can handle extload and zextload of fpr, so
5536 // long as they are scalar.
5537 bool IsFPR = RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID;
5538 if ((IsSExt && IsFPR) || Ty.isVector())
5539 return false;
5540
5541 unsigned Opc = 0;
5542 LLT NewLdDstTy;
5543 LLT s32 = LLT::scalar(SizeInBits: 32);
5544 LLT s64 = LLT::scalar(SizeInBits: 64);
5545
5546 if (MemSizeBits == 8) {
5547 if (IsSExt) {
5548 if (IsDst64)
5549 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5550 else
5551 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5552 NewLdDstTy = IsDst64 ? s64 : s32;
5553 } else if (IsFPR) {
5554 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5555 InsertIntoSubReg = AArch64::bsub;
5556 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5557 } else {
5558 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5559 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5560 NewLdDstTy = s32;
5561 }
5562 } else if (MemSizeBits == 16) {
5563 if (IsSExt) {
5564 if (IsDst64)
5565 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5566 else
5567 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5568 NewLdDstTy = IsDst64 ? s64 : s32;
5569 } else if (IsFPR) {
5570 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5571 InsertIntoSubReg = AArch64::hsub;
5572 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5573 } else {
5574 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5575 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5576 NewLdDstTy = s32;
5577 }
5578 } else if (MemSizeBits == 32) {
5579 if (IsSExt) {
5580 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5581 NewLdDstTy = s64;
5582 } else if (IsFPR) {
5583 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5584 InsertIntoSubReg = AArch64::ssub;
5585 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5586 } else {
5587 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5588 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5589 NewLdDstTy = s32;
5590 }
5591 } else {
5592 llvm_unreachable("Unexpected size for indexed load");
5593 }
5594
5595 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5596 if (!Cst)
5597 return false; // Shouldn't happen, but just in case.
5598
5599 auto LdMI = MIB.buildInstr(Opc, DstOps: {WriteBack, NewLdDstTy}, SrcOps: {Base})
5600 .addImm(Val: Cst->getSExtValue());
5601 LdMI.cloneMemRefs(OtherMI: ExtLd);
5602 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5603 // Make sure to select the load with the MemTy as the dest type, and then
5604 // insert into a larger reg if needed.
5605 if (InsertIntoSubReg) {
5606 // Generate a SUBREG_TO_REG.
5607 auto SubToReg = MIB.buildInstr(Opc: TargetOpcode::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5608 .addUse(RegNo: LdMI.getReg(Idx: 1))
5609 .addImm(Val: InsertIntoSubReg);
5610 RBI.constrainGenericRegister(
5611 Reg: SubToReg.getReg(Idx: 0),
5612 RC: *getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst),
5613 RB: *RBI.getRegBank(Reg: Dst, MRI, TRI)),
5614 MRI);
5615 } else {
5616 auto Copy = MIB.buildCopy(Res: Dst, Op: LdMI.getReg(Idx: 1));
5617 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
5618 }
5619 MI.eraseFromParent();
5620
5621 return true;
5622}
5623
5624bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
5625 MachineRegisterInfo &MRI) {
5626 auto &Ld = cast<GIndexedLoad>(Val&: MI);
5627 Register Dst = Ld.getDstReg();
5628 Register WriteBack = Ld.getWritebackReg();
5629 Register Base = Ld.getBaseReg();
5630 Register Offset = Ld.getOffsetReg();
5631 assert(MRI.getType(Dst).getSizeInBits() <= 128 &&
5632 "Unexpected type for indexed load");
5633 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5634
5635 if (MemSize < MRI.getType(Reg: Dst).getSizeInBytes())
5636 return selectIndexedExtLoad(MI, MRI);
5637
5638 unsigned Opc = 0;
5639 if (Ld.isPre()) {
5640 static constexpr unsigned GPROpcodes[] = {
5641 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5642 AArch64::LDRXpre};
5643 static constexpr unsigned FPROpcodes[] = {
5644 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5645 AArch64::LDRQpre};
5646 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5647 ? FPROpcodes[Log2_32(Value: MemSize)]
5648 : GPROpcodes[Log2_32(Value: MemSize)];
5649 ;
5650 } else {
5651 static constexpr unsigned GPROpcodes[] = {
5652 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5653 AArch64::LDRXpost};
5654 static constexpr unsigned FPROpcodes[] = {
5655 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5656 AArch64::LDRDpost, AArch64::LDRQpost};
5657 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5658 ? FPROpcodes[Log2_32(Value: MemSize)]
5659 : GPROpcodes[Log2_32(Value: MemSize)];
5660 ;
5661 }
5662 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5663 if (!Cst)
5664 return false; // Shouldn't happen, but just in case.
5665 auto LdMI =
5666 MIB.buildInstr(Opc, DstOps: {WriteBack, Dst}, SrcOps: {Base}).addImm(Val: Cst->getSExtValue());
5667 LdMI.cloneMemRefs(OtherMI: Ld);
5668 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5669 MI.eraseFromParent();
5670 return true;
5671}
5672
5673bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &I,
5674 MachineRegisterInfo &MRI) {
5675 Register Dst = I.getWritebackReg();
5676 Register Val = I.getValueReg();
5677 Register Base = I.getBaseReg();
5678 Register Offset = I.getOffsetReg();
5679 assert(MRI.getType(Val).getSizeInBits() <= 128 &&
5680 "Unexpected type for indexed store");
5681
5682 LocationSize MemSize = I.getMMO().getSize();
5683 unsigned MemSizeInBytes = MemSize.getValue();
5684
5685 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5686 "Unexpected indexed store size");
5687 unsigned MemSizeLog2 = Log2_32(Value: MemSizeInBytes);
5688
5689 unsigned Opc = 0;
5690 if (I.isPre()) {
5691 static constexpr unsigned GPROpcodes[] = {
5692 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5693 AArch64::STRXpre};
5694 static constexpr unsigned FPROpcodes[] = {
5695 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5696 AArch64::STRQpre};
5697
5698 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5699 Opc = FPROpcodes[MemSizeLog2];
5700 else
5701 Opc = GPROpcodes[MemSizeLog2];
5702 } else {
5703 static constexpr unsigned GPROpcodes[] = {
5704 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5705 AArch64::STRXpost};
5706 static constexpr unsigned FPROpcodes[] = {
5707 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5708 AArch64::STRDpost, AArch64::STRQpost};
5709
5710 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5711 Opc = FPROpcodes[MemSizeLog2];
5712 else
5713 Opc = GPROpcodes[MemSizeLog2];
5714 }
5715
5716 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5717 if (!Cst)
5718 return false; // Shouldn't happen, but just in case.
5719 auto Str =
5720 MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Val, Base}).addImm(Val: Cst->getSExtValue());
5721 Str.cloneMemRefs(OtherMI: I);
5722 constrainSelectedInstRegOperands(I&: *Str, TII, TRI, RBI);
5723 I.eraseFromParent();
5724 return true;
5725}
5726
5727MachineInstr *
5728AArch64InstructionSelector::emitConstantVector(Register Dst, Constant *CV,
5729 MachineIRBuilder &MIRBuilder,
5730 MachineRegisterInfo &MRI) {
5731 LLT DstTy = MRI.getType(Reg: Dst);
5732 unsigned DstSize = DstTy.getSizeInBits();
5733 assert((DstSize == 64 || DstSize == 128) &&
5734 "Unexpected vector constant size");
5735
5736 if (CV->isNullValue()) {
5737 if (DstSize == 128) {
5738 auto Mov =
5739 MIRBuilder.buildInstr(Opc: AArch64::MOVIv2d_ns, DstOps: {Dst}, SrcOps: {}).addImm(Val: 0);
5740 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5741 return &*Mov;
5742 }
5743
5744 if (DstSize == 64) {
5745 auto Mov =
5746 MIRBuilder
5747 .buildInstr(Opc: AArch64::MOVIv2d_ns, DstOps: {&AArch64::FPR128RegClass}, SrcOps: {})
5748 .addImm(Val: 0);
5749 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5750 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5751 RBI.constrainGenericRegister(Reg: Dst, RC: AArch64::FPR64RegClass, MRI);
5752 return &*Copy;
5753 }
5754 }
5755
5756 if (Constant *SplatValue = CV->getSplatValue()) {
5757 APInt SplatValueAsInt =
5758 isa<ConstantFP>(Val: SplatValue)
5759 ? cast<ConstantFP>(Val: SplatValue)->getValueAPF().bitcastToAPInt()
5760 : SplatValue->getUniqueInteger();
5761 APInt DefBits = APInt::getSplat(
5762 NewLen: DstSize, V: SplatValueAsInt.trunc(width: DstTy.getScalarSizeInBits()));
5763 auto TryMOVIWithBits = [&](APInt DefBits) -> MachineInstr * {
5764 MachineInstr *NewOp;
5765 bool Inv = false;
5766 if ((NewOp = tryAdvSIMDModImm64(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)) ||
5767 (NewOp =
5768 tryAdvSIMDModImm32(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5769 (NewOp =
5770 tryAdvSIMDModImm321s(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5771 (NewOp =
5772 tryAdvSIMDModImm16(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5773 (NewOp = tryAdvSIMDModImm8(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)) ||
5774 (NewOp = tryAdvSIMDModImmFP(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)))
5775 return NewOp;
5776
5777 DefBits = ~DefBits;
5778 Inv = true;
5779 if ((NewOp =
5780 tryAdvSIMDModImm32(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5781 (NewOp =
5782 tryAdvSIMDModImm321s(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5783 (NewOp = tryAdvSIMDModImm16(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)))
5784 return NewOp;
5785 return nullptr;
5786 };
5787
5788 if (auto *NewOp = TryMOVIWithBits(DefBits))
5789 return NewOp;
5790
5791 // See if a fneg of the constant can be materialized with a MOVI, etc
5792 auto TryWithFNeg = [&](APInt DefBits, int NumBits,
5793 unsigned NegOpc) -> MachineInstr * {
5794 // FNegate each sub-element of the constant
5795 APInt Neg = APInt::getHighBitsSet(numBits: NumBits, hiBitsSet: 1).zext(width: DstSize);
5796 APInt NegBits(DstSize, 0);
5797 unsigned NumElts = DstSize / NumBits;
5798 for (unsigned i = 0; i < NumElts; i++)
5799 NegBits |= Neg << (NumBits * i);
5800 NegBits = DefBits ^ NegBits;
5801
5802 // Try to create the new constants with MOVI, and if so generate a fneg
5803 // for it.
5804 if (auto *NewOp = TryMOVIWithBits(NegBits)) {
5805 Register NewDst = MRI.createVirtualRegister(
5806 RegClass: DstSize == 64 ? &AArch64::FPR64RegClass : &AArch64::FPR128RegClass);
5807 NewOp->getOperand(i: 0).setReg(NewDst);
5808 return MIRBuilder.buildInstr(Opc: NegOpc, DstOps: {Dst}, SrcOps: {NewDst});
5809 }
5810 return nullptr;
5811 };
5812 MachineInstr *R;
5813 if ((R = TryWithFNeg(DefBits, 32,
5814 DstSize == 64 ? AArch64::FNEGv2f32
5815 : AArch64::FNEGv4f32)) ||
5816 (R = TryWithFNeg(DefBits, 64,
5817 DstSize == 64 ? AArch64::FNEGDr
5818 : AArch64::FNEGv2f64)) ||
5819 (STI.hasFullFP16() &&
5820 (R = TryWithFNeg(DefBits, 16,
5821 DstSize == 64 ? AArch64::FNEGv4f16
5822 : AArch64::FNEGv8f16))))
5823 return R;
5824 }
5825
5826 auto *CPLoad = emitLoadFromConstantPool(CPVal: CV, MIRBuilder);
5827 if (!CPLoad) {
5828 LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!");
5829 return nullptr;
5830 }
5831
5832 auto Copy = MIRBuilder.buildCopy(Res: Dst, Op: CPLoad->getOperand(i: 0));
5833 RBI.constrainGenericRegister(
5834 Reg: Dst, RC: *MRI.getRegClass(Reg: CPLoad->getOperand(i: 0).getReg()), MRI);
5835 return &*Copy;
5836}
5837
5838bool AArch64InstructionSelector::tryOptConstantBuildVec(
5839 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
5840 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5841 unsigned DstSize = DstTy.getSizeInBits();
5842 assert(DstSize <= 128 && "Unexpected build_vec type!");
5843 if (DstSize < 32)
5844 return false;
5845 // Check if we're building a constant vector, in which case we want to
5846 // generate a constant pool load instead of a vector insert sequence.
5847 SmallVector<Constant *, 16> Csts;
5848 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
5849 Register OpReg = I.getOperand(i: Idx).getReg();
5850 if (auto AnyConst = getAnyConstantVRegValWithLookThrough(
5851 VReg: OpReg, MRI, /*LookThroughInstrs=*/true,
5852 /*LookThroughAnyExt=*/true)) {
5853 MachineInstr *DefMI = MRI.getVRegDef(Reg: AnyConst->VReg);
5854
5855 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT) {
5856 Csts.emplace_back(
5857 Args: ConstantInt::get(Context&: MIB.getMF().getFunction().getContext(),
5858 V: std::move(AnyConst->Value)));
5859 continue;
5860 }
5861
5862 if (DefMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
5863 Csts.emplace_back(
5864 Args: const_cast<ConstantFP *>(DefMI->getOperand(i: 1).getFPImm()));
5865 continue;
5866 }
5867 }
5868 return false;
5869 }
5870 Constant *CV = ConstantVector::get(V: Csts);
5871 if (!emitConstantVector(Dst: I.getOperand(i: 0).getReg(), CV, MIRBuilder&: MIB, MRI))
5872 return false;
5873 I.eraseFromParent();
5874 return true;
5875}
5876
5877bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5878 MachineInstr &I, MachineRegisterInfo &MRI) {
5879 // Given:
5880 // %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
5881 //
5882 // Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
5883 Register Dst = I.getOperand(i: 0).getReg();
5884 Register EltReg = I.getOperand(i: 1).getReg();
5885 LLT EltTy = MRI.getType(Reg: EltReg);
5886 // If the index isn't on the same bank as its elements, then this can't be a
5887 // SUBREG_TO_REG.
5888 const RegisterBank &EltRB = *RBI.getRegBank(Reg: EltReg, MRI, TRI);
5889 const RegisterBank &DstRB = *RBI.getRegBank(Reg: Dst, MRI, TRI);
5890 if (EltRB != DstRB)
5891 return false;
5892 if (any_of(Range: drop_begin(RangeOrContainer: I.operands(), N: 2), P: [&MRI](const MachineOperand &Op) {
5893 return !getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Op.getReg(), MRI);
5894 }))
5895 return false;
5896 unsigned SubReg;
5897 const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(Ty: EltTy, RB: EltRB);
5898 if (!EltRC)
5899 return false;
5900 const TargetRegisterClass *DstRC =
5901 getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst), RB: DstRB);
5902 if (!DstRC)
5903 return false;
5904 if (!getSubRegForClass(RC: EltRC, TRI, SubReg))
5905 return false;
5906 auto SubregToReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5907 .addUse(RegNo: EltReg)
5908 .addImm(Val: SubReg);
5909 I.eraseFromParent();
5910 constrainSelectedInstRegOperands(I&: *SubregToReg, TII, TRI, RBI);
5911 return RBI.constrainGenericRegister(Reg: Dst, RC: *DstRC, MRI);
5912}
5913
5914bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
5915 MachineRegisterInfo &MRI) {
5916 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5917 // Until we port more of the optimized selections, for now just use a vector
5918 // insert sequence.
5919 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5920 const LLT EltTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
5921 unsigned EltSize = EltTy.getSizeInBits();
5922
5923 if (tryOptConstantBuildVec(I, DstTy, MRI))
5924 return true;
5925 if (tryOptBuildVecToSubregToReg(I, MRI))
5926 return true;
5927
5928 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5929 return false; // Don't support all element types yet.
5930 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
5931
5932 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5933 MachineInstr *ScalarToVec =
5934 emitScalarToVector(EltSize: DstTy.getElementType().getSizeInBits(), DstRC,
5935 Scalar: I.getOperand(i: 1).getReg(), MIRBuilder&: MIB);
5936 if (!ScalarToVec)
5937 return false;
5938
5939 Register DstVec = ScalarToVec->getOperand(i: 0).getReg();
5940 unsigned DstSize = DstTy.getSizeInBits();
5941
5942 // Keep track of the last MI we inserted. Later on, we might be able to save
5943 // a copy using it.
5944 MachineInstr *PrevMI = ScalarToVec;
5945 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
5946 // Note that if we don't do a subregister copy, we can end up making an
5947 // extra register.
5948 Register OpReg = I.getOperand(i).getReg();
5949 // Do not emit inserts for undefs
5950 if (!getOpcodeDef<GImplicitDef>(Reg: OpReg, MRI)) {
5951 PrevMI = &*emitLaneInsert(DstReg: std::nullopt, SrcReg: DstVec, EltReg: OpReg, LaneIdx: i - 1, RB, MIRBuilder&: MIB);
5952 DstVec = PrevMI->getOperand(i: 0).getReg();
5953 }
5954 }
5955
5956 // If DstTy's size in bits is less than 128, then emit a subregister copy
5957 // from DstVec to the last register we've defined.
5958 if (DstSize < 128) {
5959 // Force this to be FPR using the destination vector.
5960 const TargetRegisterClass *RC =
5961 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
5962 if (!RC)
5963 return false;
5964 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5965 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
5966 return false;
5967 }
5968
5969 unsigned SubReg = 0;
5970 if (!getSubRegForClass(RC, TRI, SubReg))
5971 return false;
5972 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
5973 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize
5974 << "\n");
5975 return false;
5976 }
5977
5978 Register Reg = MRI.createVirtualRegister(RegClass: RC);
5979 Register DstReg = I.getOperand(i: 0).getReg();
5980
5981 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {}).addReg(RegNo: DstVec, Flags: {}, SubReg);
5982 MachineOperand &RegOp = I.getOperand(i: 1);
5983 RegOp.setReg(Reg);
5984 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
5985 } else {
5986 // We either have a vector with all elements (except the first one) undef or
5987 // at least one non-undef non-first element. In the first case, we need to
5988 // constrain the output register ourselves as we may have generated an
5989 // INSERT_SUBREG operation which is a generic operation for which the
5990 // output regclass cannot be automatically chosen.
5991 //
5992 // In the second case, there is no need to do this as it may generate an
5993 // instruction like INSvi32gpr where the regclass can be automatically
5994 // chosen.
5995 //
5996 // Also, we save a copy by re-using the destination register on the final
5997 // insert.
5998 PrevMI->getOperand(i: 0).setReg(I.getOperand(i: 0).getReg());
5999 constrainSelectedInstRegOperands(I&: *PrevMI, TII, TRI, RBI);
6000
6001 Register DstReg = PrevMI->getOperand(i: 0).getReg();
6002 if (PrevMI == ScalarToVec && DstReg.isVirtual()) {
6003 const TargetRegisterClass *RC =
6004 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
6005 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
6006 }
6007 }
6008
6009 I.eraseFromParent();
6010 return true;
6011}
6012
6013bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
6014 unsigned NumVecs,
6015 MachineInstr &I) {
6016 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6017 assert(Opc && "Expected an opcode?");
6018 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6019 auto &MRI = *MIB.getMRI();
6020 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6021 unsigned Size = Ty.getSizeInBits();
6022 assert((Size == 64 || Size == 128) &&
6023 "Destination must be 64 bits or 128 bits?");
6024 unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
6025 auto Ptr = I.getOperand(i: I.getNumOperands() - 1).getReg();
6026 assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?");
6027 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {Ptr});
6028 Load.cloneMemRefs(OtherMI: I);
6029 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6030 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6031 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6032 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: Idx)}, SrcOps: {})
6033 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6034 // Emit the subreg copies and immediately select them.
6035 // FIXME: We should refactor our copy code into an emitCopy helper and
6036 // clean up uses of this pattern elsewhere in the selector.
6037 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6038 }
6039 return true;
6040}
6041
6042bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6043 unsigned Opc, unsigned NumVecs, MachineInstr &I) {
6044 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6045 assert(Opc && "Expected an opcode?");
6046 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6047 auto &MRI = *MIB.getMRI();
6048 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6049 bool Narrow = Ty.getSizeInBits() == 64;
6050
6051 auto FirstSrcRegIt = I.operands_begin() + NumVecs + 1;
6052 SmallVector<Register, 4> Regs(NumVecs);
6053 std::transform(first: FirstSrcRegIt, last: FirstSrcRegIt + NumVecs, result: Regs.begin(),
6054 unary_op: [](auto MO) { return MO.getReg(); });
6055
6056 if (Narrow) {
6057 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6058 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6059 ->getOperand(i: 0)
6060 .getReg();
6061 });
6062 Ty = Ty.multiplyElements(Factor: 2);
6063 }
6064
6065 Register Tuple = createQTuple(Regs, MIB);
6066 auto LaneNo = getIConstantVRegVal(VReg: (FirstSrcRegIt + NumVecs)->getReg(), MRI);
6067 if (!LaneNo)
6068 return false;
6069
6070 Register Ptr = (FirstSrcRegIt + NumVecs + 1)->getReg();
6071 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {})
6072 .addReg(RegNo: Tuple)
6073 .addImm(Val: LaneNo->getZExtValue())
6074 .addReg(RegNo: Ptr);
6075 Load.cloneMemRefs(OtherMI: I);
6076 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6077 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6078 unsigned SubReg = AArch64::qsub0;
6079 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6080 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY,
6081 DstOps: {Narrow ? DstOp(&AArch64::FPR128RegClass)
6082 : DstOp(I.getOperand(i: Idx).getReg())},
6083 SrcOps: {})
6084 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6085 Register WideReg = Vec.getReg(Idx: 0);
6086 // Emit the subreg copies and immediately select them.
6087 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6088 if (Narrow &&
6089 !emitNarrowVector(DstReg: I.getOperand(i: Idx).getReg(), SrcReg: WideReg, MIB, MRI))
6090 return false;
6091 }
6092 return true;
6093}
6094
6095void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &I,
6096 unsigned NumVecs,
6097 unsigned Opc) {
6098 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6099 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6100 Register Ptr = I.getOperand(i: 1 + NumVecs).getReg();
6101
6102 SmallVector<Register, 2> Regs(NumVecs);
6103 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6104 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6105
6106 Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
6107 : createDTuple(Regs, MIB);
6108 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {Tuple, Ptr});
6109 Store.cloneMemRefs(OtherMI: I);
6110 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6111}
6112
6113bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6114 MachineInstr &I, unsigned NumVecs, unsigned Opc) {
6115 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6116 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6117 bool Narrow = Ty.getSizeInBits() == 64;
6118
6119 SmallVector<Register, 2> Regs(NumVecs);
6120 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6121 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6122
6123 if (Narrow)
6124 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6125 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6126 ->getOperand(i: 0)
6127 .getReg();
6128 });
6129
6130 Register Tuple = createQTuple(Regs, MIB);
6131
6132 auto LaneNo = getIConstantVRegVal(VReg: I.getOperand(i: 1 + NumVecs).getReg(), MRI);
6133 if (!LaneNo)
6134 return false;
6135 Register Ptr = I.getOperand(i: 1 + NumVecs + 1).getReg();
6136 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {})
6137 .addReg(RegNo: Tuple)
6138 .addImm(Val: LaneNo->getZExtValue())
6139 .addReg(RegNo: Ptr);
6140 Store.cloneMemRefs(OtherMI: I);
6141 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6142 return true;
6143}
6144
6145bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6146 MachineInstr &I, MachineRegisterInfo &MRI) {
6147 // Find the intrinsic ID.
6148 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6149
6150 const LLT S8 = LLT::scalar(SizeInBits: 8);
6151 const LLT S16 = LLT::scalar(SizeInBits: 16);
6152 const LLT S32 = LLT::scalar(SizeInBits: 32);
6153 const LLT S64 = LLT::scalar(SizeInBits: 64);
6154 const LLT P0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
6155 // Select the instruction.
6156 switch (IntrinID) {
6157 default:
6158 return false;
6159 case Intrinsic::aarch64_ldxp:
6160 case Intrinsic::aarch64_ldaxp: {
6161 auto NewI = MIB.buildInstr(
6162 Opc: IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6163 DstOps: {I.getOperand(i: 0).getReg(), I.getOperand(i: 1).getReg()},
6164 SrcOps: {I.getOperand(i: 3)});
6165 NewI.cloneMemRefs(OtherMI: I);
6166 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
6167 break;
6168 }
6169 case Intrinsic::aarch64_neon_ld1x2: {
6170 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6171 unsigned Opc = 0;
6172 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6173 Opc = AArch64::LD1Twov8b;
6174 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6175 Opc = AArch64::LD1Twov16b;
6176 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6177 Opc = AArch64::LD1Twov4h;
6178 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6179 Opc = AArch64::LD1Twov8h;
6180 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6181 Opc = AArch64::LD1Twov2s;
6182 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6183 Opc = AArch64::LD1Twov4s;
6184 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6185 Opc = AArch64::LD1Twov2d;
6186 else if (Ty == S64 || Ty == P0)
6187 Opc = AArch64::LD1Twov1d;
6188 else
6189 llvm_unreachable("Unexpected type for ld1x2!");
6190 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6191 break;
6192 }
6193 case Intrinsic::aarch64_neon_ld1x3: {
6194 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6195 unsigned Opc = 0;
6196 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6197 Opc = AArch64::LD1Threev8b;
6198 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6199 Opc = AArch64::LD1Threev16b;
6200 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6201 Opc = AArch64::LD1Threev4h;
6202 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6203 Opc = AArch64::LD1Threev8h;
6204 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6205 Opc = AArch64::LD1Threev2s;
6206 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6207 Opc = AArch64::LD1Threev4s;
6208 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6209 Opc = AArch64::LD1Threev2d;
6210 else if (Ty == S64 || Ty == P0)
6211 Opc = AArch64::LD1Threev1d;
6212 else
6213 llvm_unreachable("Unexpected type for ld1x3!");
6214 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6215 break;
6216 }
6217 case Intrinsic::aarch64_neon_ld1x4: {
6218 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6219 unsigned Opc = 0;
6220 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6221 Opc = AArch64::LD1Fourv8b;
6222 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6223 Opc = AArch64::LD1Fourv16b;
6224 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6225 Opc = AArch64::LD1Fourv4h;
6226 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6227 Opc = AArch64::LD1Fourv8h;
6228 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6229 Opc = AArch64::LD1Fourv2s;
6230 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6231 Opc = AArch64::LD1Fourv4s;
6232 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6233 Opc = AArch64::LD1Fourv2d;
6234 else if (Ty == S64 || Ty == P0)
6235 Opc = AArch64::LD1Fourv1d;
6236 else
6237 llvm_unreachable("Unexpected type for ld1x4!");
6238 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6239 break;
6240 }
6241 case Intrinsic::aarch64_neon_ld2: {
6242 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6243 unsigned Opc = 0;
6244 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6245 Opc = AArch64::LD2Twov8b;
6246 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6247 Opc = AArch64::LD2Twov16b;
6248 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6249 Opc = AArch64::LD2Twov4h;
6250 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6251 Opc = AArch64::LD2Twov8h;
6252 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6253 Opc = AArch64::LD2Twov2s;
6254 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6255 Opc = AArch64::LD2Twov4s;
6256 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6257 Opc = AArch64::LD2Twov2d;
6258 else if (Ty == S64 || Ty == P0)
6259 Opc = AArch64::LD1Twov1d;
6260 else
6261 llvm_unreachable("Unexpected type for ld2!");
6262 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6263 break;
6264 }
6265 case Intrinsic::aarch64_neon_ld2lane: {
6266 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6267 unsigned Opc;
6268 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6269 Opc = AArch64::LD2i8;
6270 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6271 Opc = AArch64::LD2i16;
6272 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6273 Opc = AArch64::LD2i32;
6274 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6275 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6276 Opc = AArch64::LD2i64;
6277 else
6278 llvm_unreachable("Unexpected type for st2lane!");
6279 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 2, I))
6280 return false;
6281 break;
6282 }
6283 case Intrinsic::aarch64_neon_ld2r: {
6284 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6285 unsigned Opc = 0;
6286 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6287 Opc = AArch64::LD2Rv8b;
6288 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6289 Opc = AArch64::LD2Rv16b;
6290 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6291 Opc = AArch64::LD2Rv4h;
6292 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6293 Opc = AArch64::LD2Rv8h;
6294 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6295 Opc = AArch64::LD2Rv2s;
6296 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6297 Opc = AArch64::LD2Rv4s;
6298 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6299 Opc = AArch64::LD2Rv2d;
6300 else if (Ty == S64 || Ty == P0)
6301 Opc = AArch64::LD2Rv1d;
6302 else
6303 llvm_unreachable("Unexpected type for ld2r!");
6304 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6305 break;
6306 }
6307 case Intrinsic::aarch64_neon_ld3: {
6308 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6309 unsigned Opc = 0;
6310 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6311 Opc = AArch64::LD3Threev8b;
6312 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6313 Opc = AArch64::LD3Threev16b;
6314 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6315 Opc = AArch64::LD3Threev4h;
6316 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6317 Opc = AArch64::LD3Threev8h;
6318 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6319 Opc = AArch64::LD3Threev2s;
6320 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6321 Opc = AArch64::LD3Threev4s;
6322 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6323 Opc = AArch64::LD3Threev2d;
6324 else if (Ty == S64 || Ty == P0)
6325 Opc = AArch64::LD1Threev1d;
6326 else
6327 llvm_unreachable("Unexpected type for ld3!");
6328 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6329 break;
6330 }
6331 case Intrinsic::aarch64_neon_ld3lane: {
6332 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6333 unsigned Opc;
6334 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6335 Opc = AArch64::LD3i8;
6336 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6337 Opc = AArch64::LD3i16;
6338 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6339 Opc = AArch64::LD3i32;
6340 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6341 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6342 Opc = AArch64::LD3i64;
6343 else
6344 llvm_unreachable("Unexpected type for st3lane!");
6345 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 3, I))
6346 return false;
6347 break;
6348 }
6349 case Intrinsic::aarch64_neon_ld3r: {
6350 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6351 unsigned Opc = 0;
6352 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6353 Opc = AArch64::LD3Rv8b;
6354 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6355 Opc = AArch64::LD3Rv16b;
6356 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6357 Opc = AArch64::LD3Rv4h;
6358 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6359 Opc = AArch64::LD3Rv8h;
6360 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6361 Opc = AArch64::LD3Rv2s;
6362 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6363 Opc = AArch64::LD3Rv4s;
6364 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6365 Opc = AArch64::LD3Rv2d;
6366 else if (Ty == S64 || Ty == P0)
6367 Opc = AArch64::LD3Rv1d;
6368 else
6369 llvm_unreachable("Unexpected type for ld3r!");
6370 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6371 break;
6372 }
6373 case Intrinsic::aarch64_neon_ld4: {
6374 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6375 unsigned Opc = 0;
6376 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6377 Opc = AArch64::LD4Fourv8b;
6378 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6379 Opc = AArch64::LD4Fourv16b;
6380 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6381 Opc = AArch64::LD4Fourv4h;
6382 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6383 Opc = AArch64::LD4Fourv8h;
6384 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6385 Opc = AArch64::LD4Fourv2s;
6386 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6387 Opc = AArch64::LD4Fourv4s;
6388 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6389 Opc = AArch64::LD4Fourv2d;
6390 else if (Ty == S64 || Ty == P0)
6391 Opc = AArch64::LD1Fourv1d;
6392 else
6393 llvm_unreachable("Unexpected type for ld4!");
6394 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6395 break;
6396 }
6397 case Intrinsic::aarch64_neon_ld4lane: {
6398 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6399 unsigned Opc;
6400 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6401 Opc = AArch64::LD4i8;
6402 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6403 Opc = AArch64::LD4i16;
6404 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6405 Opc = AArch64::LD4i32;
6406 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6407 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6408 Opc = AArch64::LD4i64;
6409 else
6410 llvm_unreachable("Unexpected type for st4lane!");
6411 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 4, I))
6412 return false;
6413 break;
6414 }
6415 case Intrinsic::aarch64_neon_ld4r: {
6416 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6417 unsigned Opc = 0;
6418 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6419 Opc = AArch64::LD4Rv8b;
6420 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6421 Opc = AArch64::LD4Rv16b;
6422 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6423 Opc = AArch64::LD4Rv4h;
6424 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6425 Opc = AArch64::LD4Rv8h;
6426 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6427 Opc = AArch64::LD4Rv2s;
6428 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6429 Opc = AArch64::LD4Rv4s;
6430 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6431 Opc = AArch64::LD4Rv2d;
6432 else if (Ty == S64 || Ty == P0)
6433 Opc = AArch64::LD4Rv1d;
6434 else
6435 llvm_unreachable("Unexpected type for ld4r!");
6436 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6437 break;
6438 }
6439 case Intrinsic::aarch64_neon_st1x2: {
6440 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6441 unsigned Opc;
6442 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6443 Opc = AArch64::ST1Twov8b;
6444 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6445 Opc = AArch64::ST1Twov16b;
6446 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6447 Opc = AArch64::ST1Twov4h;
6448 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6449 Opc = AArch64::ST1Twov8h;
6450 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6451 Opc = AArch64::ST1Twov2s;
6452 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6453 Opc = AArch64::ST1Twov4s;
6454 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6455 Opc = AArch64::ST1Twov2d;
6456 else if (Ty == S64 || Ty == P0)
6457 Opc = AArch64::ST1Twov1d;
6458 else
6459 llvm_unreachable("Unexpected type for st1x2!");
6460 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6461 break;
6462 }
6463 case Intrinsic::aarch64_neon_st1x3: {
6464 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6465 unsigned Opc;
6466 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6467 Opc = AArch64::ST1Threev8b;
6468 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6469 Opc = AArch64::ST1Threev16b;
6470 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6471 Opc = AArch64::ST1Threev4h;
6472 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6473 Opc = AArch64::ST1Threev8h;
6474 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6475 Opc = AArch64::ST1Threev2s;
6476 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6477 Opc = AArch64::ST1Threev4s;
6478 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6479 Opc = AArch64::ST1Threev2d;
6480 else if (Ty == S64 || Ty == P0)
6481 Opc = AArch64::ST1Threev1d;
6482 else
6483 llvm_unreachable("Unexpected type for st1x3!");
6484 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6485 break;
6486 }
6487 case Intrinsic::aarch64_neon_st1x4: {
6488 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6489 unsigned Opc;
6490 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6491 Opc = AArch64::ST1Fourv8b;
6492 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6493 Opc = AArch64::ST1Fourv16b;
6494 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6495 Opc = AArch64::ST1Fourv4h;
6496 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6497 Opc = AArch64::ST1Fourv8h;
6498 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6499 Opc = AArch64::ST1Fourv2s;
6500 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6501 Opc = AArch64::ST1Fourv4s;
6502 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6503 Opc = AArch64::ST1Fourv2d;
6504 else if (Ty == S64 || Ty == P0)
6505 Opc = AArch64::ST1Fourv1d;
6506 else
6507 llvm_unreachable("Unexpected type for st1x4!");
6508 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6509 break;
6510 }
6511 case Intrinsic::aarch64_neon_st2: {
6512 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6513 unsigned Opc;
6514 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6515 Opc = AArch64::ST2Twov8b;
6516 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6517 Opc = AArch64::ST2Twov16b;
6518 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6519 Opc = AArch64::ST2Twov4h;
6520 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6521 Opc = AArch64::ST2Twov8h;
6522 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6523 Opc = AArch64::ST2Twov2s;
6524 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6525 Opc = AArch64::ST2Twov4s;
6526 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6527 Opc = AArch64::ST2Twov2d;
6528 else if (Ty == S64 || Ty == P0)
6529 Opc = AArch64::ST1Twov1d;
6530 else
6531 llvm_unreachable("Unexpected type for st2!");
6532 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6533 break;
6534 }
6535 case Intrinsic::aarch64_neon_st3: {
6536 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6537 unsigned Opc;
6538 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6539 Opc = AArch64::ST3Threev8b;
6540 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6541 Opc = AArch64::ST3Threev16b;
6542 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6543 Opc = AArch64::ST3Threev4h;
6544 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6545 Opc = AArch64::ST3Threev8h;
6546 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6547 Opc = AArch64::ST3Threev2s;
6548 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6549 Opc = AArch64::ST3Threev4s;
6550 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6551 Opc = AArch64::ST3Threev2d;
6552 else if (Ty == S64 || Ty == P0)
6553 Opc = AArch64::ST1Threev1d;
6554 else
6555 llvm_unreachable("Unexpected type for st3!");
6556 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6557 break;
6558 }
6559 case Intrinsic::aarch64_neon_st4: {
6560 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6561 unsigned Opc;
6562 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6563 Opc = AArch64::ST4Fourv8b;
6564 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6565 Opc = AArch64::ST4Fourv16b;
6566 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6567 Opc = AArch64::ST4Fourv4h;
6568 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6569 Opc = AArch64::ST4Fourv8h;
6570 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6571 Opc = AArch64::ST4Fourv2s;
6572 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6573 Opc = AArch64::ST4Fourv4s;
6574 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6575 Opc = AArch64::ST4Fourv2d;
6576 else if (Ty == S64 || Ty == P0)
6577 Opc = AArch64::ST1Fourv1d;
6578 else
6579 llvm_unreachable("Unexpected type for st4!");
6580 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6581 break;
6582 }
6583 case Intrinsic::aarch64_neon_st2lane: {
6584 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6585 unsigned Opc;
6586 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6587 Opc = AArch64::ST2i8;
6588 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6589 Opc = AArch64::ST2i16;
6590 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6591 Opc = AArch64::ST2i32;
6592 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6593 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6594 Opc = AArch64::ST2i64;
6595 else
6596 llvm_unreachable("Unexpected type for st2lane!");
6597 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 2, Opc))
6598 return false;
6599 break;
6600 }
6601 case Intrinsic::aarch64_neon_st3lane: {
6602 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6603 unsigned Opc;
6604 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6605 Opc = AArch64::ST3i8;
6606 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6607 Opc = AArch64::ST3i16;
6608 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6609 Opc = AArch64::ST3i32;
6610 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6611 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6612 Opc = AArch64::ST3i64;
6613 else
6614 llvm_unreachable("Unexpected type for st3lane!");
6615 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 3, Opc))
6616 return false;
6617 break;
6618 }
6619 case Intrinsic::aarch64_neon_st4lane: {
6620 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6621 unsigned Opc;
6622 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6623 Opc = AArch64::ST4i8;
6624 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6625 Opc = AArch64::ST4i16;
6626 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6627 Opc = AArch64::ST4i32;
6628 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6629 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6630 Opc = AArch64::ST4i64;
6631 else
6632 llvm_unreachable("Unexpected type for st4lane!");
6633 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 4, Opc))
6634 return false;
6635 break;
6636 }
6637 case Intrinsic::aarch64_mops_memset_tag: {
6638 // Transform
6639 // %dst:gpr(p0) = \
6640 // G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
6641 // \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
6642 // where %dst is updated, into
6643 // %Rd:GPR64common, %Rn:GPR64) = \
6644 // MOPSMemorySetTaggingPseudo \
6645 // %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
6646 // where Rd and Rn are tied.
6647 // It is expected that %val has been extended to s64 in legalization.
6648 // Note that the order of the size/value operands are swapped.
6649
6650 Register DstDef = I.getOperand(i: 0).getReg();
6651 // I.getOperand(1) is the intrinsic function
6652 Register DstUse = I.getOperand(i: 2).getReg();
6653 Register ValUse = I.getOperand(i: 3).getReg();
6654 Register SizeUse = I.getOperand(i: 4).getReg();
6655
6656 // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
6657 // Therefore an additional virtual register is required for the updated size
6658 // operand. This value is not accessible via the semantics of the intrinsic.
6659 Register SizeDef = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
6660
6661 auto Memset = MIB.buildInstr(Opc: AArch64::MOPSMemorySetTaggingPseudo,
6662 DstOps: {DstDef, SizeDef}, SrcOps: {DstUse, SizeUse, ValUse});
6663 Memset.cloneMemRefs(OtherMI: I);
6664 constrainSelectedInstRegOperands(I&: *Memset, TII, TRI, RBI);
6665 break;
6666 }
6667 case Intrinsic::ptrauth_resign_load_relative: {
6668 Register DstReg = I.getOperand(i: 0).getReg();
6669 Register ValReg = I.getOperand(i: 2).getReg();
6670 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6671 Register AUTDisc = I.getOperand(i: 4).getReg();
6672 uint64_t PACKey = I.getOperand(i: 5).getImm();
6673 Register PACDisc = I.getOperand(i: 6).getReg();
6674 int64_t Addend = I.getOperand(i: 7).getImm();
6675
6676 Register AUTAddrDisc = AUTDisc;
6677 uint16_t AUTConstDiscC = 0;
6678 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6679 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6680
6681 Register PACAddrDisc = PACDisc;
6682 uint16_t PACConstDiscC = 0;
6683 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6684 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6685
6686 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6687
6688 MIB.buildInstr(Opcode: AArch64::AUTRELLOADPAC)
6689 .addImm(Val: AUTKey)
6690 .addImm(Val: AUTConstDiscC)
6691 .addUse(RegNo: AUTAddrDisc)
6692 .addImm(Val: PACKey)
6693 .addImm(Val: PACConstDiscC)
6694 .addUse(RegNo: PACAddrDisc)
6695 .addImm(Val: Addend)
6696 .constrainAllUses(TII, TRI, RBI);
6697 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6698
6699 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6700 I.eraseFromParent();
6701 return true;
6702 }
6703 }
6704
6705 I.eraseFromParent();
6706 return true;
6707}
6708
6709bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
6710 MachineRegisterInfo &MRI) {
6711 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6712
6713 switch (IntrinID) {
6714 default:
6715 break;
6716 case Intrinsic::ptrauth_resign: {
6717 Register DstReg = I.getOperand(i: 0).getReg();
6718 Register ValReg = I.getOperand(i: 2).getReg();
6719 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6720 Register AUTDisc = I.getOperand(i: 4).getReg();
6721 uint64_t PACKey = I.getOperand(i: 5).getImm();
6722 Register PACDisc = I.getOperand(i: 6).getReg();
6723
6724 Register AUTAddrDisc = AUTDisc;
6725 uint16_t AUTConstDiscC = 0;
6726 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6727 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6728
6729 Register PACAddrDisc = PACDisc;
6730 uint16_t PACConstDiscC = 0;
6731 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6732 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6733
6734 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6735 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
6736 MIB.buildInstr(Opcode: AArch64::AUTPAC)
6737 .addImm(Val: AUTKey)
6738 .addImm(Val: AUTConstDiscC)
6739 .addUse(RegNo: AUTAddrDisc)
6740 .addImm(Val: PACKey)
6741 .addImm(Val: PACConstDiscC)
6742 .addUse(RegNo: PACAddrDisc)
6743 .constrainAllUses(TII, TRI, RBI);
6744 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6745
6746 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6747 I.eraseFromParent();
6748 return true;
6749 }
6750 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6751 Register DstReg = I.getOperand(i: 0).getReg();
6752 Register ValReg = I.getOperand(i: 2).getReg();
6753 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6754 Register AUTDisc = I.getOperand(i: 4).getReg();
6755 Register AUTPC = I.getOperand(i: 5).getReg();
6756 uint64_t PACKey = I.getOperand(i: 6).getImm();
6757 Register PACDisc = I.getOperand(i: 7).getReg();
6758
6759 assert((AUTKey == AArch64PACKey::IA || AUTKey == AArch64PACKey::IB) &&
6760 "auth_with_pc_and_resign only supports IA and IB keys");
6761
6762 uint16_t PACConstDiscC = 0;
6763 Register PACAddrDisc;
6764 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6765 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6766
6767 if (PACAddrDisc == AArch64::NoRegister)
6768 PACAddrDisc = AArch64::XZR;
6769
6770 MIB.buildCopy(Res: {AArch64::X17}, Op: {ValReg});
6771 MIB.buildCopy(Res: {AArch64::X16}, Op: {AUTDisc});
6772 MIB.buildCopy(Res: {AArch64::X15}, Op: {AUTPC});
6773
6774 MIB.buildInstr(Opcode: AArch64::AUTPCPAC)
6775 .addImm(Val: AUTKey)
6776 .addImm(Val: PACKey)
6777 .addImm(Val: PACConstDiscC)
6778 .addUse(RegNo: PACAddrDisc)
6779 .constrainAllUses(TII, TRI, RBI);
6780
6781 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X17));
6782 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6783 I.eraseFromParent();
6784 return true;
6785 }
6786 case Intrinsic::ptrauth_auth: {
6787 Register DstReg = I.getOperand(i: 0).getReg();
6788 Register ValReg = I.getOperand(i: 2).getReg();
6789 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6790 Register AUTDisc = I.getOperand(i: 4).getReg();
6791
6792 Register AUTAddrDisc = AUTDisc;
6793 uint16_t AUTConstDiscC = 0;
6794 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6795 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6796
6797 if (STI.isX16X17Safer()) {
6798 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6799 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
6800 MIB.buildInstr(Opcode: AArch64::AUTx16x17)
6801 .addImm(Val: AUTKey)
6802 .addImm(Val: AUTConstDiscC)
6803 .addUse(RegNo: AUTAddrDisc)
6804 .constrainAllUses(TII, TRI, RBI);
6805 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6806 } else {
6807 Register ScratchReg =
6808 MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
6809 MIB.buildInstr(Opcode: AArch64::AUTxMxN)
6810 .addDef(RegNo: DstReg)
6811 .addDef(RegNo: ScratchReg)
6812 .addUse(RegNo: ValReg)
6813 .addImm(Val: AUTKey)
6814 .addImm(Val: AUTConstDiscC)
6815 .addUse(RegNo: AUTAddrDisc)
6816 .constrainAllUses(TII, TRI, RBI);
6817 }
6818
6819 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6820 I.eraseFromParent();
6821 return true;
6822 }
6823 case Intrinsic::frameaddress:
6824 case Intrinsic::returnaddress: {
6825 MachineFunction &MF = *I.getParent()->getParent();
6826 MachineFrameInfo &MFI = MF.getFrameInfo();
6827
6828 unsigned Depth = I.getOperand(i: 2).getImm();
6829 Register DstReg = I.getOperand(i: 0).getReg();
6830 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6831
6832 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6833 if (!MFReturnAddr) {
6834 // Insert the copy from LR/X30 into the entry block, before it can be
6835 // clobbered by anything.
6836 MFI.setReturnAddressIsTaken(true);
6837 MFReturnAddr = getFunctionLiveInPhysReg(
6838 MF, TII, PhysReg: AArch64::LR, RC: AArch64::GPR64RegClass, DL: I.getDebugLoc());
6839 }
6840
6841 if (STI.hasPAuth()) {
6842 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {MFReturnAddr});
6843 } else {
6844 MIB.buildCopy(Res: {Register(AArch64::LR)}, Op: {MFReturnAddr});
6845 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6846 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6847 }
6848
6849 I.eraseFromParent();
6850 return true;
6851 }
6852
6853 MFI.setFrameAddressIsTaken(true);
6854 Register FrameAddr(AArch64::FP);
6855 while (Depth--) {
6856 Register NextFrame = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
6857 auto Ldr =
6858 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {NextFrame}, SrcOps: {FrameAddr}).addImm(Val: 0);
6859 constrainSelectedInstRegOperands(I&: *Ldr, TII, TRI, RBI);
6860 FrameAddr = NextFrame;
6861 }
6862
6863 if (IntrinID == Intrinsic::frameaddress)
6864 MIB.buildCopy(Res: {DstReg}, Op: {FrameAddr});
6865 else {
6866 MFI.setReturnAddressIsTaken(true);
6867
6868 if (STI.hasPAuth()) {
6869 Register TmpReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
6870 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {TmpReg}, SrcOps: {FrameAddr}).addImm(Val: 1);
6871 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {TmpReg});
6872 } else {
6873 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {Register(AArch64::LR)}, SrcOps: {FrameAddr})
6874 .addImm(Val: 1);
6875 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6876 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6877 }
6878 }
6879
6880 I.eraseFromParent();
6881 return true;
6882 }
6883 case Intrinsic::aarch64_neon_tbl2:
6884 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBLv8i8Two, Opc2: AArch64::TBLv16i8Two, isExt: false);
6885 return true;
6886 case Intrinsic::aarch64_neon_tbl3:
6887 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBLv8i8Three, Opc2: AArch64::TBLv16i8Three,
6888 isExt: false);
6889 return true;
6890 case Intrinsic::aarch64_neon_tbl4:
6891 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBLv8i8Four, Opc2: AArch64::TBLv16i8Four, isExt: false);
6892 return true;
6893 case Intrinsic::aarch64_neon_tbx2:
6894 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBXv8i8Two, Opc2: AArch64::TBXv16i8Two, isExt: true);
6895 return true;
6896 case Intrinsic::aarch64_neon_tbx3:
6897 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBXv8i8Three, Opc2: AArch64::TBXv16i8Three, isExt: true);
6898 return true;
6899 case Intrinsic::aarch64_neon_tbx4:
6900 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBXv8i8Four, Opc2: AArch64::TBXv16i8Four, isExt: true);
6901 return true;
6902 case Intrinsic::swift_async_context_addr:
6903 auto Sub = MIB.buildInstr(Opc: AArch64::SUBXri, DstOps: {I.getOperand(i: 0).getReg()},
6904 SrcOps: {Register(AArch64::FP)})
6905 .addImm(Val: 8)
6906 .addImm(Val: 0);
6907 constrainSelectedInstRegOperands(I&: *Sub, TII, TRI, RBI);
6908
6909 MF->getFrameInfo().setFrameAddressIsTaken(true);
6910 MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6911 I.eraseFromParent();
6912 return true;
6913 }
6914 return false;
6915}
6916
6917// G_PTRAUTH_GLOBAL_VALUE lowering
6918//
6919// We have 3 lowering alternatives to choose from:
6920// - MOVaddrPAC: similar to MOVaddr, with added PAC.
6921// If the GV doesn't need a GOT load (i.e., is locally defined)
6922// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
6923//
6924// - LOADgotPAC: similar to LOADgot, with added PAC.
6925// If the GV needs a GOT load, materialize the pointer using the usual
6926// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
6927// section is assumed to be read-only (for example, via relro mechanism). See
6928// LowerMOVaddrPAC.
6929//
6930// - LOADauthptrstatic: similar to LOADgot, but use a
6931// special stub slot instead of a GOT slot.
6932// Load a signed pointer for symbol 'sym' from a stub slot named
6933// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
6934// resolving. This usually lowers to adrp+ldr, but also emits an entry into
6935// .data with an
6936// @AUTH relocation. See LowerLOADauthptrstatic.
6937//
6938// All 3 are pseudos that are expand late to longer sequences: this lets us
6939// provide integrity guarantees on the to-be-signed intermediate values.
6940//
6941// LOADauthptrstatic is undesirable because it requires a large section filled
6942// with often similarly-signed pointers, making it a good harvesting target.
6943// Thus, it's only used for ptrauth references to extern_weak to avoid null
6944// checks.
6945
6946bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6947 MachineInstr &I, MachineRegisterInfo &MRI) const {
6948 Register DefReg = I.getOperand(i: 0).getReg();
6949 Register Addr = I.getOperand(i: 1).getReg();
6950 uint64_t Key = I.getOperand(i: 2).getImm();
6951 Register AddrDisc = I.getOperand(i: 3).getReg();
6952 uint64_t Disc = I.getOperand(i: 4).getImm();
6953 int64_t Offset = 0;
6954
6955 if (Key > AArch64PACKey::LAST)
6956 report_fatal_error(reason: "key in ptrauth global out of range [0, " +
6957 Twine((int)AArch64PACKey::LAST) + "]");
6958
6959 // Blend only works if the integer discriminator is 16-bit wide.
6960 if (!isUInt<16>(x: Disc))
6961 report_fatal_error(
6962 reason: "constant discriminator in ptrauth global out of range [0, 0xffff]");
6963
6964 // Choosing between 3 lowering alternatives is target-specific.
6965 if (!STI.isTargetELF() && !STI.isTargetMachO())
6966 report_fatal_error(reason: "ptrauth global lowering only supported on MachO/ELF");
6967
6968 if (!MRI.hasOneDef(RegNo: Addr))
6969 return false;
6970
6971 // First match any offset we take from the real global.
6972 const MachineInstr *DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6973 if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6974 Register OffsetReg = DefMI->getOperand(i: 2).getReg();
6975 if (!MRI.hasOneDef(RegNo: OffsetReg))
6976 return false;
6977 const MachineInstr &OffsetMI = *MRI.def_instr_begin(RegNo: OffsetReg);
6978 if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
6979 return false;
6980
6981 Addr = DefMI->getOperand(i: 1).getReg();
6982 if (!MRI.hasOneDef(RegNo: Addr))
6983 return false;
6984
6985 DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6986 Offset = OffsetMI.getOperand(i: 1).getCImm()->getSExtValue();
6987 }
6988
6989 // We should be left with a genuine unauthenticated GlobalValue.
6990 const GlobalValue *GV;
6991 if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
6992 GV = DefMI->getOperand(i: 1).getGlobal();
6993 Offset += DefMI->getOperand(i: 1).getOffset();
6994 } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
6995 GV = DefMI->getOperand(i: 2).getGlobal();
6996 Offset += DefMI->getOperand(i: 2).getOffset();
6997 } else {
6998 return false;
6999 }
7000
7001 MachineIRBuilder MIB(I);
7002
7003 // Classify the reference to determine whether it needs a GOT load.
7004 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
7005 const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
7006 assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
7007 "unsupported non-GOT op flags on ptrauth global reference");
7008 assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
7009 "unsupported non-GOT reference to weak ptrauth global");
7010
7011 std::optional<APInt> AddrDiscVal = getIConstantVRegVal(VReg: AddrDisc, MRI);
7012 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
7013
7014 // Non-extern_weak:
7015 // - No GOT load needed -> MOVaddrPAC
7016 // - GOT load for non-extern_weak -> LOADgotPAC
7017 // Note that we disallow extern_weak refs to avoid null checks later.
7018 if (!GV->hasExternalWeakLinkage()) {
7019 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X16}, SrcOps: {});
7020 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
7021 MIB.buildInstr(Opcode: NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
7022 .addGlobalAddress(GV, Offset)
7023 .addImm(Val: Key)
7024 .addReg(RegNo: HasAddrDisc ? AddrDisc : AArch64::XZR)
7025 .addImm(Val: Disc)
7026 .constrainAllUses(TII, TRI, RBI);
7027 MIB.buildCopy(Res: DefReg, Op: Register(AArch64::X16));
7028 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7029 I.eraseFromParent();
7030 return true;
7031 }
7032
7033 // extern_weak -> LOADauthptrstatic
7034
7035 // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
7036 // offset alone as a pointer if the symbol wasn't available, which would
7037 // probably break null checks in users. Ptrauth complicates things further:
7038 // error out.
7039 if (Offset != 0)
7040 report_fatal_error(
7041 reason: "unsupported non-zero offset in weak ptrauth global reference");
7042
7043 if (HasAddrDisc)
7044 report_fatal_error(reason: "unsupported weak addr-div ptrauth global");
7045
7046 MIB.buildInstr(Opc: AArch64::LOADauthptrstatic, DstOps: {DefReg}, SrcOps: {})
7047 .addGlobalAddress(GV, Offset)
7048 .addImm(Val: Key)
7049 .addImm(Val: Disc);
7050 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7051
7052 I.eraseFromParent();
7053 return true;
7054}
7055
7056void AArch64InstructionSelector::SelectTable(MachineInstr &I,
7057 MachineRegisterInfo &MRI,
7058 unsigned NumVec, unsigned Opc1,
7059 unsigned Opc2, bool isExt) {
7060 Register DstReg = I.getOperand(i: 0).getReg();
7061 unsigned Opc = MRI.getType(Reg: DstReg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) ? Opc1 : Opc2;
7062
7063 // Create the REG_SEQUENCE
7064 SmallVector<Register, 4> Regs;
7065 for (unsigned i = 0; i < NumVec; i++)
7066 Regs.push_back(Elt: I.getOperand(i: i + 2 + isExt).getReg());
7067 Register RegSeq = createQTuple(Regs, MIB);
7068
7069 Register IdxReg = I.getOperand(i: 2 + NumVec + isExt).getReg();
7070 MachineInstrBuilder Instr;
7071 if (isExt) {
7072 Register Reg = I.getOperand(i: 2).getReg();
7073 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Reg, RegSeq, IdxReg});
7074 } else
7075 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {RegSeq, IdxReg});
7076 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
7077 I.eraseFromParent();
7078}
7079
7080InstructionSelector::ComplexRendererFns
7081AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
7082 auto MaybeImmed = getImmedFromMO(Root);
7083 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7084 return std::nullopt;
7085 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7086 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7087}
7088
7089InstructionSelector::ComplexRendererFns
7090AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
7091 auto MaybeImmed = getImmedFromMO(Root);
7092 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7093 return std::nullopt;
7094 uint64_t Enc = 31 - *MaybeImmed;
7095 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7096}
7097
7098InstructionSelector::ComplexRendererFns
7099AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
7100 auto MaybeImmed = getImmedFromMO(Root);
7101 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7102 return std::nullopt;
7103 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7104 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7105}
7106
7107InstructionSelector::ComplexRendererFns
7108AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
7109 auto MaybeImmed = getImmedFromMO(Root);
7110 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7111 return std::nullopt;
7112 uint64_t Enc = 63 - *MaybeImmed;
7113 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7114}
7115
7116/// Helper to select an immediate value that can be represented as a 12-bit
7117/// value shifted left by either 0 or 12. If it is possible to do so, return
7118/// the immediate and shift value. If not, return std::nullopt.
7119///
7120/// Used by selectArithImmed and selectNegArithImmed.
7121InstructionSelector::ComplexRendererFns
7122AArch64InstructionSelector::select12BitValueWithLeftShift(
7123 uint64_t Immed) const {
7124 unsigned ShiftAmt;
7125 if (Immed >> 12 == 0) {
7126 ShiftAmt = 0;
7127 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7128 ShiftAmt = 12;
7129 Immed = Immed >> 12;
7130 } else
7131 return std::nullopt;
7132
7133 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
7134 return {{
7135 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Immed); },
7136 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShVal); },
7137 }};
7138}
7139
7140/// SelectArithImmed - Select an immediate value that can be represented as
7141/// a 12-bit value shifted left by either 0 or 12. If so, return true with
7142/// Val set to the 12-bit value and Shift set to the shifter operand.
7143InstructionSelector::ComplexRendererFns
7144AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
7145 // This function is called from the addsub_shifted_imm ComplexPattern,
7146 // which lists [imm] as the list of opcode it's interested in, however
7147 // we still need to check whether the operand is actually an immediate
7148 // here because the ComplexPattern opcode list is only used in
7149 // root-level opcode matching.
7150 auto MaybeImmed = getImmedFromMO(Root);
7151 if (MaybeImmed == std::nullopt)
7152 return std::nullopt;
7153 return select12BitValueWithLeftShift(Immed: *MaybeImmed);
7154}
7155
7156/// SelectNegArithImmed - As above, but negates the value before trying to
7157/// select it.
7158InstructionSelector::ComplexRendererFns
7159AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
7160 // We need a register here, because we need to know if we have a 64 or 32
7161 // bit immediate.
7162 if (!Root.isReg())
7163 return std::nullopt;
7164 auto MaybeImmed = getImmedFromMO(Root);
7165 if (MaybeImmed == std::nullopt)
7166 return std::nullopt;
7167 uint64_t Immed = *MaybeImmed;
7168
7169 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
7170 // have the opposite effect on the C flag, so this pattern mustn't match under
7171 // those circumstances.
7172 if (Immed == 0)
7173 return std::nullopt;
7174
7175 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
7176 // the root.
7177 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7178 if (MRI.getType(Reg: Root.getReg()).getSizeInBits() == 32)
7179 Immed = ~((uint32_t)Immed) + 1;
7180 else
7181 Immed = ~Immed + 1ULL;
7182
7183 if (Immed & 0xFFFFFFFFFF000000ULL)
7184 return std::nullopt;
7185
7186 Immed &= 0xFFFFFFULL;
7187 return select12BitValueWithLeftShift(Immed);
7188}
7189
7190/// Checks if we are sure that folding MI into load/store addressing mode is
7191/// beneficial or not.
7192///
7193/// Returns:
7194/// - true if folding MI would be beneficial.
7195/// - false if folding MI would be bad.
7196/// - std::nullopt if it is not sure whether folding MI is beneficial.
7197///
7198/// \p MI can be the offset operand of G_PTR_ADD, e.g. G_SHL in the example:
7199///
7200/// %13:gpr(s64) = G_CONSTANT i64 1
7201/// %8:gpr(s64) = G_SHL %6, %13(s64)
7202/// %9:gpr(p0) = G_PTR_ADD %0, %8(s64)
7203/// %12:gpr(s32) = G_LOAD %9(p0) :: (load (s16))
7204std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7205 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
7206 if (MI.getOpcode() == AArch64::G_SHL) {
7207 // Address operands with shifts are free, except for running on subtargets
7208 // with AddrLSLSlow14.
7209 if (const auto ValAndVeg = getIConstantVRegValWithLookThrough(
7210 VReg: MI.getOperand(i: 2).getReg(), MRI)) {
7211 const APInt ShiftVal = ValAndVeg->Value;
7212
7213 // Don't fold if we know this will be slow.
7214 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7215 }
7216 }
7217 return std::nullopt;
7218}
7219
7220/// Return true if it is worth folding MI into an extended register. That is,
7221/// if it's safe to pull it into the addressing mode of a load or store as a
7222/// shift.
7223/// \p IsAddrOperand whether the def of MI is used as an address operand
7224/// (e.g. feeding into an LDR/STR).
7225bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7226 const MachineInstr &MI, const MachineRegisterInfo &MRI,
7227 bool IsAddrOperand) const {
7228
7229 // Always fold if there is one use, or if we're optimizing for size.
7230 Register DefReg = MI.getOperand(i: 0).getReg();
7231 if (MRI.hasOneNonDBGUse(RegNo: DefReg) ||
7232 MI.getParent()->getParent()->getFunction().hasOptSize())
7233 return true;
7234
7235 if (IsAddrOperand) {
7236 // If we are already sure that folding MI is good or bad, return the result.
7237 if (const auto Worth = isWorthFoldingIntoAddrMode(MI, MRI))
7238 return *Worth;
7239
7240 // Fold G_PTR_ADD if its offset operand can be folded
7241 if (MI.getOpcode() == AArch64::G_PTR_ADD) {
7242 MachineInstr *OffsetInst =
7243 getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
7244
7245 // Note, we already know G_PTR_ADD is used by at least two instructions.
7246 // If we are also sure about whether folding is beneficial or not,
7247 // return the result.
7248 if (const auto Worth = isWorthFoldingIntoAddrMode(MI: *OffsetInst, MRI))
7249 return *Worth;
7250 }
7251 }
7252
7253 // FIXME: Consider checking HasALULSLFast as appropriate.
7254
7255 // We have a fastpath, so folding a shift in and potentially computing it
7256 // many times may be beneficial. Check if this is only used in memory ops.
7257 // If it is, then we should fold.
7258 return all_of(Range: MRI.use_nodbg_instructions(Reg: DefReg),
7259 P: [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7260}
7261
7262InstructionSelector::ComplexRendererFns
7263AArch64InstructionSelector::selectExtendedSHL(
7264 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
7265 unsigned SizeInBytes, bool WantsExt) const {
7266 assert(Base.isReg() && "Expected base to be a register operand");
7267 assert(Offset.isReg() && "Expected offset to be a register operand");
7268
7269 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7270 MachineInstr *OffsetInst = MRI.getVRegDef(Reg: Offset.getReg());
7271
7272 unsigned OffsetOpc = OffsetInst->getOpcode();
7273 bool LookedThroughZExt = false;
7274 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7275 // Try to look through a ZEXT.
7276 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7277 return std::nullopt;
7278
7279 OffsetInst = MRI.getVRegDef(Reg: OffsetInst->getOperand(i: 1).getReg());
7280 OffsetOpc = OffsetInst->getOpcode();
7281 LookedThroughZExt = true;
7282
7283 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7284 return std::nullopt;
7285 }
7286 // Make sure that the memory op is a valid size.
7287 int64_t LegalShiftVal = Log2_32(Value: SizeInBytes);
7288 if (LegalShiftVal == 0)
7289 return std::nullopt;
7290 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7291 return std::nullopt;
7292
7293 // Now, try to find the specific G_CONSTANT. Start by assuming that the
7294 // register we will offset is the LHS, and the register containing the
7295 // constant is the RHS.
7296 Register OffsetReg = OffsetInst->getOperand(i: 1).getReg();
7297 Register ConstantReg = OffsetInst->getOperand(i: 2).getReg();
7298 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7299 if (!ValAndVReg) {
7300 // We didn't get a constant on the RHS. If the opcode is a shift, then
7301 // we're done.
7302 if (OffsetOpc == TargetOpcode::G_SHL)
7303 return std::nullopt;
7304
7305 // If we have a G_MUL, we can use either register. Try looking at the RHS.
7306 std::swap(a&: OffsetReg, b&: ConstantReg);
7307 ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7308 if (!ValAndVReg)
7309 return std::nullopt;
7310 }
7311
7312 // The value must fit into 3 bits, and must be positive. Make sure that is
7313 // true.
7314 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7315
7316 // Since we're going to pull this into a shift, the constant value must be
7317 // a power of 2. If we got a multiply, then we need to check this.
7318 if (OffsetOpc == TargetOpcode::G_MUL) {
7319 if (!llvm::has_single_bit<uint32_t>(Value: ImmVal))
7320 return std::nullopt;
7321
7322 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
7323 ImmVal = Log2_32(Value: ImmVal);
7324 }
7325
7326 if ((ImmVal & 0x7) != ImmVal)
7327 return std::nullopt;
7328
7329 // We are only allowed to shift by LegalShiftVal. This shift value is built
7330 // into the instruction, so we can't just use whatever we want.
7331 if (ImmVal != LegalShiftVal)
7332 return std::nullopt;
7333
7334 unsigned SignExtend = 0;
7335 if (WantsExt) {
7336 // Check if the offset is defined by an extend, unless we looked through a
7337 // G_ZEXT earlier.
7338 if (!LookedThroughZExt) {
7339 MachineInstr *ExtInst = getDefIgnoringCopies(Reg: OffsetReg, MRI);
7340 auto Ext = getExtendTypeForInst(MI&: *ExtInst, MRI, IsLoadStore: true);
7341 if (Ext == AArch64_AM::InvalidShiftExtend)
7342 return std::nullopt;
7343
7344 SignExtend = AArch64_AM::isSignExtendShiftType(Type: Ext) ? 1 : 0;
7345 // We only support SXTW for signed extension here.
7346 if (SignExtend && Ext != AArch64_AM::SXTW)
7347 return std::nullopt;
7348 OffsetReg = ExtInst->getOperand(i: 1).getReg();
7349 }
7350
7351 // Need a 32-bit wide register here.
7352 MachineIRBuilder MIB(*MRI.getVRegDef(Reg: Root.getReg()));
7353 OffsetReg = moveScalarRegClass(Reg: OffsetReg, RC: AArch64::GPR32RegClass, MIB);
7354 }
7355
7356 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
7357 // offset. Signify that we are shifting by setting the shift flag to 1.
7358 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base.getReg()); },
7359 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: OffsetReg); },
7360 [=](MachineInstrBuilder &MIB) {
7361 // Need to add both immediates here to make sure that they are both
7362 // added to the instruction.
7363 MIB.addImm(Val: SignExtend);
7364 MIB.addImm(Val: 1);
7365 }}};
7366}
7367
7368/// This is used for computing addresses like this:
7369///
7370/// ldr x1, [x2, x3, lsl #3]
7371///
7372/// Where x2 is the base register, and x3 is an offset register. The shift-left
7373/// is a constant value specific to this load instruction. That is, we'll never
7374/// see anything other than a 3 here (which corresponds to the size of the
7375/// element being loaded.)
7376InstructionSelector::ComplexRendererFns
7377AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7378 MachineOperand &Root, unsigned SizeInBytes) const {
7379 if (!Root.isReg())
7380 return std::nullopt;
7381 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7382
7383 // We want to find something like this:
7384 //
7385 // val = G_CONSTANT LegalShiftVal
7386 // shift = G_SHL off_reg val
7387 // ptr = G_PTR_ADD base_reg shift
7388 // x = G_LOAD ptr
7389 //
7390 // And fold it into this addressing mode:
7391 //
7392 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
7393
7394 // Check if we can find the G_PTR_ADD.
7395 MachineInstr *PtrAdd =
7396 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7397 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7398 return std::nullopt;
7399
7400 // Now, try to match an opcode which will match our specific offset.
7401 // We want a G_SHL or a G_MUL.
7402 MachineInstr *OffsetInst =
7403 getDefIgnoringCopies(Reg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7404 return selectExtendedSHL(Root, Base&: PtrAdd->getOperand(i: 1),
7405 Offset&: OffsetInst->getOperand(i: 0), SizeInBytes,
7406 /*WantsExt=*/false);
7407}
7408
7409/// This is used for computing addresses like this:
7410///
7411/// ldr x1, [x2, x3]
7412///
7413/// Where x2 is the base register, and x3 is an offset register.
7414///
7415/// When possible (or profitable) to fold a G_PTR_ADD into the address
7416/// calculation, this will do so. Otherwise, it will return std::nullopt.
7417InstructionSelector::ComplexRendererFns
7418AArch64InstructionSelector::selectAddrModeRegisterOffset(
7419 MachineOperand &Root) const {
7420 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7421
7422 // We need a GEP.
7423 Register Base, Offset;
7424 if (!mi_match(R: Root.getReg(), MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))))
7425 return std::nullopt;
7426
7427 // If this is used more than once, let's not bother folding.
7428 // TODO: Check if they are memory ops. If they are, then we can still fold
7429 // without having to recompute anything.
7430 if (!MRI.hasOneNonDBGUse(RegNo: Root.getReg()))
7431 return std::nullopt;
7432
7433 // Base is the GEP's LHS, offset is its RHS.
7434 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base); },
7435 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Offset); },
7436 [=](MachineInstrBuilder &MIB) {
7437 // Need to add both immediates here to make sure that they are both
7438 // added to the instruction.
7439 MIB.addImm(Val: 0);
7440 MIB.addImm(Val: 0);
7441 }}};
7442}
7443
7444/// This is intended to be equivalent to selectAddrModeXRO in
7445/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
7446InstructionSelector::ComplexRendererFns
7447AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7448 unsigned SizeInBytes) const {
7449 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7450 if (!Root.isReg())
7451 return std::nullopt;
7452 MachineInstr *PtrAdd =
7453 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7454 if (!PtrAdd)
7455 return std::nullopt;
7456
7457 // Check for an immediates which cannot be encoded in the [base + imm]
7458 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
7459 // end up with code like:
7460 //
7461 // mov x0, wide
7462 // add x1 base, x0
7463 // ldr x2, [x1, x0]
7464 //
7465 // In this situation, we can use the [base, xreg] addressing mode to save an
7466 // add/sub:
7467 //
7468 // mov x0, wide
7469 // ldr x2, [base, x0]
7470 auto ValAndVReg =
7471 getIConstantVRegValWithLookThrough(VReg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7472 if (ValAndVReg) {
7473 unsigned Scale = Log2_32(Value: SizeInBytes);
7474 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7475
7476 // Skip immediates that can be selected in the load/store addressing
7477 // mode.
7478 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7479 ImmOff < (0x1000 << Scale))
7480 return std::nullopt;
7481
7482 // Helper lambda to decide whether or not it is preferable to emit an add.
7483 auto isPreferredADD = [](int64_t ImmOff) {
7484 // Constants in [0x0, 0xfff] can be encoded in an add.
7485 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7486 return true;
7487
7488 // Can it be encoded in an add lsl #12?
7489 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7490 return false;
7491
7492 // It can be encoded in an add lsl #12, but we may not want to. If it is
7493 // possible to select this as a single movz, then prefer that. A single
7494 // movz is faster than an add with a shift.
7495 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7496 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7497 };
7498
7499 // If the immediate can be encoded in a single add/sub, then bail out.
7500 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
7501 return std::nullopt;
7502 }
7503
7504 // Try to fold shifts into the addressing mode.
7505 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7506 if (AddrModeFns)
7507 return AddrModeFns;
7508
7509 // If that doesn't work, see if it's possible to fold in registers from
7510 // a GEP.
7511 return selectAddrModeRegisterOffset(Root);
7512}
7513
7514/// This is used for computing addresses like this:
7515///
7516/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
7517///
7518/// Where we have a 64-bit base register, a 32-bit offset register, and an
7519/// extend (which may or may not be signed).
7520InstructionSelector::ComplexRendererFns
7521AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7522 unsigned SizeInBytes) const {
7523 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7524
7525 MachineInstr *PtrAdd =
7526 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7527 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7528 return std::nullopt;
7529
7530 MachineOperand &LHS = PtrAdd->getOperand(i: 1);
7531 MachineOperand &RHS = PtrAdd->getOperand(i: 2);
7532 MachineInstr *OffsetInst = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
7533
7534 // The first case is the same as selectAddrModeXRO, except we need an extend.
7535 // In this case, we try to find a shift and extend, and fold them into the
7536 // addressing mode.
7537 //
7538 // E.g.
7539 //
7540 // off_reg = G_Z/S/ANYEXT ext_reg
7541 // val = G_CONSTANT LegalShiftVal
7542 // shift = G_SHL off_reg val
7543 // ptr = G_PTR_ADD base_reg shift
7544 // x = G_LOAD ptr
7545 //
7546 // In this case we can get a load like this:
7547 //
7548 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
7549 auto ExtendedShl = selectExtendedSHL(Root, Base&: LHS, Offset&: OffsetInst->getOperand(i: 0),
7550 SizeInBytes, /*WantsExt=*/true);
7551 if (ExtendedShl)
7552 return ExtendedShl;
7553
7554 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
7555 //
7556 // e.g.
7557 // ldr something, [base_reg, ext_reg, sxtw]
7558 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7559 return std::nullopt;
7560
7561 // Check if this is an extend. We'll get an extend type if it is.
7562 AArch64_AM::ShiftExtendType Ext =
7563 getExtendTypeForInst(MI&: *OffsetInst, MRI, /*IsLoadStore=*/true);
7564 if (Ext == AArch64_AM::InvalidShiftExtend)
7565 return std::nullopt;
7566
7567 // Need a 32-bit wide register.
7568 MachineIRBuilder MIB(*PtrAdd);
7569 Register ExtReg = moveScalarRegClass(Reg: OffsetInst->getOperand(i: 1).getReg(),
7570 RC: AArch64::GPR32RegClass, MIB);
7571 unsigned SignExtend = Ext == AArch64_AM::SXTW;
7572
7573 // Base is LHS, offset is ExtReg.
7574 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: LHS.getReg()); },
7575 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7576 [=](MachineInstrBuilder &MIB) {
7577 MIB.addImm(Val: SignExtend);
7578 MIB.addImm(Val: 0);
7579 }}};
7580}
7581
7582/// Select a "register plus unscaled signed 9-bit immediate" address. This
7583/// should only match when there is an offset that is not valid for a scaled
7584/// immediate addressing mode. The "Size" argument is the size in bytes of the
7585/// memory reference, which is needed here to know what is valid for a scaled
7586/// immediate.
7587InstructionSelector::ComplexRendererFns
7588AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7589 unsigned Size) const {
7590 MachineRegisterInfo &MRI =
7591 Root.getParent()->getParent()->getParent()->getRegInfo();
7592
7593 if (!Root.isReg())
7594 return std::nullopt;
7595
7596 if (!isBaseWithConstantOffset(Root, MRI))
7597 return std::nullopt;
7598
7599 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7600
7601 MachineOperand &OffImm = RootDef->getOperand(i: 2);
7602 if (!OffImm.isReg())
7603 return std::nullopt;
7604 MachineInstr *RHS = MRI.getVRegDef(Reg: OffImm.getReg());
7605 if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7606 return std::nullopt;
7607 int64_t RHSC;
7608 MachineOperand &RHSOp1 = RHS->getOperand(i: 1);
7609 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
7610 return std::nullopt;
7611 RHSC = RHSOp1.getCImm()->getSExtValue();
7612
7613 if (RHSC >= -256 && RHSC < 256) {
7614 MachineOperand &Base = RootDef->getOperand(i: 1);
7615 return {{
7616 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Base); },
7617 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
7618 }};
7619 }
7620 return std::nullopt;
7621}
7622
7623InstructionSelector::ComplexRendererFns
7624AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7625 unsigned Size,
7626 MachineRegisterInfo &MRI) const {
7627 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
7628 return std::nullopt;
7629 MachineInstr &Adrp = *MRI.getVRegDef(Reg: RootDef.getOperand(i: 1).getReg());
7630 if (Adrp.getOpcode() != AArch64::ADRP)
7631 return std::nullopt;
7632
7633 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
7634 auto Offset = Adrp.getOperand(i: 1).getOffset();
7635 if (Offset % Size != 0)
7636 return std::nullopt;
7637
7638 auto GV = Adrp.getOperand(i: 1).getGlobal();
7639 if (GV->isThreadLocal())
7640 return std::nullopt;
7641
7642 auto &MF = *RootDef.getParent()->getParent();
7643 if (GV->getPointerAlignment(DL: MF.getDataLayout()) < Size)
7644 return std::nullopt;
7645
7646 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM: MF.getTarget());
7647 MachineIRBuilder MIRBuilder(RootDef);
7648 Register AdrpReg = Adrp.getOperand(i: 0).getReg();
7649 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: AdrpReg); },
7650 [=](MachineInstrBuilder &MIB) {
7651 MIB.addGlobalAddress(GV, Offset,
7652 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF |
7653 AArch64II::MO_NC);
7654 }}};
7655}
7656
7657/// Select a "register plus scaled unsigned 12-bit immediate" address. The
7658/// "Size" argument is the size in bytes of the memory reference, which
7659/// determines the scale.
7660InstructionSelector::ComplexRendererFns
7661AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7662 unsigned Size) const {
7663 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
7664 MachineRegisterInfo &MRI = MF.getRegInfo();
7665
7666 if (!Root.isReg())
7667 return std::nullopt;
7668
7669 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7670 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7671 return {{
7672 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
7673 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7674 }};
7675 }
7676
7677 CodeModel::Model CM = MF.getTarget().getCodeModel();
7678 // Check if we can fold in the ADD of small code model ADRP + ADD address.
7679 // HACK: ld64 on Darwin doesn't support relocations on PRFM, so we can't fold
7680 // globals into the offset.
7681 MachineInstr *RootParent = Root.getParent();
7682 if (CM == CodeModel::Small &&
7683 !(RootParent->getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7684 STI.isTargetDarwin())) {
7685 auto OpFns = tryFoldAddLowIntoImm(RootDef&: *RootDef, Size, MRI);
7686 if (OpFns)
7687 return OpFns;
7688 }
7689
7690 if (isBaseWithConstantOffset(Root, MRI)) {
7691 MachineOperand &LHS = RootDef->getOperand(i: 1);
7692 MachineOperand &RHS = RootDef->getOperand(i: 2);
7693 MachineInstr *LHSDef = MRI.getVRegDef(Reg: LHS.getReg());
7694 MachineInstr *RHSDef = MRI.getVRegDef(Reg: RHS.getReg());
7695
7696 int64_t RHSC = (int64_t)RHSDef->getOperand(i: 1).getCImm()->getZExtValue();
7697 unsigned Scale = Log2_32(Value: Size);
7698 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7699 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
7700 return {{
7701 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
7702 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7703 }};
7704
7705 return {{
7706 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
7707 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7708 }};
7709 }
7710 }
7711
7712 // Before falling back to our general case, check if the unscaled
7713 // instructions can handle this. If so, that's preferable.
7714 if (selectAddrModeUnscaled(Root, Size))
7715 return std::nullopt;
7716
7717 return {{
7718 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
7719 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7720 }};
7721}
7722
7723/// Given a shift instruction, return the correct shift type for that
7724/// instruction.
7725static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
7726 switch (MI.getOpcode()) {
7727 default:
7728 return AArch64_AM::InvalidShiftExtend;
7729 case TargetOpcode::G_SHL:
7730 return AArch64_AM::LSL;
7731 case TargetOpcode::G_LSHR:
7732 return AArch64_AM::LSR;
7733 case TargetOpcode::G_ASHR:
7734 return AArch64_AM::ASR;
7735 case TargetOpcode::G_ROTR:
7736 return AArch64_AM::ROR;
7737 }
7738}
7739
7740/// Select a "shifted register" operand. If the value is not shifted, set the
7741/// shift operand to a default value of "lsl 0".
7742InstructionSelector::ComplexRendererFns
7743AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7744 bool AllowROR) const {
7745 if (!Root.isReg())
7746 return std::nullopt;
7747 MachineRegisterInfo &MRI =
7748 Root.getParent()->getParent()->getParent()->getRegInfo();
7749
7750 // Check if the operand is defined by an instruction which corresponds to
7751 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
7752 MachineInstr *ShiftInst = MRI.getVRegDef(Reg: Root.getReg());
7753 AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(MI&: *ShiftInst);
7754 if (ShType == AArch64_AM::InvalidShiftExtend)
7755 return std::nullopt;
7756 if (ShType == AArch64_AM::ROR && !AllowROR)
7757 return std::nullopt;
7758 if (!isWorthFoldingIntoExtendedReg(MI: *ShiftInst, MRI, IsAddrOperand: false))
7759 return std::nullopt;
7760
7761 // Need an immediate on the RHS.
7762 MachineOperand &ShiftRHS = ShiftInst->getOperand(i: 2);
7763 auto Immed = getImmedFromMO(Root: ShiftRHS);
7764 if (!Immed)
7765 return std::nullopt;
7766
7767 // We have something that we can fold. Fold in the shift's LHS and RHS into
7768 // the instruction.
7769 MachineOperand &ShiftLHS = ShiftInst->getOperand(i: 1);
7770 Register ShiftReg = ShiftLHS.getReg();
7771
7772 unsigned NumBits = MRI.getType(Reg: ShiftReg).getSizeInBits();
7773 unsigned Val = *Immed & (NumBits - 1);
7774 unsigned ShiftVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
7775
7776 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ShiftReg); },
7777 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShiftVal); }}};
7778}
7779
7780AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
7781 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
7782 unsigned Opc = MI.getOpcode();
7783
7784 // Handle explicit extend instructions first.
7785 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7786 unsigned Size;
7787 if (Opc == TargetOpcode::G_SEXT)
7788 Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7789 else
7790 Size = MI.getOperand(i: 2).getImm();
7791 assert(Size != 64 && "Extend from 64 bits?");
7792 switch (Size) {
7793 case 8:
7794 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
7795 case 16:
7796 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
7797 case 32:
7798 return AArch64_AM::SXTW;
7799 default:
7800 return AArch64_AM::InvalidShiftExtend;
7801 }
7802 }
7803
7804 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7805 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7806 assert(Size != 64 && "Extend from 64 bits?");
7807 switch (Size) {
7808 case 8:
7809 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
7810 case 16:
7811 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
7812 case 32:
7813 return AArch64_AM::UXTW;
7814 default:
7815 return AArch64_AM::InvalidShiftExtend;
7816 }
7817 }
7818
7819 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
7820 // on the RHS.
7821 if (Opc != TargetOpcode::G_AND)
7822 return AArch64_AM::InvalidShiftExtend;
7823
7824 std::optional<uint64_t> MaybeAndMask = getImmedFromMO(Root: MI.getOperand(i: 2));
7825 if (!MaybeAndMask)
7826 return AArch64_AM::InvalidShiftExtend;
7827 uint64_t AndMask = *MaybeAndMask;
7828 switch (AndMask) {
7829 default:
7830 return AArch64_AM::InvalidShiftExtend;
7831 case 0xFF:
7832 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
7833 case 0xFFFF:
7834 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
7835 case 0xFFFFFFFF:
7836 return AArch64_AM::UXTW;
7837 }
7838}
7839
7840Register AArch64InstructionSelector::moveScalarRegClass(
7841 Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
7842 MachineRegisterInfo &MRI = *MIB.getMRI();
7843 auto Ty = MRI.getType(Reg);
7844 assert(!Ty.isVector() && "Expected scalars only!");
7845 if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
7846 return Reg;
7847
7848 // Create a copy and immediately select it.
7849 // FIXME: We should have an emitCopy function?
7850 auto Copy = MIB.buildCopy(Res: {&RC}, Op: {Reg});
7851 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
7852 return Copy.getReg(Idx: 0);
7853}
7854
7855/// Select an "extended register" operand. This operand folds in an extend
7856/// followed by an optional left shift.
7857InstructionSelector::ComplexRendererFns
7858AArch64InstructionSelector::selectArithExtendedRegister(
7859 MachineOperand &Root) const {
7860 if (!Root.isReg())
7861 return std::nullopt;
7862 MachineRegisterInfo &MRI =
7863 Root.getParent()->getParent()->getParent()->getRegInfo();
7864
7865 uint64_t ShiftVal = 0;
7866 Register ExtReg;
7867 AArch64_AM::ShiftExtendType Ext;
7868 MachineInstr *RootDef = getDefIgnoringCopies(Reg: Root.getReg(), MRI);
7869 if (!RootDef)
7870 return std::nullopt;
7871
7872 if (!isWorthFoldingIntoExtendedReg(MI: *RootDef, MRI, IsAddrOperand: false))
7873 return std::nullopt;
7874
7875 // Check if we can fold a shift and an extend.
7876 if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
7877 // Look for a constant on the RHS of the shift.
7878 MachineOperand &RHS = RootDef->getOperand(i: 2);
7879 std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(Root: RHS);
7880 if (!MaybeShiftVal)
7881 return std::nullopt;
7882 ShiftVal = *MaybeShiftVal;
7883 if (ShiftVal > 4)
7884 return std::nullopt;
7885 // Look for a valid extend instruction on the LHS of the shift.
7886 MachineOperand &LHS = RootDef->getOperand(i: 1);
7887 MachineInstr *ExtDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
7888 if (!ExtDef)
7889 return std::nullopt;
7890 Ext = getExtendTypeForInst(MI&: *ExtDef, MRI);
7891 if (Ext == AArch64_AM::InvalidShiftExtend)
7892 return std::nullopt;
7893 ExtReg = ExtDef->getOperand(i: 1).getReg();
7894 } else {
7895 // Didn't get a shift. Try just folding an extend.
7896 Ext = getExtendTypeForInst(MI&: *RootDef, MRI);
7897 if (Ext == AArch64_AM::InvalidShiftExtend)
7898 return std::nullopt;
7899 ExtReg = RootDef->getOperand(i: 1).getReg();
7900
7901 // If we have a 32 bit instruction which zeroes out the high half of a
7902 // register, we get an implicit zero extend for free. Check if we have one.
7903 // FIXME: We actually emit the extend right now even though we don't have
7904 // to.
7905 if (Ext == AArch64_AM::UXTW && MRI.getType(Reg: ExtReg).getSizeInBits() == 32) {
7906 MachineInstr *ExtInst = MRI.getVRegDef(Reg: ExtReg);
7907 if (isDef32(MI: *ExtInst))
7908 return std::nullopt;
7909 }
7910 }
7911
7912 // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
7913 // copy.
7914 MachineIRBuilder MIB(*RootDef);
7915 ExtReg = moveScalarRegClass(Reg: ExtReg, RC: AArch64::GPR32RegClass, MIB);
7916
7917 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7918 [=](MachineInstrBuilder &MIB) {
7919 MIB.addImm(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal));
7920 }}};
7921}
7922
7923InstructionSelector::ComplexRendererFns
7924AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {
7925 if (!Root.isReg())
7926 return std::nullopt;
7927 MachineRegisterInfo &MRI =
7928 Root.getParent()->getParent()->getParent()->getRegInfo();
7929
7930 auto Extract = getDefSrcRegIgnoringCopies(Reg: Root.getReg(), MRI);
7931 while (Extract && Extract->MI->getOpcode() == TargetOpcode::G_BITCAST &&
7932 STI.isLittleEndian())
7933 Extract =
7934 getDefSrcRegIgnoringCopies(Reg: Extract->MI->getOperand(i: 1).getReg(), MRI);
7935 if (!Extract)
7936 return std::nullopt;
7937
7938 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: Extract->MI)) {
7939 if (Unmerge->getNumDefs() == 2 &&
7940 Extract->Reg == Unmerge->getOperand(i: 1).getReg()) {
7941 Register ExtReg = Unmerge->getSourceReg();
7942 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
7943 }
7944 }
7945 if (auto *ExtElt = dyn_cast<GExtractVectorElement>(Val: Extract->MI)) {
7946 LLT SrcTy = MRI.getType(Reg: ExtElt->getVectorReg());
7947 auto LaneIdx =
7948 getIConstantVRegValWithLookThrough(VReg: ExtElt->getIndexReg(), MRI);
7949 if (LaneIdx && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
7950 LaneIdx->Value.getSExtValue() == 1) {
7951 Register ExtReg = ExtElt->getVectorReg();
7952 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
7953 }
7954 }
7955 if (auto *Subvec = dyn_cast<GExtractSubvector>(Val: Extract->MI)) {
7956 LLT SrcTy = MRI.getType(Reg: Subvec->getSrcVec());
7957 auto LaneIdx = Subvec->getIndexImm();
7958 if (LaneIdx == SrcTy.getNumElements() / 2) {
7959 Register ExtReg = Subvec->getSrcVec();
7960 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
7961 }
7962 }
7963
7964 return std::nullopt;
7965}
7966
7967InstructionSelector::ComplexRendererFns
7968AArch64InstructionSelector::selectCVTFixedPointBase(const MachineOperand &Root,
7969 unsigned DstElemWidth,
7970 bool isReciprocal) const {
7971 if (!Root.isReg())
7972 return std::nullopt;
7973 const MachineRegisterInfo &MRI =
7974 Root.getParent()->getParent()->getParent()->getRegInfo();
7975
7976 Register Reg = Root.getReg();
7977 MachineInstr *Dup = getDefIgnoringCopies(Reg, MRI);
7978
7979 if (Dup && Dup->getOpcode() == AArch64::G_DUP)
7980 Reg = Dup->getOperand(i: 1).getReg();
7981
7982 std::optional<ValueAndVReg> CstVal =
7983 getAnyConstantVRegValWithLookThrough(VReg: Reg, MRI);
7984
7985 if (!CstVal)
7986 return std::nullopt;
7987
7988 unsigned CstElemWidth = MRI.getType(Reg).getScalarSizeInBits();
7989 APFloat FVal(0.0);
7990 switch (CstElemWidth) {
7991 case 16:
7992 FVal = APFloat(APFloat::IEEEhalf(), CstVal->Value);
7993 break;
7994 case 32:
7995 FVal = APFloat(APFloat::IEEEsingle(), CstVal->Value);
7996 break;
7997 case 64:
7998 FVal = APFloat(APFloat::IEEEdouble(), CstVal->Value);
7999 break;
8000 default:
8001 return std::nullopt;
8002 };
8003 if (unsigned FBits =
8004 CheckFixedPointOperandConstant(FVal, RegWidth: DstElemWidth, isReciprocal))
8005 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: FBits); }}};
8006
8007 return std::nullopt;
8008}
8009
8010unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8011 const MachineOperand &Root) const {
8012 return Root.getParent()
8013 ->getMF()
8014 ->getRegInfo()
8015 .getType(Reg: Root.getReg())
8016 .getScalarSizeInBits();
8017}
8018
8019template <unsigned Width>
8020InstructionSelector::ComplexRendererFns
8021AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root) const {
8022 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ false);
8023}
8024
8025InstructionSelector::ComplexRendererFns
8026AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
8027 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8028 /*isReciprocal*/ false);
8029}
8030
8031InstructionSelector::ComplexRendererFns
8032AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8033 MachineOperand &Root) const {
8034 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8035 /*isReciprocal*/ true);
8036}
8037
8038void AArch64InstructionSelector::renderFixedPointScalarXForm(
8039 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8040 assert(OpIdx == 3 && MI.getOperand(OpIdx).isImm() &&
8041 "Expected vecshift immediate operand");
8042 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm());
8043}
8044
8045void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8046 const MachineOperand &Root,
8047 unsigned Width,
8048 bool isReciprocal) const {
8049 // FIXME: This is only needed to satisfy the type checking in tablegen, and
8050 // should be able to reuse the Renderers already calculated by
8051 // selectCVTFixedPointBase.
8052 InstructionSelector::ComplexRendererFns Renderer =
8053 selectCVTFixedPointBase(Root, DstElemWidth: Width, isReciprocal);
8054 assert((Renderer && Renderer->size() == 1) &&
8055 "Expected selectCVTFixedPointBase to provide a function\n");
8056 (Renderer->front())(MIB);
8057}
8058
8059void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8060 const MachineInstr &MI,
8061 int OpIdx) const {
8062 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8063 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8064 /*isReciprocal*/ false);
8065}
8066
8067void AArch64InstructionSelector::renderFixedPointRecipXForm(
8068 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8069 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8070 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8071 /*isReciprocal*/ true);
8072}
8073
8074void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8075 const MachineInstr &MI,
8076 int OpIdx) const {
8077 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8078 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8079 "Expected G_CONSTANT");
8080 std::optional<int64_t> CstVal =
8081 getIConstantVRegSExtVal(VReg: MI.getOperand(i: 0).getReg(), MRI);
8082 assert(CstVal && "Expected constant value");
8083 MIB.addImm(Val: *CstVal);
8084}
8085
8086void AArch64InstructionSelector::renderLogicalImm32(
8087 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8088 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8089 "Expected G_CONSTANT");
8090 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8091 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 32);
8092 MIB.addImm(Val: Enc);
8093}
8094
8095void AArch64InstructionSelector::renderLogicalImm64(
8096 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8097 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8098 "Expected G_CONSTANT");
8099 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8100 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 64);
8101 MIB.addImm(Val: Enc);
8102}
8103
8104void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8105 const MachineInstr &MI,
8106 int OpIdx) const {
8107 assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8108 "Expected G_UBSANTRAP");
8109 MIB.addImm(Val: MI.getOperand(i: 0).getImm() | ('U' << 8));
8110}
8111
8112void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8113 const MachineInstr &MI,
8114 int OpIdx) const {
8115 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8116 "Expected G_FCONSTANT");
8117 MIB.addImm(
8118 Val: AArch64_AM::getFP16Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8119}
8120
8121void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8122 const MachineInstr &MI,
8123 int OpIdx) const {
8124 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8125 "Expected G_FCONSTANT");
8126 MIB.addImm(
8127 Val: AArch64_AM::getFP32Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8128}
8129
8130void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8131 const MachineInstr &MI,
8132 int OpIdx) const {
8133 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8134 "Expected G_FCONSTANT");
8135 MIB.addImm(
8136 Val: AArch64_AM::getFP64Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8137}
8138
8139void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
8140 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8141 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8142 "Expected G_FCONSTANT");
8143 MIB.addImm(Val: AArch64_AM::encodeAdvSIMDModImmType4(Imm: MI.getOperand(i: 1)
8144 .getFPImm()
8145 ->getValueAPF()
8146 .bitcastToAPInt()
8147 .getZExtValue()));
8148}
8149
8150bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8151 const MachineInstr &MI, unsigned NumBytes) const {
8152 if (!MI.mayLoadOrStore())
8153 return false;
8154 assert(MI.hasOneMemOperand() &&
8155 "Expected load/store to have only one mem op!");
8156 return (*MI.memoperands_begin())->getSize() == NumBytes;
8157}
8158
8159bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
8160 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8161 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() != 32)
8162 return false;
8163
8164 // Only return true if we know the operation will zero-out the high half of
8165 // the 64-bit register. Truncates can be subregister copies, which don't
8166 // zero out the high bits. Copies and other copy-like instructions can be
8167 // fed by truncates, or could be lowered as subregister copies.
8168 switch (MI.getOpcode()) {
8169 default:
8170 return true;
8171 case TargetOpcode::COPY:
8172 case TargetOpcode::G_BITCAST:
8173 case TargetOpcode::G_TRUNC:
8174 case TargetOpcode::G_PHI:
8175 return false;
8176 }
8177}
8178
8179
8180// Perform fixups on the given PHI instruction's operands to force them all
8181// to be the same as the destination regbank.
8182static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
8183 const AArch64RegisterBankInfo &RBI) {
8184 assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI");
8185 Register DstReg = MI.getOperand(i: 0).getReg();
8186 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: DstReg);
8187 assert(DstRB && "Expected PHI dst to have regbank assigned");
8188 MachineIRBuilder MIB(MI);
8189
8190 // Go through each operand and ensure it has the same regbank.
8191 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
8192 if (!MO.isReg())
8193 continue;
8194 Register OpReg = MO.getReg();
8195 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: OpReg);
8196 if (RB != DstRB) {
8197 // Insert a cross-bank copy.
8198 auto *OpDef = MRI.getVRegDef(Reg: OpReg);
8199 const LLT &Ty = MRI.getType(Reg: OpReg);
8200 MachineBasicBlock &OpDefBB = *OpDef->getParent();
8201
8202 // Any instruction we insert must appear after all PHIs in the block
8203 // for the block to be valid MIR.
8204 MachineBasicBlock::iterator InsertPt = std::next(x: OpDef->getIterator());
8205 if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
8206 InsertPt = OpDefBB.getFirstNonPHI();
8207 MIB.setInsertPt(MBB&: *OpDef->getParent(), II: InsertPt);
8208 auto Copy = MIB.buildCopy(Res: Ty, Op: OpReg);
8209 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: *DstRB);
8210 MO.setReg(Copy.getReg(Idx: 0));
8211 }
8212 }
8213}
8214
8215void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
8216 // We're looking for PHIs, build a list so we don't invalidate iterators.
8217 MachineRegisterInfo &MRI = MF.getRegInfo();
8218 SmallVector<MachineInstr *, 32> Phis;
8219 for (auto &BB : MF) {
8220 for (auto &MI : BB) {
8221 if (MI.getOpcode() == TargetOpcode::G_PHI)
8222 Phis.emplace_back(Args: &MI);
8223 }
8224 }
8225
8226 for (auto *MI : Phis) {
8227 // We need to do some work here if the operand types are < 16 bit and they
8228 // are split across fpr/gpr banks. Since all types <32b on gpr
8229 // end up being assigned gpr32 regclasses, we can end up with PHIs here
8230 // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
8231 // be selecting heterogenous regbanks for operands if possible, but we
8232 // still need to be able to deal with it here.
8233 //
8234 // To fix this, if we have a gpr-bank operand < 32b in size and at least
8235 // one other operand is on the fpr bank, then we add cross-bank copies
8236 // to homogenize the operand banks. For simplicity the bank that we choose
8237 // to settle on is whatever bank the def operand has. For example:
8238 //
8239 // %endbb:
8240 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
8241 // =>
8242 // %bb2:
8243 // ...
8244 // %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
8245 // ...
8246 // %endbb:
8247 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
8248 bool HasGPROp = false, HasFPROp = false;
8249 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
8250 if (!MO.isReg())
8251 continue;
8252 const LLT &Ty = MRI.getType(Reg: MO.getReg());
8253 if (!Ty.isValid() || !Ty.isScalar())
8254 break;
8255 if (Ty.getSizeInBits() >= 32)
8256 break;
8257 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: MO.getReg());
8258 // If for some reason we don't have a regbank yet. Don't try anything.
8259 if (!RB)
8260 break;
8261
8262 if (RB->getID() == AArch64::GPRRegBankID)
8263 HasGPROp = true;
8264 else
8265 HasFPROp = true;
8266 }
8267 // We have heterogenous regbanks, need to fixup.
8268 if (HasGPROp && HasFPROp)
8269 fixupPHIOpBanks(MI&: *MI, MRI, RBI);
8270 }
8271}
8272
8273namespace llvm {
8274InstructionSelector *
8275createAArch64InstructionSelector(const AArch64TargetMachine &TM,
8276 const AArch64Subtarget &Subtarget,
8277 const AArch64RegisterBankInfo &RBI) {
8278 return new AArch64InstructionSelector(TM, Subtarget, RBI);
8279}
8280}
8281