1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64GlobalISelUtils.h"
15#include "AArch64InstrInfo.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterBankInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetMachine.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64MCTargetDesc.h"
23#include "llvm/BinaryFormat/Dwarf.h"
24#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
25#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
29#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
30#include "llvm/CodeGen/GlobalISel/Utils.h"
31#include "llvm/CodeGen/MachineBasicBlock.h"
32#include "llvm/CodeGen/MachineConstantPool.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstr.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/TargetOpcodes.h"
41#include "llvm/CodeGen/TargetRegisterInfo.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/Instructions.h"
45#include "llvm/IR/IntrinsicsAArch64.h"
46#include "llvm/IR/Type.h"
47#include "llvm/Pass.h"
48#include "llvm/Support/Debug.h"
49#include "llvm/Support/raw_ostream.h"
50#include <optional>
51
52#define DEBUG_TYPE "aarch64-isel"
53
54using namespace llvm;
55using namespace MIPatternMatch;
56using namespace AArch64GISelUtils;
57
58namespace llvm {
59class BlockFrequencyInfo;
60class ProfileSummaryInfo;
61}
62
63namespace {
64
65#define GET_GLOBALISEL_PREDICATE_BITSET
66#include "AArch64GenGlobalISel.inc"
67#undef GET_GLOBALISEL_PREDICATE_BITSET
68
69
70class AArch64InstructionSelector : public InstructionSelector {
71public:
72 AArch64InstructionSelector(const AArch64TargetMachine &TM,
73 const AArch64Subtarget &STI,
74 const AArch64RegisterBankInfo &RBI);
75
76 bool select(MachineInstr &I) override;
77 static const char *getName() { return DEBUG_TYPE; }
78
79 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
80 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
81 BlockFrequencyInfo *BFI) override {
82 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
83 MIB.setMF(MF);
84
85 // hasFnAttribute() is expensive to call on every BRCOND selection, so
86 // cache it here for each run of the selector.
87 ProduceNonFlagSettingCondBr =
88 !MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening);
89 MFReturnAddr = Register();
90
91 processPHIs(MF);
92 }
93
94private:
95 /// tblgen-erated 'select' implementation, used as the initial selector for
96 /// the patterns that don't require complex C++.
97 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
98
99 // A lowering phase that runs before any selection attempts.
100 // Returns true if the instruction was modified.
101 bool preISelLower(MachineInstr &I);
102
103 // An early selection function that runs before the selectImpl() call.
104 bool earlySelect(MachineInstr &I);
105
106 /// Save state that is shared between select calls, call select on \p I and
107 /// then restore the saved state. This can be used to recursively call select
108 /// within a select call.
109 bool selectAndRestoreState(MachineInstr &I);
110
111 // Do some preprocessing of G_PHIs before we begin selection.
112 void processPHIs(MachineFunction &MF);
113
114 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);
115
116 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
117 bool contractCrossBankCopyIntoStore(MachineInstr &I,
118 MachineRegisterInfo &MRI);
119
120 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
121
122 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
123 MachineRegisterInfo &MRI) const;
124 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
125 MachineRegisterInfo &MRI) const;
126
127 ///@{
128 /// Helper functions for selectCompareBranch.
129 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
130 MachineIRBuilder &MIB) const;
131 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
132 MachineIRBuilder &MIB) const;
133 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
134 MachineIRBuilder &MIB) const;
135 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
136 MachineBasicBlock *DstMBB,
137 MachineIRBuilder &MIB) const;
138 ///@}
139
140 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
141 MachineRegisterInfo &MRI);
142
143 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
144 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);
145
146 // Helper to generate an equivalent of scalar_to_vector into a new register,
147 // returned via 'Dst'.
148 MachineInstr *emitScalarToVector(unsigned EltSize,
149 const TargetRegisterClass *DstRC,
150 Register Scalar,
151 MachineIRBuilder &MIRBuilder) const;
152 /// Helper to narrow vector that was widened by emitScalarToVector.
153 /// Copy lowest part of 128-bit or 64-bit vector to 64-bit or 32-bit
154 /// vector, correspondingly.
155 MachineInstr *emitNarrowVector(Register DstReg, Register SrcReg,
156 MachineIRBuilder &MIRBuilder,
157 MachineRegisterInfo &MRI) const;
158
159 /// Emit a lane insert into \p DstReg, or a new vector register if
160 /// std::nullopt is provided.
161 ///
162 /// The lane inserted into is defined by \p LaneIdx. The vector source
163 /// register is given by \p SrcReg. The register containing the element is
164 /// given by \p EltReg.
165 MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
166 Register EltReg, unsigned LaneIdx,
167 const RegisterBank &RB,
168 MachineIRBuilder &MIRBuilder) const;
169
170 /// Emit a sequence of instructions representing a constant \p CV for a
171 /// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
172 ///
173 /// \returns the last instruction in the sequence on success, and nullptr
174 /// otherwise.
175 MachineInstr *emitConstantVector(Register Dst, Constant *CV,
176 MachineIRBuilder &MIRBuilder,
177 MachineRegisterInfo &MRI);
178
179 MachineInstr *tryAdvSIMDModImm8(Register Dst, unsigned DstSize, APInt Bits,
180 MachineIRBuilder &MIRBuilder);
181
182 MachineInstr *tryAdvSIMDModImm16(Register Dst, unsigned DstSize, APInt Bits,
183 MachineIRBuilder &MIRBuilder, bool Inv);
184
185 MachineInstr *tryAdvSIMDModImm32(Register Dst, unsigned DstSize, APInt Bits,
186 MachineIRBuilder &MIRBuilder, bool Inv);
187 MachineInstr *tryAdvSIMDModImm64(Register Dst, unsigned DstSize, APInt Bits,
188 MachineIRBuilder &MIRBuilder);
189 MachineInstr *tryAdvSIMDModImm321s(Register Dst, unsigned DstSize, APInt Bits,
190 MachineIRBuilder &MIRBuilder, bool Inv);
191 MachineInstr *tryAdvSIMDModImmFP(Register Dst, unsigned DstSize, APInt Bits,
192 MachineIRBuilder &MIRBuilder);
193
194 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
195 MachineRegisterInfo &MRI);
196 /// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
197 /// SUBREG_TO_REG.
198 bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
199 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
200 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
201 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
202
203 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
204 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
205 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
206 bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);
207
208 /// Helper function to select vector load intrinsics like
209 /// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
210 /// \p Opc is the opcode that the selected instruction should use.
211 /// \p NumVecs is the number of vector destinations for the instruction.
212 /// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
213 bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
214 MachineInstr &I);
215 bool selectVectorLoadLaneIntrinsic(unsigned Opc, unsigned NumVecs,
216 MachineInstr &I);
217 void selectVectorStoreIntrinsic(MachineInstr &I, unsigned NumVecs,
218 unsigned Opc);
219 bool selectVectorStoreLaneIntrinsic(MachineInstr &I, unsigned NumVecs,
220 unsigned Opc);
221 bool selectIntrinsicWithSideEffects(MachineInstr &I,
222 MachineRegisterInfo &MRI);
223 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
224 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
225 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
226 bool selectTLSGlobalValueELF(MachineInstr &I, MachineRegisterInfo &MRI);
227 bool selectTLSLocalExecELF(const GlobalValue *GV, MachineInstr &I,
228 MachineRegisterInfo &MRI);
229 bool selectTLSGlobalValueMachO(MachineInstr &I, MachineRegisterInfo &MRI);
230 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
231 bool selectPtrAuthGlobalValue(MachineInstr &I,
232 MachineRegisterInfo &MRI) const;
233 bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);
234 bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
235 bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
236 void SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVecs,
237 unsigned Opc1, unsigned Opc2, bool isExt);
238
239 bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
240 bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
241 bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);
242
243 unsigned emitConstantPoolEntry(const Constant *CPVal,
244 MachineFunction &MF) const;
245 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
246 MachineIRBuilder &MIRBuilder) const;
247
248 // Emit a vector concat operation.
249 MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
250 Register Op2,
251 MachineIRBuilder &MIRBuilder) const;
252
253 // Emit an integer compare between LHS and RHS, which checks for Predicate.
254 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
255 MachineOperand &Predicate,
256 MachineIRBuilder &MIRBuilder) const;
257
258 /// Emit a floating point comparison between \p LHS and \p RHS.
259 /// \p Pred if given is the intended predicate to use.
260 MachineInstr *
261 emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
262 std::optional<CmpInst::Predicate> = std::nullopt) const;
263
264 MachineInstr *
265 emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
266 std::initializer_list<llvm::SrcOp> SrcOps,
267 MachineIRBuilder &MIRBuilder,
268 const ComplexRendererFns &RenderFns = std::nullopt) const;
269 /// Helper function to emit an add or sub instruction.
270 ///
271 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
272 /// in a specific order.
273 ///
274 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
275 ///
276 /// \code
277 /// const std::array<std::array<unsigned, 2>, 4> Table {
278 /// {{AArch64::ADDXri, AArch64::ADDWri},
279 /// {AArch64::ADDXrs, AArch64::ADDWrs},
280 /// {AArch64::ADDXrr, AArch64::ADDWrr},
281 /// {AArch64::SUBXri, AArch64::SUBWri},
282 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
283 /// \endcode
284 ///
285 /// Each row in the table corresponds to a different addressing mode. Each
286 /// column corresponds to a different register size.
287 ///
288 /// \attention Rows must be structured as follows:
289 /// - Row 0: The ri opcode variants
290 /// - Row 1: The rs opcode variants
291 /// - Row 2: The rr opcode variants
292 /// - Row 3: The ri opcode variants for negative immediates
293 /// - Row 4: The rx opcode variants
294 ///
295 /// \attention Columns must be structured as follows:
296 /// - Column 0: The 64-bit opcode variants
297 /// - Column 1: The 32-bit opcode variants
298 ///
299 /// \p Dst is the destination register of the binop to emit.
300 /// \p LHS is the left-hand operand of the binop to emit.
301 /// \p RHS is the right-hand operand of the binop to emit.
302 MachineInstr *emitAddSub(
303 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
304 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
305 MachineIRBuilder &MIRBuilder) const;
306 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
307 MachineOperand &RHS,
308 MachineIRBuilder &MIRBuilder) const;
309 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
310 MachineIRBuilder &MIRBuilder) const;
311 MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
312 MachineIRBuilder &MIRBuilder) const;
313 MachineInstr *emitADCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
314 MachineIRBuilder &MIRBuilder) const;
315 MachineInstr *emitSBCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
316 MachineIRBuilder &MIRBuilder) const;
317 MachineInstr *emitCMP(MachineOperand &LHS, MachineOperand &RHS,
318 MachineIRBuilder &MIRBuilder) const;
319 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
320 MachineIRBuilder &MIRBuilder) const;
321 MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
322 MachineIRBuilder &MIRBuilder) const;
323 MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
324 AArch64CC::CondCode CC,
325 MachineIRBuilder &MIRBuilder) const;
326 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
327 const RegisterBank &DstRB, LLT ScalarTy,
328 Register VecReg, unsigned LaneIdx,
329 MachineIRBuilder &MIRBuilder) const;
330 MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
331 AArch64CC::CondCode Pred,
332 MachineIRBuilder &MIRBuilder) const;
333 /// Emit a CSet for a FP compare.
334 ///
335 /// \p Dst is expected to be a 32-bit scalar register.
336 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
337 MachineIRBuilder &MIRBuilder) const;
338
339 /// Emit an instruction that sets NZCV to the carry-in expected by \p I.
340 /// Might elide the instruction if the previous instruction already sets NZCV
341 /// correctly.
342 MachineInstr *emitCarryIn(MachineInstr &I, Register CarryReg);
343
344 /// Emit the overflow op for \p Opcode.
345 ///
346 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
347 /// G_USUBO, etc.
348 std::pair<MachineInstr *, AArch64CC::CondCode>
349 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
350 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
351
352 bool selectOverflowOp(MachineInstr &I, MachineRegisterInfo &MRI);
353
354 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
355 /// In some cases this is even possible with OR operations in the expression.
356 MachineInstr *emitConjunction(Register Val, AArch64CC::CondCode &OutCC,
357 MachineIRBuilder &MIB) const;
358 MachineInstr *emitConditionalComparison(Register LHS, Register RHS,
359 CmpInst::Predicate CC,
360 AArch64CC::CondCode Predicate,
361 AArch64CC::CondCode OutCC,
362 MachineIRBuilder &MIB) const;
363 MachineInstr *emitConjunctionRec(Register Val, AArch64CC::CondCode &OutCC,
364 bool Negate, Register CCOp,
365 AArch64CC::CondCode Predicate,
366 MachineIRBuilder &MIB) const;
367
368 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
369 /// \p IsNegative is true if the test should be "not zero".
370 /// This will also optimize the test bit instruction when possible.
371 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
372 MachineBasicBlock *DstMBB,
373 MachineIRBuilder &MIB) const;
374
375 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
376 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
377 MachineBasicBlock *DestMBB,
378 MachineIRBuilder &MIB) const;
379
380 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
381 // We use these manually instead of using the importer since it doesn't
382 // support SDNodeXForm.
383 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
384 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
385 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
386 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
387
388 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
389 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
390 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
391
392 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
393 unsigned Size) const;
394
395 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
396 return selectAddrModeUnscaled(Root, Size: 1);
397 }
398 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
399 return selectAddrModeUnscaled(Root, Size: 2);
400 }
401 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
402 return selectAddrModeUnscaled(Root, Size: 4);
403 }
404 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
405 return selectAddrModeUnscaled(Root, Size: 8);
406 }
407 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
408 return selectAddrModeUnscaled(Root, Size: 16);
409 }
410
411 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
412 /// from complex pattern matchers like selectAddrModeIndexed().
413 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
414 MachineRegisterInfo &MRI) const;
415
416 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
417 unsigned Size) const;
418 template <int Width>
419 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
420 return selectAddrModeIndexed(Root, Size: Width / 8);
421 }
422
423 std::optional<bool>
424 isWorthFoldingIntoAddrMode(const MachineInstr &MI,
425 const MachineRegisterInfo &MRI) const;
426
427 bool isWorthFoldingIntoExtendedReg(const MachineInstr &MI,
428 const MachineRegisterInfo &MRI,
429 bool IsAddrOperand) const;
430 ComplexRendererFns
431 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
432 unsigned SizeInBytes) const;
433
434 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
435 /// or not a shift + extend should be folded into an addressing mode. Returns
436 /// None when this is not profitable or possible.
437 ComplexRendererFns
438 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
439 MachineOperand &Offset, unsigned SizeInBytes,
440 bool WantsExt) const;
441 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
442 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
443 unsigned SizeInBytes) const;
444 template <int Width>
445 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
446 return selectAddrModeXRO(Root, SizeInBytes: Width / 8);
447 }
448
449 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
450 unsigned SizeInBytes) const;
451 template <int Width>
452 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
453 return selectAddrModeWRO(Root, SizeInBytes: Width / 8);
454 }
455
456 ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
457 bool AllowROR = false) const;
458
459 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
460 return selectShiftedRegister(Root);
461 }
462
463 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
464 return selectShiftedRegister(Root, AllowROR: true);
465 }
466
467 /// Given an extend instruction, determine the correct shift-extend type for
468 /// that instruction.
469 ///
470 /// If the instruction is going to be used in a load or store, pass
471 /// \p IsLoadStore = true.
472 AArch64_AM::ShiftExtendType
473 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
474 bool IsLoadStore = false) const;
475
476 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
477 ///
478 /// \returns Either \p Reg if no change was necessary, or the new register
479 /// created by moving \p Reg.
480 ///
481 /// Note: This uses emitCopy right now.
482 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
483 MachineIRBuilder &MIB) const;
484
485 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
486
487 ComplexRendererFns selectExtractHigh(MachineOperand &Root) const;
488 template <unsigned Width>
489 ComplexRendererFns selectCVTFixedPoint(MachineOperand &Root) const;
490 ComplexRendererFns selectCVTFixedPointBase(const MachineOperand &Root,
491 unsigned width,
492 bool isReciprocal = false) const;
493 ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
494 ComplexRendererFns
495 selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
496 void renderFixedPointScalarXForm(MachineInstrBuilder &MIB,
497 const MachineInstr &MI, int OpIdx) const;
498 unsigned getFixedPointWidthFromOperand(const MachineOperand &Root) const;
499 void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
500 int OpIdx = -1) const;
501 void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
502 const MachineInstr &MI, int OpIdx = -1) const;
503 void renderFixedPointImm(MachineInstrBuilder &MIB, const MachineOperand &Root,
504 unsigned Width, bool isReciprocal) const;
505 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
506 int OpIdx = -1) const;
507 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
508 int OpIdx = -1) const;
509 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
510 int OpIdx = -1) const;
511 void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
512 int OpIdx) const;
513 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
514 int OpIdx = -1) const;
515 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
516 int OpIdx = -1) const;
517 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
518 int OpIdx = -1) const;
519 void renderFPImm32SIMDModImmType4(MachineInstrBuilder &MIB,
520 const MachineInstr &MI,
521 int OpIdx = -1) const;
522
523 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
524 void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);
525
526 // Optimization methods.
527 bool tryOptSelect(GSelect &Sel);
528 bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
529 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
530 MachineOperand &Predicate,
531 MachineIRBuilder &MIRBuilder) const;
532
533 /// Return true if \p MI is a load or store of \p NumBytes bytes.
534 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
535
536 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
537 /// register zeroed out. In other words, the result of MI has been explicitly
538 /// zero extended.
539 bool isDef32(const MachineInstr &MI) const;
540
541 const AArch64TargetMachine &TM;
542 const AArch64Subtarget &STI;
543 const AArch64InstrInfo &TII;
544 const AArch64RegisterInfo &TRI;
545 const AArch64RegisterBankInfo &RBI;
546
547 bool ProduceNonFlagSettingCondBr = false;
548
549 // Some cached values used during selection.
550 // We use LR as a live-in register, and we keep track of it here as it can be
551 // clobbered by calls.
552 Register MFReturnAddr;
553
554 MachineIRBuilder MIB;
555
556#define GET_GLOBALISEL_PREDICATES_DECL
557#include "AArch64GenGlobalISel.inc"
558#undef GET_GLOBALISEL_PREDICATES_DECL
559
560// We declare the temporaries used by selectImpl() in the class to minimize the
561// cost of constructing placeholder values.
562#define GET_GLOBALISEL_TEMPORARIES_DECL
563#include "AArch64GenGlobalISel.inc"
564#undef GET_GLOBALISEL_TEMPORARIES_DECL
565};
566
567} // end anonymous namespace
568
569#define GET_GLOBALISEL_IMPL
570#include "AArch64GenGlobalISel.inc"
571#undef GET_GLOBALISEL_IMPL
572
573AArch64InstructionSelector::AArch64InstructionSelector(
574 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
575 const AArch64RegisterBankInfo &RBI)
576 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
577 RBI(RBI),
578#define GET_GLOBALISEL_PREDICATES_INIT
579#include "AArch64GenGlobalISel.inc"
580#undef GET_GLOBALISEL_PREDICATES_INIT
581#define GET_GLOBALISEL_TEMPORARIES_INIT
582#include "AArch64GenGlobalISel.inc"
583#undef GET_GLOBALISEL_TEMPORARIES_INIT
584{
585}
586
587// FIXME: This should be target-independent, inferred from the types declared
588// for each class in the bank.
589//
590/// Given a register bank, and a type, return the smallest register class that
591/// can represent that combination.
592static const TargetRegisterClass *
593getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
594 bool GetAllRegSet = false) {
595 if (RB.getID() == AArch64::GPRRegBankID) {
596 if (Ty.getSizeInBits() <= 32)
597 return GetAllRegSet ? &AArch64::GPR32allRegClass
598 : &AArch64::GPR32RegClass;
599 if (Ty.getSizeInBits() == 64)
600 return GetAllRegSet ? &AArch64::GPR64allRegClass
601 : &AArch64::GPR64RegClass;
602 if (Ty.getSizeInBits() == 128)
603 return &AArch64::XSeqPairsClassRegClass;
604 return nullptr;
605 }
606
607 if (RB.getID() == AArch64::FPRRegBankID) {
608 switch (Ty.getSizeInBits()) {
609 case 8:
610 return &AArch64::FPR8RegClass;
611 case 16:
612 return &AArch64::FPR16RegClass;
613 case 32:
614 return &AArch64::FPR32RegClass;
615 case 64:
616 return &AArch64::FPR64RegClass;
617 case 128:
618 return &AArch64::FPR128RegClass;
619 }
620 return nullptr;
621 }
622
623 return nullptr;
624}
625
626/// Given a register bank, and size in bits, return the smallest register class
627/// that can represent that combination.
628static const TargetRegisterClass *
629getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
630 bool GetAllRegSet = false) {
631 if (SizeInBits.isScalable()) {
632 assert(RB.getID() == AArch64::FPRRegBankID &&
633 "Expected FPR regbank for scalable type size");
634 return &AArch64::ZPRRegClass;
635 }
636
637 unsigned RegBankID = RB.getID();
638
639 if (RegBankID == AArch64::GPRRegBankID) {
640 assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
641 if (SizeInBits <= 32)
642 return GetAllRegSet ? &AArch64::GPR32allRegClass
643 : &AArch64::GPR32RegClass;
644 if (SizeInBits == 64)
645 return GetAllRegSet ? &AArch64::GPR64allRegClass
646 : &AArch64::GPR64RegClass;
647 if (SizeInBits == 128)
648 return &AArch64::XSeqPairsClassRegClass;
649 }
650
651 if (RegBankID == AArch64::FPRRegBankID) {
652 if (SizeInBits.isScalable()) {
653 assert(SizeInBits == TypeSize::getScalable(128) &&
654 "Unexpected scalable register size");
655 return &AArch64::ZPRRegClass;
656 }
657
658 switch (SizeInBits) {
659 default:
660 return nullptr;
661 case 8:
662 return &AArch64::FPR8RegClass;
663 case 16:
664 return &AArch64::FPR16RegClass;
665 case 32:
666 return &AArch64::FPR32RegClass;
667 case 64:
668 return &AArch64::FPR64RegClass;
669 case 128:
670 return &AArch64::FPR128RegClass;
671 }
672 }
673
674 return nullptr;
675}
676
677/// Returns the correct subregister to use for a given register class.
678static bool getSubRegForClass(const TargetRegisterClass *RC,
679 const TargetRegisterInfo &TRI, unsigned &SubReg) {
680 switch (TRI.getRegSizeInBits(RC: *RC)) {
681 case 8:
682 SubReg = AArch64::bsub;
683 break;
684 case 16:
685 SubReg = AArch64::hsub;
686 break;
687 case 32:
688 if (RC != &AArch64::FPR32RegClass)
689 SubReg = AArch64::sub_32;
690 else
691 SubReg = AArch64::ssub;
692 break;
693 case 64:
694 SubReg = AArch64::dsub;
695 break;
696 default:
697 LLVM_DEBUG(
698 dbgs() << "Couldn't find appropriate subregister for register class.");
699 return false;
700 }
701
702 return true;
703}
704
705/// Returns the minimum size the given register bank can hold.
706static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
707 switch (RB.getID()) {
708 case AArch64::GPRRegBankID:
709 return 32;
710 case AArch64::FPRRegBankID:
711 return 8;
712 default:
713 llvm_unreachable("Tried to get minimum size for unknown register bank.");
714 }
715}
716
717/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
718/// Helper function for functions like createDTuple and createQTuple.
719///
720/// \p RegClassIDs - The list of register class IDs available for some tuple of
721/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
722/// expected to contain between 2 and 4 tuple classes.
723///
724/// \p SubRegs - The list of subregister classes associated with each register
725/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
726/// subregister class. The index of each subregister class is expected to
727/// correspond with the index of each register class.
728///
729/// \returns Either the destination register of REG_SEQUENCE instruction that
730/// was created, or the 0th element of \p Regs if \p Regs contains a single
731/// element.
732static Register createTuple(ArrayRef<Register> Regs,
733 const unsigned RegClassIDs[],
734 const unsigned SubRegs[], MachineIRBuilder &MIB) {
735 unsigned NumRegs = Regs.size();
736 if (NumRegs == 1)
737 return Regs[0];
738 assert(NumRegs >= 2 && NumRegs <= 4 &&
739 "Only support between two and 4 registers in a tuple!");
740 const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
741 auto *DesiredClass = TRI->getRegClass(i: RegClassIDs[NumRegs - 2]);
742 auto RegSequence =
743 MIB.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {DesiredClass}, SrcOps: {});
744 for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
745 RegSequence.addUse(RegNo: Regs[I]);
746 RegSequence.addImm(Val: SubRegs[I]);
747 }
748 return RegSequence.getReg(Idx: 0);
749}
750
751/// Create a tuple of D-registers using the registers in \p Regs.
752static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
753 static const unsigned RegClassIDs[] = {
754 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
755 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
756 AArch64::dsub2, AArch64::dsub3};
757 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
758}
759
760/// Create a tuple of Q-registers using the registers in \p Regs.
761static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
762 static const unsigned RegClassIDs[] = {
763 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
764 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
765 AArch64::qsub2, AArch64::qsub3};
766 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
767}
768
769static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
770 auto &MI = *Root.getParent();
771 auto &MBB = *MI.getParent();
772 auto &MF = *MBB.getParent();
773 auto &MRI = MF.getRegInfo();
774 uint64_t Immed;
775 if (Root.isImm())
776 Immed = Root.getImm();
777 else if (Root.isCImm())
778 Immed = Root.getCImm()->getZExtValue();
779 else if (Root.isReg()) {
780 auto ValAndVReg =
781 getIConstantVRegValWithLookThrough(VReg: Root.getReg(), MRI, LookThroughInstrs: true);
782 if (!ValAndVReg)
783 return std::nullopt;
784 Immed = ValAndVReg->Value.getSExtValue();
785 } else
786 return std::nullopt;
787 return Immed;
788}
789
790/// Select the AArch64 opcode for the basic binary operation \p GenericOpc,
791/// appropriate for the register bank \p RegBankID and of size \p OpSize.
792/// \returns \p GenericOpc if the combination is unsupported.
793static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
794 unsigned OpSize) {
795 if (RegBankID == AArch64::GPRRegBankID) {
796 if (OpSize == 32) {
797 switch (GenericOpc) {
798 case TargetOpcode::G_SHL:
799 return AArch64::LSLVWr;
800 case TargetOpcode::G_LSHR:
801 return AArch64::LSRVWr;
802 case TargetOpcode::G_ASHR:
803 return AArch64::ASRVWr;
804 default:
805 return GenericOpc;
806 }
807 } else if (OpSize == 64) {
808 switch (GenericOpc) {
809 case TargetOpcode::G_SHL:
810 return AArch64::LSLVXr;
811 case TargetOpcode::G_LSHR:
812 return AArch64::LSRVXr;
813 case TargetOpcode::G_ASHR:
814 return AArch64::ASRVXr;
815 default:
816 return GenericOpc;
817 }
818 }
819 }
820 return GenericOpc;
821}
822
823/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
824/// appropriate for the (value) register bank \p RegBankID and of memory access
825/// size \p OpSize. This returns the variant with the base+unsigned-immediate
826/// addressing mode (e.g., LDRXui).
827/// \returns \p GenericOpc if the combination is unsupported.
828static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
829 unsigned OpSize) {
830 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
831 switch (RegBankID) {
832 case AArch64::GPRRegBankID:
833 switch (OpSize) {
834 case 8:
835 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
836 case 16:
837 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
838 case 32:
839 return isStore ? AArch64::STRWui : AArch64::LDRWui;
840 case 64:
841 return isStore ? AArch64::STRXui : AArch64::LDRXui;
842 }
843 break;
844 case AArch64::FPRRegBankID:
845 switch (OpSize) {
846 case 8:
847 return isStore ? AArch64::STRBui : AArch64::LDRBui;
848 case 16:
849 return isStore ? AArch64::STRHui : AArch64::LDRHui;
850 case 32:
851 return isStore ? AArch64::STRSui : AArch64::LDRSui;
852 case 64:
853 return isStore ? AArch64::STRDui : AArch64::LDRDui;
854 case 128:
855 return isStore ? AArch64::STRQui : AArch64::LDRQui;
856 }
857 break;
858 }
859 return GenericOpc;
860}
861
862/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
863/// to \p *To.
864///
865/// E.g "To = COPY SrcReg:SubReg"
866static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
867 const RegisterBankInfo &RBI, Register SrcReg,
868 const TargetRegisterClass *To, unsigned SubReg) {
869 assert(SrcReg.isValid() && "Expected a valid source register?");
870 assert(To && "Destination register class cannot be null");
871 assert(SubReg && "Expected a valid subregister");
872
873 MachineIRBuilder MIB(I);
874 auto SubRegCopy =
875 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {To}, SrcOps: {}).addReg(RegNo: SrcReg, Flags: {}, SubReg);
876 MachineOperand &RegOp = I.getOperand(i: 1);
877 RegOp.setReg(SubRegCopy.getReg(Idx: 0));
878
879 // It's possible that the destination register won't be constrained. Make
880 // sure that happens.
881 if (!I.getOperand(i: 0).getReg().isPhysical())
882 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: *To, MRI);
883
884 return true;
885}
886
887// FIXME: We need some sort of API in RBI/TRI to allow generic code to
888// constrain operands of simple instructions given a TargetRegisterClass
889// and LLT
890static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI,
891 const RegisterBankInfo &RBI) {
892 for (MachineOperand &MO : I.operands()) {
893 if (!MO.isReg())
894 continue;
895 Register Reg = MO.getReg();
896 if (!Reg)
897 continue;
898 if (Reg.isPhysical())
899 continue;
900 LLT Ty = MRI.getType(Reg);
901 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
902 const TargetRegisterClass *RC =
903 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
904 if (!RC) {
905 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
906 RC = getRegClassForTypeOnBank(Ty, RB);
907 if (!RC) {
908 LLVM_DEBUG(
909 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
910 break;
911 }
912 }
913 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
914 }
915
916 return true;
917}
918
919static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
920 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
921 const RegisterBankInfo &RBI) {
922 Register DstReg = I.getOperand(i: 0).getReg();
923 Register SrcReg = I.getOperand(i: 1).getReg();
924 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
925 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
926
927 TypeSize DstRegSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
928 TypeSize SrcRegSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
929
930 // Special casing for cross-bank copies of s1s. We can technically represent
931 // a 1-bit value with any size of register. The minimum size for a GPR is 32
932 // bits. So, we need to put the FPR on 32 bits as well.
933 //
934 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
935 // then we can pull it into the helpers that get the appropriate class for a
936 // register bank. Or make a new helper that carries along some constraint
937 // information.
938 if (SrcRegBank != DstRegBank && (DstRegSize == TypeSize::getFixed(ExactSize: 1) &&
939 SrcRegSize == TypeSize::getFixed(ExactSize: 1)))
940 SrcRegSize = DstRegSize = TypeSize::getFixed(ExactSize: 32);
941
942 // Find the correct register classes for the source and destination registers.
943 const TargetRegisterClass *SrcRC =
944 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: SrcRegSize, GetAllRegSet: true);
945 const TargetRegisterClass *DstRC =
946 getMinClassForRegBank(RB: DstRegBank, SizeInBits: DstRegSize, GetAllRegSet: true);
947
948 if (!DstRC) {
949 LLVM_DEBUG(dbgs() << "Unexpected dest size "
950 << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n');
951 return false;
952 }
953
954 if (I.getOpcode() == TargetOpcode::G_BITCAST &&
955 RBI.getSizeInBits(Reg: DstReg, MRI, TRI) == TypeSize::getFixed(ExactSize: 16)) {
956 if (DstRegBank.getID() == AArch64::FPRRegBankID &&
957 SrcRegBank.getID() == AArch64::GPRRegBankID) {
958 if (!SrcReg.isPhysical() &&
959 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass, MRI))
960 return false;
961 if (!DstReg.isPhysical() &&
962 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::FPR16RegClass, MRI))
963 return false;
964
965 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
966 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::FMOVWSr))
967 .addDef(RegNo: FPR32)
968 .addUse(RegNo: SrcReg);
969 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
970 I.getOperand(i: 1).setReg(FPR32);
971 I.getOperand(i: 1).setSubReg(AArch64::hsub);
972 return true;
973 }
974
975 if (DstRegBank.getID() == AArch64::GPRRegBankID &&
976 SrcRegBank.getID() == AArch64::FPRRegBankID) {
977 if (!SrcReg.isPhysical() &&
978 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::FPR16RegClass, MRI))
979 return false;
980 if (!DstReg.isPhysical() &&
981 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR32RegClass, MRI))
982 return false;
983
984 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
985 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
986 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
987 .addDef(RegNo: FPR32)
988 .addUse(RegNo: SrcReg)
989 .addImm(Val: AArch64::hsub);
990 I.setDesc(TII.get(Opcode: AArch64::FMOVSWr));
991 I.getOperand(i: 1).setReg(FPR32);
992 return true;
993 }
994 }
995
996 // Is this a copy? If so, then we may need to insert a subregister copy.
997 if (I.isCopy()) {
998 // Yes. Check if there's anything to fix up.
999 if (!SrcRC) {
1000 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
1001 return false;
1002 }
1003
1004 const TypeSize SrcSize = TRI.getRegSizeInBits(RC: *SrcRC);
1005 const TypeSize DstSize = TRI.getRegSizeInBits(RC: *DstRC);
1006 unsigned SrcSubReg = I.getOperand(i: 1).getSubReg();
1007 unsigned SubReg;
1008
1009 if (SrcSubReg)
1010 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
1011
1012 // If the source bank doesn't support a subregister copy small enough,
1013 // then we first need to copy to the destination bank.
1014 if (getMinSizeForRegBank(RB: SrcRegBank) > DstSize) {
1015 const TargetRegisterClass *DstTempRC =
1016 getMinClassForRegBank(RB: DstRegBank, SizeInBits: SrcSize, /* GetAllRegSet */ true);
1017 getSubRegForClass(RC: DstRC, TRI, SubReg);
1018
1019 MachineIRBuilder MIB(I);
1020 auto Copy = MIB.buildCopy(Res: {DstTempRC}, Op: {SrcReg});
1021 copySubReg(I, MRI, RBI, SrcReg: Copy.getReg(Idx: 0), To: DstRC, SubReg);
1022 } else if (SrcSize > DstSize) {
1023 // If the source register is bigger than the destination we need to
1024 // perform a subregister copy.
1025 const TargetRegisterClass *SubRegRC =
1026 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
1027 getSubRegForClass(RC: SubRegRC, TRI, SubReg);
1028 copySubReg(I, MRI, RBI, SrcReg, To: DstRC, SubReg);
1029 } else if (DstSize > SrcSize) {
1030 // If the destination register is bigger than the source we need to do
1031 // a promotion using SUBREG_TO_REG.
1032 const TargetRegisterClass *PromotionRC =
1033 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
1034 getSubRegForClass(RC: SrcRC, TRI, SubReg);
1035
1036 Register PromoteReg = MRI.createVirtualRegister(RegClass: PromotionRC);
1037 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1038 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG), DestReg: PromoteReg)
1039 .addUse(RegNo: SrcReg)
1040 .addImm(Val: SubReg);
1041 MachineOperand &RegOp = I.getOperand(i: 1);
1042 RegOp.setReg(PromoteReg);
1043 }
1044
1045 // If the destination is a physical register, then there's nothing to
1046 // change, so we're done.
1047 if (DstReg.isPhysical())
1048 return true;
1049 }
1050
1051 // No need to constrain SrcReg. It will get constrained when we hit another
1052 // of its use or its defs. Copies do not have constraints.
1053 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1054 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1055 << " operand\n");
1056 return false;
1057 }
1058
1059 // If this a GPR ZEXT that we want to just reduce down into a copy.
1060 // The sizes will be mismatched with the source < 32b but that's ok.
1061 if (I.getOpcode() == TargetOpcode::G_ZEXT) {
1062 I.setDesc(TII.get(Opcode: AArch64::COPY));
1063 assert(SrcRegBank.getID() == AArch64::GPRRegBankID);
1064 return selectCopy(I, TII, MRI, TRI, RBI);
1065 }
1066
1067 I.setDesc(TII.get(Opcode: AArch64::COPY));
1068 return true;
1069}
1070
1071MachineInstr *
1072AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1073 Register False, AArch64CC::CondCode CC,
1074 MachineIRBuilder &MIB) const {
1075 MachineRegisterInfo &MRI = *MIB.getMRI();
1076 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==
1077 RBI.getRegBank(True, MRI, TRI)->getID() &&
1078 "Expected both select operands to have the same regbank?");
1079 LLT Ty = MRI.getType(Reg: True);
1080 if (Ty.isVector())
1081 return nullptr;
1082 const unsigned Size = Ty.getSizeInBits();
1083 assert((Size == 32 || Size == 64) &&
1084 "Expected 32 bit or 64 bit select only?");
1085 const bool Is32Bit = Size == 32;
1086 if (RBI.getRegBank(Reg: True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1087 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1088 auto FCSel = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1089 constrainSelectedInstRegOperands(I&: *FCSel, TII, TRI, RBI);
1090 return &*FCSel;
1091 }
1092
1093 // By default, we'll try and emit a CSEL.
1094 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1095 bool Optimized = false;
1096 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1097 &Optimized](Register &Reg, Register &OtherReg,
1098 bool Invert) {
1099 if (Optimized)
1100 return false;
1101
1102 // Attempt to fold:
1103 //
1104 // %sub = G_SUB 0, %x
1105 // %select = G_SELECT cc, %reg, %sub
1106 //
1107 // Into:
1108 // %select = CSNEG %reg, %x, cc
1109 Register MatchReg;
1110 if (mi_match(R: Reg, MRI, P: m_Neg(Src: m_Reg(R&: MatchReg)))) {
1111 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1112 Reg = MatchReg;
1113 if (Invert) {
1114 CC = AArch64CC::getInvertedCondCode(Code: CC);
1115 std::swap(a&: Reg, b&: OtherReg);
1116 }
1117 return true;
1118 }
1119
1120 // Attempt to fold:
1121 //
1122 // %xor = G_XOR %x, -1
1123 // %select = G_SELECT cc, %reg, %xor
1124 //
1125 // Into:
1126 // %select = CSINV %reg, %x, cc
1127 if (mi_match(R: Reg, MRI, P: m_Not(Src: m_Reg(R&: MatchReg)))) {
1128 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1129 Reg = MatchReg;
1130 if (Invert) {
1131 CC = AArch64CC::getInvertedCondCode(Code: CC);
1132 std::swap(a&: Reg, b&: OtherReg);
1133 }
1134 return true;
1135 }
1136
1137 // Attempt to fold:
1138 //
1139 // %add = G_ADD %x, 1
1140 // %select = G_SELECT cc, %reg, %add
1141 //
1142 // Into:
1143 // %select = CSINC %reg, %x, cc
1144 if (mi_match(R: Reg, MRI,
1145 P: m_any_of(preds: m_GAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1)),
1146 preds: m_GPtrAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1))))) {
1147 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1148 Reg = MatchReg;
1149 if (Invert) {
1150 CC = AArch64CC::getInvertedCondCode(Code: CC);
1151 std::swap(a&: Reg, b&: OtherReg);
1152 }
1153 return true;
1154 }
1155
1156 return false;
1157 };
1158
1159 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1160 // true/false values are constants.
1161 // FIXME: All of these patterns already exist in tablegen. We should be
1162 // able to import these.
1163 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1164 &Optimized]() {
1165 if (Optimized)
1166 return false;
1167 auto TrueCst = getIConstantVRegValWithLookThrough(VReg: True, MRI);
1168 auto FalseCst = getIConstantVRegValWithLookThrough(VReg: False, MRI);
1169 if (!TrueCst && !FalseCst)
1170 return false;
1171
1172 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1173 if (TrueCst && FalseCst) {
1174 int64_t T = TrueCst->Value.getSExtValue();
1175 int64_t F = FalseCst->Value.getSExtValue();
1176
1177 if (T == 0 && F == 1) {
1178 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1179 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1180 True = ZReg;
1181 False = ZReg;
1182 return true;
1183 }
1184
1185 if (T == 0 && F == -1) {
1186 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1187 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1188 True = ZReg;
1189 False = ZReg;
1190 return true;
1191 }
1192 }
1193
1194 if (TrueCst) {
1195 int64_t T = TrueCst->Value.getSExtValue();
1196 if (T == 1) {
1197 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1198 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1199 True = False;
1200 False = ZReg;
1201 CC = AArch64CC::getInvertedCondCode(Code: CC);
1202 return true;
1203 }
1204
1205 if (T == -1) {
1206 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1207 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1208 True = False;
1209 False = ZReg;
1210 CC = AArch64CC::getInvertedCondCode(Code: CC);
1211 return true;
1212 }
1213 }
1214
1215 if (FalseCst) {
1216 int64_t F = FalseCst->Value.getSExtValue();
1217 if (F == 1) {
1218 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1219 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1220 False = ZReg;
1221 return true;
1222 }
1223
1224 if (F == -1) {
1225 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1226 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1227 False = ZReg;
1228 return true;
1229 }
1230 }
1231 return false;
1232 };
1233
1234 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1235 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1236 Optimized |= TryOptSelectCst();
1237 auto SelectInst = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1238 constrainSelectedInstRegOperands(I&: *SelectInst, TII, TRI, RBI);
1239 return &*SelectInst;
1240}
1241
1242static AArch64CC::CondCode
1243changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS = {},
1244 MachineRegisterInfo *MRI = nullptr) {
1245 switch (P) {
1246 default:
1247 llvm_unreachable("Unknown condition code!");
1248 case CmpInst::ICMP_NE:
1249 return AArch64CC::NE;
1250 case CmpInst::ICMP_EQ:
1251 return AArch64CC::EQ;
1252 case CmpInst::ICMP_SGT:
1253 return AArch64CC::GT;
1254 case CmpInst::ICMP_SGE:
1255 if (RHS && MRI) {
1256 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1257 if (ValAndVReg && ValAndVReg->Value == 0)
1258 return AArch64CC::PL;
1259 }
1260 return AArch64CC::GE;
1261 case CmpInst::ICMP_SLT:
1262 if (RHS && MRI) {
1263 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1264 if (ValAndVReg && ValAndVReg->Value == 0)
1265 return AArch64CC::MI;
1266 }
1267 return AArch64CC::LT;
1268 case CmpInst::ICMP_SLE:
1269 return AArch64CC::LE;
1270 case CmpInst::ICMP_UGT:
1271 return AArch64CC::HI;
1272 case CmpInst::ICMP_UGE:
1273 return AArch64CC::HS;
1274 case CmpInst::ICMP_ULT:
1275 return AArch64CC::LO;
1276 case CmpInst::ICMP_ULE:
1277 return AArch64CC::LS;
1278 }
1279}
1280
1281/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1282static void changeFPCCToORAArch64CC(CmpInst::Predicate CC,
1283 AArch64CC::CondCode &CondCode,
1284 AArch64CC::CondCode &CondCode2) {
1285 CondCode2 = AArch64CC::AL;
1286 switch (CC) {
1287 default:
1288 llvm_unreachable("Unknown FP condition!");
1289 case CmpInst::FCMP_OEQ:
1290 CondCode = AArch64CC::EQ;
1291 break;
1292 case CmpInst::FCMP_OGT:
1293 CondCode = AArch64CC::GT;
1294 break;
1295 case CmpInst::FCMP_OGE:
1296 CondCode = AArch64CC::GE;
1297 break;
1298 case CmpInst::FCMP_OLT:
1299 CondCode = AArch64CC::MI;
1300 break;
1301 case CmpInst::FCMP_OLE:
1302 CondCode = AArch64CC::LS;
1303 break;
1304 case CmpInst::FCMP_ONE:
1305 CondCode = AArch64CC::MI;
1306 CondCode2 = AArch64CC::GT;
1307 break;
1308 case CmpInst::FCMP_ORD:
1309 CondCode = AArch64CC::VC;
1310 break;
1311 case CmpInst::FCMP_UNO:
1312 CondCode = AArch64CC::VS;
1313 break;
1314 case CmpInst::FCMP_UEQ:
1315 CondCode = AArch64CC::EQ;
1316 CondCode2 = AArch64CC::VS;
1317 break;
1318 case CmpInst::FCMP_UGT:
1319 CondCode = AArch64CC::HI;
1320 break;
1321 case CmpInst::FCMP_UGE:
1322 CondCode = AArch64CC::PL;
1323 break;
1324 case CmpInst::FCMP_ULT:
1325 CondCode = AArch64CC::LT;
1326 break;
1327 case CmpInst::FCMP_ULE:
1328 CondCode = AArch64CC::LE;
1329 break;
1330 case CmpInst::FCMP_UNE:
1331 CondCode = AArch64CC::NE;
1332 break;
1333 }
1334}
1335
1336/// Convert an IR fp condition code to an AArch64 CC.
1337/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1338/// should be AND'ed instead of OR'ed.
1339static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC,
1340 AArch64CC::CondCode &CondCode,
1341 AArch64CC::CondCode &CondCode2) {
1342 CondCode2 = AArch64CC::AL;
1343 switch (CC) {
1344 default:
1345 changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
1346 assert(CondCode2 == AArch64CC::AL);
1347 break;
1348 case CmpInst::FCMP_ONE:
1349 // (a one b)
1350 // == ((a olt b) || (a ogt b))
1351 // == ((a ord b) && (a une b))
1352 CondCode = AArch64CC::VC;
1353 CondCode2 = AArch64CC::NE;
1354 break;
1355 case CmpInst::FCMP_UEQ:
1356 // (a ueq b)
1357 // == ((a uno b) || (a oeq b))
1358 // == ((a ule b) && (a uge b))
1359 CondCode = AArch64CC::PL;
1360 CondCode2 = AArch64CC::LE;
1361 break;
1362 }
1363}
1364
1365/// Return a register which can be used as a bit to test in a TB(N)Z.
1366static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1367 MachineRegisterInfo &MRI) {
1368 assert(Reg.isValid() && "Expected valid register!");
1369 bool HasZext = false;
1370 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1371 unsigned Opc = MI->getOpcode();
1372
1373 if (!MI->getOperand(i: 0).isReg() ||
1374 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1375 break;
1376
1377 // (tbz (any_ext x), b) -> (tbz x, b) and
1378 // (tbz (zext x), b) -> (tbz x, b) if we don't use the extended bits.
1379 //
1380 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1381 // on the truncated x is the same as the bit number on x.
1382 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1383 Opc == TargetOpcode::G_TRUNC) {
1384 if (Opc == TargetOpcode::G_ZEXT)
1385 HasZext = true;
1386
1387 Register NextReg = MI->getOperand(i: 1).getReg();
1388 // Did we find something worth folding?
1389 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(RegNo: NextReg))
1390 break;
1391 TypeSize InSize = MRI.getType(Reg: NextReg).getSizeInBits();
1392 if (Bit >= InSize)
1393 break;
1394
1395 // NextReg is worth folding. Keep looking.
1396 Reg = NextReg;
1397 continue;
1398 }
1399
1400 // Attempt to find a suitable operation with a constant on one side.
1401 std::optional<uint64_t> C;
1402 Register TestReg;
1403 switch (Opc) {
1404 default:
1405 break;
1406 case TargetOpcode::G_AND:
1407 case TargetOpcode::G_XOR: {
1408 TestReg = MI->getOperand(i: 1).getReg();
1409 Register ConstantReg = MI->getOperand(i: 2).getReg();
1410 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1411 if (!VRegAndVal) {
1412 // AND commutes, check the other side for a constant.
1413 // FIXME: Can we canonicalize the constant so that it's always on the
1414 // same side at some point earlier?
1415 std::swap(a&: ConstantReg, b&: TestReg);
1416 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1417 }
1418 if (VRegAndVal) {
1419 if (HasZext)
1420 C = VRegAndVal->Value.getZExtValue();
1421 else
1422 C = VRegAndVal->Value.getSExtValue();
1423 }
1424 break;
1425 }
1426 case TargetOpcode::G_ASHR:
1427 case TargetOpcode::G_LSHR:
1428 case TargetOpcode::G_SHL: {
1429 TestReg = MI->getOperand(i: 1).getReg();
1430 auto VRegAndVal =
1431 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
1432 if (VRegAndVal)
1433 C = VRegAndVal->Value.getSExtValue();
1434 break;
1435 }
1436 }
1437
1438 // Didn't find a constant or viable register. Bail out of the loop.
1439 if (!C || !TestReg.isValid())
1440 break;
1441
1442 // We found a suitable instruction with a constant. Check to see if we can
1443 // walk through the instruction.
1444 Register NextReg;
1445 unsigned TestRegSize = MRI.getType(Reg: TestReg).getSizeInBits();
1446 switch (Opc) {
1447 default:
1448 break;
1449 case TargetOpcode::G_AND:
1450 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1451 if ((*C >> Bit) & 1)
1452 NextReg = TestReg;
1453 break;
1454 case TargetOpcode::G_SHL:
1455 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1456 // the type of the register.
1457 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1458 NextReg = TestReg;
1459 Bit = Bit - *C;
1460 }
1461 break;
1462 case TargetOpcode::G_ASHR:
1463 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1464 // in x
1465 NextReg = TestReg;
1466 Bit = Bit + *C;
1467 if (Bit >= TestRegSize)
1468 Bit = TestRegSize - 1;
1469 break;
1470 case TargetOpcode::G_LSHR:
1471 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1472 if ((Bit + *C) < TestRegSize) {
1473 NextReg = TestReg;
1474 Bit = Bit + *C;
1475 }
1476 break;
1477 case TargetOpcode::G_XOR:
1478 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1479 // appropriate.
1480 //
1481 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1482 //
1483 // tbz x', b -> tbnz x, b
1484 //
1485 // Because x' only has the b-th bit set if x does not.
1486 if ((*C >> Bit) & 1)
1487 Invert = !Invert;
1488 NextReg = TestReg;
1489 break;
1490 }
1491
1492 // Check if we found anything worth folding.
1493 if (!NextReg.isValid())
1494 return Reg;
1495 Reg = NextReg;
1496 }
1497
1498 return Reg;
1499}
1500
1501MachineInstr *AArch64InstructionSelector::emitTestBit(
1502 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1503 MachineIRBuilder &MIB) const {
1504 assert(TestReg.isValid());
1505 assert(ProduceNonFlagSettingCondBr &&
1506 "Cannot emit TB(N)Z with speculation tracking!");
1507 MachineRegisterInfo &MRI = *MIB.getMRI();
1508
1509 // Attempt to optimize the test bit by walking over instructions.
1510 TestReg = getTestBitReg(Reg: TestReg, Bit, Invert&: IsNegative, MRI);
1511 LLT Ty = MRI.getType(Reg: TestReg);
1512 unsigned Size = Ty.getSizeInBits();
1513 assert(!Ty.isVector() && "Expected a scalar!");
1514 assert(Bit < 64 && "Bit is too large!");
1515
1516 // When the test register is a 64-bit register, we have to narrow to make
1517 // TBNZW work.
1518 bool UseWReg = Bit < 32;
1519 unsigned NecessarySize = UseWReg ? 32 : 64;
1520 if (Size != NecessarySize)
1521 TestReg = moveScalarRegClass(
1522 Reg: TestReg, RC: UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1523 MIB);
1524
1525 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1526 {AArch64::TBZW, AArch64::TBNZW}};
1527 unsigned Opc = OpcTable[UseWReg][IsNegative];
1528 auto TestBitMI =
1529 MIB.buildInstr(Opcode: Opc).addReg(RegNo: TestReg).addImm(Val: Bit).addMBB(MBB: DstMBB);
1530 constrainSelectedInstRegOperands(I&: *TestBitMI, TII, TRI, RBI);
1531 return &*TestBitMI;
1532}
1533
1534bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1535 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1536 MachineIRBuilder &MIB) const {
1537 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
1538 // Given something like this:
1539 //
1540 // %x = ...Something...
1541 // %one = G_CONSTANT i64 1
1542 // %zero = G_CONSTANT i64 0
1543 // %and = G_AND %x, %one
1544 // %cmp = G_ICMP intpred(ne), %and, %zero
1545 // %cmp_trunc = G_TRUNC %cmp
1546 // G_BRCOND %cmp_trunc, %bb.3
1547 //
1548 // We want to try and fold the AND into the G_BRCOND and produce either a
1549 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1550 //
1551 // In this case, we'd get
1552 //
1553 // TBNZ %x %bb.3
1554 //
1555
1556 // Check if the AND has a constant on its RHS which we can use as a mask.
1557 // If it's a power of 2, then it's the same as checking a specific bit.
1558 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1559 auto MaybeBit = getIConstantVRegValWithLookThrough(
1560 VReg: AndInst.getOperand(i: 2).getReg(), MRI: *MIB.getMRI());
1561 if (!MaybeBit)
1562 return false;
1563
1564 int32_t Bit = MaybeBit->Value.exactLogBase2();
1565 if (Bit < 0)
1566 return false;
1567
1568 Register TestReg = AndInst.getOperand(i: 1).getReg();
1569
1570 // Emit a TB(N)Z.
1571 emitTestBit(TestReg, Bit, IsNegative: Invert, DstMBB, MIB);
1572 return true;
1573}
1574
1575MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1576 bool IsNegative,
1577 MachineBasicBlock *DestMBB,
1578 MachineIRBuilder &MIB) const {
1579 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
1580 MachineRegisterInfo &MRI = *MIB.getMRI();
1581 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
1582 AArch64::GPRRegBankID &&
1583 "Expected GPRs only?");
1584 auto Ty = MRI.getType(Reg: CompareReg);
1585 unsigned Width = Ty.getSizeInBits();
1586 assert(!Ty.isVector() && "Expected scalar only?");
1587 assert(Width <= 64 && "Expected width to be at most 64?");
1588 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1589 {AArch64::CBNZW, AArch64::CBNZX}};
1590 unsigned Opc = OpcTable[IsNegative][Width == 64];
1591 auto BranchMI = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {CompareReg}).addMBB(MBB: DestMBB);
1592 constrainSelectedInstRegOperands(I&: *BranchMI, TII, TRI, RBI);
1593 return &*BranchMI;
1594}
1595
1596bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1597 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1598 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
1599 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1600 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1601 // totally clean. Some of them require two branches to implement.
1602 auto Pred = (CmpInst::Predicate)FCmp.getOperand(i: 1).getPredicate();
1603 emitFPCompare(LHS: FCmp.getOperand(i: 2).getReg(), RHS: FCmp.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
1604 Pred);
1605 AArch64CC::CondCode CC1, CC2;
1606 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
1607 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1608 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC1).addMBB(MBB: DestMBB);
1609 if (CC2 != AArch64CC::AL)
1610 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC2).addMBB(MBB: DestMBB);
1611 I.eraseFromParent();
1612 return true;
1613}
1614
1615bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1616 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1617 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1618 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1619 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1620 //
1621 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1622 // instructions will not be produced, as they are conditional branch
1623 // instructions that do not set flags.
1624 if (!ProduceNonFlagSettingCondBr)
1625 return false;
1626
1627 MachineRegisterInfo &MRI = *MIB.getMRI();
1628 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1629 auto Pred =
1630 static_cast<CmpInst::Predicate>(ICmp.getOperand(i: 1).getPredicate());
1631 Register LHS = ICmp.getOperand(i: 2).getReg();
1632 Register RHS = ICmp.getOperand(i: 3).getReg();
1633
1634 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1635 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1636 MachineInstr *AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1637
1638 // When we can emit a TB(N)Z, prefer that.
1639 //
1640 // Handle non-commutative condition codes first.
1641 // Note that we don't want to do this when we have a G_AND because it can
1642 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1643 if (VRegAndVal && !AndInst) {
1644 int64_t C = VRegAndVal->Value.getSExtValue();
1645
1646 // When we have a greater-than comparison, we can just test if the msb is
1647 // zero.
1648 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1649 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1650 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1651 I.eraseFromParent();
1652 return true;
1653 }
1654
1655 // When we have a less than comparison, we can just test if the msb is not
1656 // zero.
1657 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1658 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1659 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ true, DstMBB: DestMBB, MIB);
1660 I.eraseFromParent();
1661 return true;
1662 }
1663
1664 // Inversely, if we have a signed greater-than-or-equal comparison to zero,
1665 // we can test if the msb is zero.
1666 if (C == 0 && Pred == CmpInst::ICMP_SGE) {
1667 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1668 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1669 I.eraseFromParent();
1670 return true;
1671 }
1672 }
1673
1674 // Attempt to handle commutative condition codes. Right now, that's only
1675 // eq/ne.
1676 if (ICmpInst::isEquality(P: Pred)) {
1677 if (!VRegAndVal) {
1678 std::swap(a&: RHS, b&: LHS);
1679 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1680 AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1681 }
1682
1683 if (VRegAndVal && VRegAndVal->Value == 0) {
1684 // If there's a G_AND feeding into this branch, try to fold it away by
1685 // emitting a TB(N)Z instead.
1686 //
1687 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1688 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1689 // would be redundant.
1690 if (AndInst &&
1691 tryOptAndIntoCompareBranch(
1692 AndInst&: *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DstMBB: DestMBB, MIB)) {
1693 I.eraseFromParent();
1694 return true;
1695 }
1696
1697 // Otherwise, try to emit a CB(N)Z instead.
1698 auto LHSTy = MRI.getType(Reg: LHS);
1699 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1700 emitCBZ(CompareReg: LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1701 I.eraseFromParent();
1702 return true;
1703 }
1704 }
1705 }
1706
1707 return false;
1708}
1709
1710bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1711 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1712 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1713 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1714 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1715 return true;
1716
1717 // Couldn't optimize. Emit a compare + a Bcc.
1718 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1719 auto &PredOp = ICmp.getOperand(i: 1);
1720 emitIntegerCompare(LHS&: ICmp.getOperand(i: 2), RHS&: ICmp.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
1721 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
1722 P: static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
1723 RHS: ICmp.getOperand(i: 3).getReg(), MRI: MIB.getMRI());
1724 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC).addMBB(MBB: DestMBB);
1725 I.eraseFromParent();
1726 return true;
1727}
1728
1729bool AArch64InstructionSelector::selectCompareBranch(
1730 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
1731 Register CondReg = I.getOperand(i: 0).getReg();
1732 MachineInstr *CCMI = MRI.getVRegDef(Reg: CondReg);
1733 // Try to select the G_BRCOND using whatever is feeding the condition if
1734 // possible.
1735 unsigned CCMIOpc = CCMI->getOpcode();
1736 if (CCMIOpc == TargetOpcode::G_FCMP)
1737 return selectCompareBranchFedByFCmp(I, FCmp&: *CCMI, MIB);
1738 if (CCMIOpc == TargetOpcode::G_ICMP)
1739 return selectCompareBranchFedByICmp(I, ICmp&: *CCMI, MIB);
1740
1741 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1742 // instructions will not be produced, as they are conditional branch
1743 // instructions that do not set flags.
1744 if (ProduceNonFlagSettingCondBr) {
1745 emitTestBit(TestReg: CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1746 DstMBB: I.getOperand(i: 1).getMBB(), MIB);
1747 I.eraseFromParent();
1748 return true;
1749 }
1750
1751 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1752 auto TstMI =
1753 MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {LLT::scalar(SizeInBits: 32)}, SrcOps: {CondReg}).addImm(Val: 1);
1754 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
1755 auto Bcc = MIB.buildInstr(Opcode: AArch64::Bcc)
1756 .addImm(Val: AArch64CC::NE)
1757 .addMBB(MBB: I.getOperand(i: 1).getMBB());
1758 I.eraseFromParent();
1759 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1760 return true;
1761}
1762
1763/// Returns the element immediate value of a vector shift operand if found.
1764/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1765static std::optional<int64_t> getVectorShiftImm(Register Reg,
1766 MachineRegisterInfo &MRI) {
1767 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand");
1768 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1769 return getAArch64VectorSplatScalar(MI: *OpMI, MRI);
1770}
1771
1772/// Matches and returns the shift immediate value for a SHL instruction given
1773/// a shift operand.
1774static std::optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
1775 MachineRegisterInfo &MRI) {
1776 std::optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
1777 if (!ShiftImm)
1778 return std::nullopt;
1779 // Check the immediate is in range for a SHL.
1780 int64_t Imm = *ShiftImm;
1781 if (Imm < 0)
1782 return std::nullopt;
1783 switch (SrcTy.getElementType().getSizeInBits()) {
1784 default:
1785 LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift");
1786 return std::nullopt;
1787 case 8:
1788 if (Imm > 7)
1789 return std::nullopt;
1790 break;
1791 case 16:
1792 if (Imm > 15)
1793 return std::nullopt;
1794 break;
1795 case 32:
1796 if (Imm > 31)
1797 return std::nullopt;
1798 break;
1799 case 64:
1800 if (Imm > 63)
1801 return std::nullopt;
1802 break;
1803 }
1804 return Imm;
1805}
1806
1807bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
1808 MachineRegisterInfo &MRI) {
1809 assert(I.getOpcode() == TargetOpcode::G_SHL);
1810 Register DstReg = I.getOperand(i: 0).getReg();
1811 const LLT Ty = MRI.getType(Reg: DstReg);
1812 Register Src1Reg = I.getOperand(i: 1).getReg();
1813 Register Src2Reg = I.getOperand(i: 2).getReg();
1814
1815 if (!Ty.isVector())
1816 return false;
1817
1818 // Check if we have a vector of constants on RHS that we can select as the
1819 // immediate form.
1820 std::optional<int64_t> ImmVal = getVectorSHLImm(SrcTy: Ty, Reg: Src2Reg, MRI);
1821
1822 unsigned Opc = 0;
1823 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1824 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1825 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1826 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1827 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1828 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1829 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1830 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1831 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1832 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1833 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1834 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1835 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1836 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1837 } else {
1838 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type");
1839 return false;
1840 }
1841
1842 auto Shl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg});
1843 if (ImmVal)
1844 Shl.addImm(Val: *ImmVal);
1845 else
1846 Shl.addUse(RegNo: Src2Reg);
1847 constrainSelectedInstRegOperands(I&: *Shl, TII, TRI, RBI);
1848 I.eraseFromParent();
1849 return true;
1850}
1851
1852bool AArch64InstructionSelector::selectVectorAshrLshr(
1853 MachineInstr &I, MachineRegisterInfo &MRI) {
1854 assert(I.getOpcode() == TargetOpcode::G_ASHR ||
1855 I.getOpcode() == TargetOpcode::G_LSHR);
1856 Register DstReg = I.getOperand(i: 0).getReg();
1857 const LLT Ty = MRI.getType(Reg: DstReg);
1858 Register Src1Reg = I.getOperand(i: 1).getReg();
1859 Register Src2Reg = I.getOperand(i: 2).getReg();
1860
1861 if (!Ty.isVector())
1862 return false;
1863
1864 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1865
1866 // We expect the immediate case to be lowered in the PostLegalCombiner to
1867 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1868
1869 // There is not a shift right register instruction, but the shift left
1870 // register instruction takes a signed value, where negative numbers specify a
1871 // right shift.
1872
1873 unsigned Opc = 0;
1874 unsigned NegOpc = 0;
1875 const TargetRegisterClass *RC =
1876 getRegClassForTypeOnBank(Ty, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID));
1877 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1878 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1879 NegOpc = AArch64::NEGv2i64;
1880 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1881 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1882 NegOpc = AArch64::NEGv4i32;
1883 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1884 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1885 NegOpc = AArch64::NEGv2i32;
1886 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1887 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1888 NegOpc = AArch64::NEGv4i16;
1889 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1890 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1891 NegOpc = AArch64::NEGv8i16;
1892 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1893 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1894 NegOpc = AArch64::NEGv16i8;
1895 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1896 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1897 NegOpc = AArch64::NEGv8i8;
1898 } else {
1899 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type");
1900 return false;
1901 }
1902
1903 auto Neg = MIB.buildInstr(Opc: NegOpc, DstOps: {RC}, SrcOps: {Src2Reg});
1904 constrainSelectedInstRegOperands(I&: *Neg, TII, TRI, RBI);
1905 auto SShl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg, Neg});
1906 constrainSelectedInstRegOperands(I&: *SShl, TII, TRI, RBI);
1907 I.eraseFromParent();
1908 return true;
1909}
1910
1911bool AArch64InstructionSelector::selectVaStartAAPCS(
1912 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1913
1914 if (STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
1915 IsVarArg: MF.getFunction().isVarArg()))
1916 return false;
1917
1918 // The layout of the va_list struct is specified in the AArch64 Procedure Call
1919 // Standard, section 10.1.5.
1920
1921 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1922 const unsigned PtrSize = STI.isTargetILP32() ? 4 : 8;
1923 const auto *PtrRegClass =
1924 STI.isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1925
1926 const MCInstrDesc &MCIDAddAddr =
1927 TII.get(Opcode: STI.isTargetILP32() ? AArch64::ADDWri : AArch64::ADDXri);
1928 const MCInstrDesc &MCIDStoreAddr =
1929 TII.get(Opcode: STI.isTargetILP32() ? AArch64::STRWui : AArch64::STRXui);
1930
1931 /*
1932 * typedef struct va_list {
1933 * void * stack; // next stack param
1934 * void * gr_top; // end of GP arg reg save area
1935 * void * vr_top; // end of FP/SIMD arg reg save area
1936 * int gr_offs; // offset from gr_top to next GP register arg
1937 * int vr_offs; // offset from vr_top to next FP/SIMD register arg
1938 * } va_list;
1939 */
1940 const auto VAList = I.getOperand(i: 0).getReg();
1941
1942 // Our current offset in bytes from the va_list struct (VAList).
1943 unsigned OffsetBytes = 0;
1944
1945 // Helper function to store (FrameIndex + Imm) to VAList at offset OffsetBytes
1946 // and increment OffsetBytes by PtrSize.
1947 const auto PushAddress = [&](const int FrameIndex, const int64_t Imm) {
1948 const Register Top = MRI.createVirtualRegister(RegClass: PtrRegClass);
1949 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDAddAddr)
1950 .addDef(RegNo: Top)
1951 .addFrameIndex(Idx: FrameIndex)
1952 .addImm(Val: Imm)
1953 .addImm(Val: 0);
1954 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1955
1956 const auto *MMO = *I.memoperands_begin();
1957 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDStoreAddr)
1958 .addUse(RegNo: Top)
1959 .addUse(RegNo: VAList)
1960 .addImm(Val: OffsetBytes / PtrSize)
1961 .addMemOperand(MMO: MF.getMachineMemOperand(
1962 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1963 F: MachineMemOperand::MOStore, Size: PtrSize, BaseAlignment: MMO->getBaseAlign()));
1964 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1965
1966 OffsetBytes += PtrSize;
1967 };
1968
1969 // void* stack at offset 0
1970 PushAddress(FuncInfo->getVarArgsStackIndex(), 0);
1971
1972 // void* gr_top at offset 8 (4 on ILP32)
1973 const unsigned GPRSize = FuncInfo->getVarArgsGPRSize();
1974 PushAddress(FuncInfo->getVarArgsGPRIndex(), GPRSize);
1975
1976 // void* vr_top at offset 16 (8 on ILP32)
1977 const unsigned FPRSize = FuncInfo->getVarArgsFPRSize();
1978 PushAddress(FuncInfo->getVarArgsFPRIndex(), FPRSize);
1979
1980 // Helper function to store a 4-byte integer constant to VAList at offset
1981 // OffsetBytes, and increment OffsetBytes by 4.
1982 const auto PushIntConstant = [&](const int32_t Value) {
1983 constexpr int IntSize = 4;
1984 const Register Temp = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
1985 auto MIB =
1986 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVi32imm))
1987 .addDef(RegNo: Temp)
1988 .addImm(Val: Value);
1989 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1990
1991 const auto *MMO = *I.memoperands_begin();
1992 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRWui))
1993 .addUse(RegNo: Temp)
1994 .addUse(RegNo: VAList)
1995 .addImm(Val: OffsetBytes / IntSize)
1996 .addMemOperand(MMO: MF.getMachineMemOperand(
1997 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1998 F: MachineMemOperand::MOStore, Size: IntSize, BaseAlignment: MMO->getBaseAlign()));
1999 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2000 OffsetBytes += IntSize;
2001 };
2002
2003 // int gr_offs at offset 24 (12 on ILP32)
2004 PushIntConstant(-static_cast<int32_t>(GPRSize));
2005
2006 // int vr_offs at offset 28 (16 on ILP32)
2007 PushIntConstant(-static_cast<int32_t>(FPRSize));
2008
2009 assert(OffsetBytes == (STI.isTargetILP32() ? 20 : 32) && "Unexpected offset");
2010
2011 I.eraseFromParent();
2012 return true;
2013}
2014
2015bool AArch64InstructionSelector::selectVaStartDarwin(
2016 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
2017 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
2018 Register ListReg = I.getOperand(i: 0).getReg();
2019
2020 Register ArgsAddrReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2021
2022 int FrameIdx = FuncInfo->getVarArgsStackIndex();
2023 if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
2024 CC: MF.getFunction().getCallingConv(), IsVarArg: MF.getFunction().isVarArg())) {
2025 FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
2026 ? FuncInfo->getVarArgsGPRIndex()
2027 : FuncInfo->getVarArgsStackIndex();
2028 }
2029
2030 auto MIB =
2031 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
2032 .addDef(RegNo: ArgsAddrReg)
2033 .addFrameIndex(Idx: FrameIdx)
2034 .addImm(Val: 0)
2035 .addImm(Val: 0);
2036
2037 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2038
2039 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRXui))
2040 .addUse(RegNo: ArgsAddrReg)
2041 .addUse(RegNo: ListReg)
2042 .addImm(Val: 0)
2043 .addMemOperand(MMO: *I.memoperands_begin());
2044
2045 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2046 I.eraseFromParent();
2047 return true;
2048}
2049
2050void AArch64InstructionSelector::materializeLargeCMVal(
2051 MachineInstr &I, const Value *V, unsigned OpFlags) {
2052 MachineBasicBlock &MBB = *I.getParent();
2053 MachineFunction &MF = *MBB.getParent();
2054 MachineRegisterInfo &MRI = MF.getRegInfo();
2055
2056 auto MovZ = MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {});
2057 MovZ->addOperand(MF, Op: I.getOperand(i: 1));
2058 MovZ->getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
2059 AArch64II::MO_NC);
2060 MovZ->addOperand(MF, Op: MachineOperand::CreateImm(Val: 0));
2061 constrainSelectedInstRegOperands(I&: *MovZ, TII, TRI, RBI);
2062
2063 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
2064 Register ForceDstReg) {
2065 Register DstReg = ForceDstReg
2066 ? ForceDstReg
2067 : MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2068 auto MovI = MIB.buildInstr(Opcode: AArch64::MOVKXi).addDef(RegNo: DstReg).addUse(RegNo: SrcReg);
2069 if (auto *GV = dyn_cast<GlobalValue>(Val: V)) {
2070 MovI->addOperand(MF, Op: MachineOperand::CreateGA(
2071 GV, Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2072 } else {
2073 MovI->addOperand(
2074 MF, Op: MachineOperand::CreateBA(BA: cast<BlockAddress>(Val: V),
2075 Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2076 }
2077 MovI->addOperand(MF, Op: MachineOperand::CreateImm(Val: Offset));
2078 constrainSelectedInstRegOperands(I&: *MovI, TII, TRI, RBI);
2079 return DstReg;
2080 };
2081 Register DstReg = BuildMovK(MovZ.getReg(Idx: 0),
2082 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
2083 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
2084 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(i: 0).getReg());
2085}
2086
2087bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
2088 MachineBasicBlock &MBB = *I.getParent();
2089 MachineFunction &MF = *MBB.getParent();
2090 MachineRegisterInfo &MRI = MF.getRegInfo();
2091
2092 switch (I.getOpcode()) {
2093 case TargetOpcode::G_CONSTANT: {
2094 Register DefReg = I.getOperand(i: 0).getReg();
2095 const LLT DefTy = MRI.getType(Reg: DefReg);
2096 if (!DefTy.isPointer()) {
2097 if (DefTy.getSizeInBits() >= 32 ||
2098 RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
2099 return false;
2100 // Widen narrow GPR constants to s32 so imported patterns can match.
2101 APInt Val = I.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
2102 I.getOperand(i: 1).setCImm(
2103 ConstantInt::get(Context&: MF.getFunction().getContext(), V: Val));
2104
2105 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2106 MRI.setRegBank(Reg: WideReg, RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2107 I.getOperand(i: 0).setReg(WideReg);
2108
2109 MIB.setInsertPt(MBB, II: std::next(x: I.getIterator()));
2110 auto Copy = MIB.buildCopy(Res: DefReg, Op: WideReg);
2111 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
2112 MIB.setInstr(I);
2113 return true;
2114 }
2115 const unsigned PtrSize = DefTy.getSizeInBits();
2116 if (PtrSize != 32 && PtrSize != 64)
2117 return false;
2118 // Convert pointer typed constants to integers so TableGen can select.
2119 MRI.setType(VReg: DefReg, Ty: LLT::integer(SizeInBits: PtrSize));
2120 return true;
2121 }
2122 case TargetOpcode::G_STORE: {
2123 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
2124 MachineOperand &SrcOp = I.getOperand(i: 0);
2125 if (MRI.getType(Reg: SrcOp.getReg()).isPointer()) {
2126 // Allow matching with imported patterns for stores of pointers. Unlike
2127 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
2128 // and constrain.
2129 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: SrcOp);
2130 Register NewSrc = Copy.getReg(Idx: 0);
2131 SrcOp.setReg(NewSrc);
2132 RBI.constrainGenericRegister(Reg: NewSrc, RC: AArch64::GPR64RegClass, MRI);
2133 Changed = true;
2134 }
2135 return Changed;
2136 }
2137 case TargetOpcode::G_PTR_ADD: {
2138 // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
2139 // arithmetic semantics instead of falling back to regular arithmetic.
2140 const auto &TL = STI.getTargetLowering();
2141 if (TL->shouldPreservePtrArith(F: MF.getFunction(), PtrVT: EVT()))
2142 return false;
2143 return convertPtrAddToAdd(I, MRI);
2144 }
2145 case TargetOpcode::G_LOAD: {
2146 // For scalar loads of pointers, we try to convert the dest type from p0
2147 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
2148 // conversion, this should be ok because all users should have been
2149 // selected already, so the type doesn't matter for them.
2150 Register DstReg = I.getOperand(i: 0).getReg();
2151 const LLT DstTy = MRI.getType(Reg: DstReg);
2152 if (!DstTy.isPointer())
2153 return false;
2154 MRI.setType(VReg: DstReg, Ty: LLT::scalar(SizeInBits: 64));
2155 return true;
2156 }
2157 case TargetOpcode::G_VECREDUCE_ADD:
2158 case TargetOpcode::G_VECREDUCE_SMAX:
2159 case TargetOpcode::G_VECREDUCE_SMIN:
2160 case TargetOpcode::G_VECREDUCE_UMAX:
2161 case TargetOpcode::G_VECREDUCE_UMIN: {
2162 // Imported patterns require an FPR result. For a GPR, use a temporary FPR
2163 // and insert a cross-bank copy.
2164 Register DstReg = I.getOperand(i: 0).getReg();
2165 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2166 if (DstRB.getID() != AArch64::GPRRegBankID)
2167 return false;
2168
2169 LLT DstTy = MRI.getType(Reg: DstReg);
2170 const TargetRegisterClass *DstRC =
2171 getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB, /*GetAllRegSet=*/true);
2172 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI))
2173 return false;
2174
2175 Register FPRDst = MRI.createGenericVirtualRegister(Ty: DstTy);
2176 MRI.setRegBank(Reg: FPRDst, RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2177 I.getOperand(i: 0).setReg(FPRDst);
2178
2179 BuildMI(BB&: MBB, I: std::next(x: I.getIterator()), MIMD: MIMetadata(I),
2180 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
2181 .addReg(RegNo: FPRDst);
2182 return true;
2183 }
2184 case AArch64::G_DUP: {
2185 // Convert the type from p0 to s64 to help selection.
2186 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2187 if (!DstTy.isPointerVector())
2188 return false;
2189 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 1).getReg());
2190 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2191 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2192 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2193 I.getOperand(i: 1).setReg(NewSrc.getReg(Idx: 0));
2194 return true;
2195 }
2196 case AArch64::G_INSERT_VECTOR_ELT: {
2197 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2198 LLT SrcVecTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
2199 if (SrcVecTy.isPointerVector()) {
2200 // Convert the type from p0 to s64 to help selection.
2201 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 2).getReg());
2202 MRI.setType(VReg: I.getOperand(i: 1).getReg(),
2203 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2204 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2205 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2206 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2207 I.getOperand(i: 2).setReg(NewSrc.getReg(Idx: 0));
2208 return true;
2209 }
2210
2211 Register EltReg = I.getOperand(i: 2).getReg();
2212 LLT EltTy = MRI.getType(Reg: EltReg);
2213 if (EltTy.isScalar() &&
2214 (EltTy.getSizeInBits() == 8 || EltTy.getSizeInBits() == 16) &&
2215 RBI.getRegBank(Reg: EltReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) {
2216 // Convert the type from s8/s16 to s32 to help selection.
2217 auto NewElt = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 32), Op: EltReg);
2218 MRI.setRegClass(Reg: NewElt.getReg(Idx: 0), RC: &AArch64::GPR32RegClass);
2219 I.getOperand(i: 2).setReg(NewElt.getReg(Idx: 0));
2220 return true;
2221 }
2222 return false;
2223 }
2224 case TargetOpcode::G_UITOFP:
2225 case TargetOpcode::G_SITOFP: {
2226 // If both source and destination regbanks are FPR, then convert the opcode
2227 // to G_SITOF so that the importer can select it to an fpr variant.
2228 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
2229 // copy.
2230 Register SrcReg = I.getOperand(i: 1).getReg();
2231 LLT SrcTy = MRI.getType(Reg: SrcReg);
2232 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2233 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
2234 return false;
2235
2236 if (RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
2237 // Need to add a copy to change the type so that the existing patterns can
2238 // match when there is an integer on an FPR bank.
2239 if (SrcTy.getScalarType().isInteger()) {
2240 auto Copy = MIB.buildCopy(Res: DstTy, Op: SrcReg);
2241 I.getOperand(i: 1).setReg(Copy.getReg(Idx: 0));
2242 MRI.setRegClass(Reg: Copy.getReg(Idx: 0),
2243 RC: getRegClassForTypeOnBank(
2244 Ty: SrcTy, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID)));
2245 }
2246 if (I.getOpcode() == TargetOpcode::G_SITOFP)
2247 I.setDesc(TII.get(Opcode: AArch64::G_SITOF));
2248 else
2249 I.setDesc(TII.get(Opcode: AArch64::G_UITOF));
2250 return true;
2251 }
2252 return false;
2253 }
2254 default:
2255 return false;
2256 }
2257}
2258
2259/// This lowering tries to look for G_PTR_ADD instructions and then converts
2260/// them to a standard G_ADD with a COPY on the source.
2261///
2262/// The motivation behind this is to expose the add semantics to the imported
2263/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2264/// because the selector works bottom up, uses before defs. By the time we
2265/// end up trying to select a G_PTR_ADD, we should have already attempted to
2266/// fold this into addressing modes and were therefore unsuccessful.
2267bool AArch64InstructionSelector::convertPtrAddToAdd(
2268 MachineInstr &I, MachineRegisterInfo &MRI) {
2269 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
2270 Register DstReg = I.getOperand(i: 0).getReg();
2271 Register AddOp1Reg = I.getOperand(i: 1).getReg();
2272 const LLT PtrTy = MRI.getType(Reg: DstReg);
2273 if (PtrTy.getAddressSpace() != 0)
2274 return false;
2275
2276 const LLT CastPtrTy = PtrTy.isVector()
2277 ? LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 64))
2278 : LLT::integer(SizeInBits: 64);
2279 auto PtrToInt = MIB.buildPtrToInt(Dst: CastPtrTy, Src: AddOp1Reg);
2280 // Set regbanks on the registers.
2281 if (PtrTy.isVector())
2282 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2283 else
2284 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2285
2286 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2287 // %dst(intty) = G_ADD %intbase, off
2288 I.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
2289 MRI.setType(VReg: DstReg, Ty: CastPtrTy);
2290 I.getOperand(i: 1).setReg(PtrToInt.getReg(Idx: 0));
2291 if (!select(I&: *PtrToInt)) {
2292 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd");
2293 return false;
2294 }
2295
2296 // Also take the opportunity here to try to do some optimization.
2297 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2298 Register NegatedReg;
2299 if (!mi_match(R: I.getOperand(i: 2).getReg(), MRI, P: m_Neg(Src: m_Reg(R&: NegatedReg))))
2300 return true;
2301 I.getOperand(i: 2).setReg(NegatedReg);
2302 I.setDesc(TII.get(Opcode: TargetOpcode::G_SUB));
2303 return true;
2304}
2305
2306bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
2307 MachineRegisterInfo &MRI) {
2308 // We try to match the immediate variant of LSL, which is actually an alias
2309 // for a special case of UBFM. Otherwise, we fall back to the imported
2310 // selector which will match the register variant.
2311 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op");
2312 const auto &MO = I.getOperand(i: 2);
2313 auto VRegAndVal = getIConstantVRegVal(VReg: MO.getReg(), MRI);
2314 if (!VRegAndVal)
2315 return false;
2316
2317 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2318 if (DstTy.isVector())
2319 return false;
2320 bool Is64Bit = DstTy.getSizeInBits() == 64;
2321 auto Imm1Fn = Is64Bit ? selectShiftA_64(Root: MO) : selectShiftA_32(Root: MO);
2322 auto Imm2Fn = Is64Bit ? selectShiftB_64(Root: MO) : selectShiftB_32(Root: MO);
2323
2324 if (!Imm1Fn || !Imm2Fn)
2325 return false;
2326
2327 auto NewI =
2328 MIB.buildInstr(Opc: Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2329 DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {I.getOperand(i: 1).getReg()});
2330
2331 for (auto &RenderFn : *Imm1Fn)
2332 RenderFn(NewI);
2333 for (auto &RenderFn : *Imm2Fn)
2334 RenderFn(NewI);
2335
2336 I.eraseFromParent();
2337 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
2338 return true;
2339}
2340
2341bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2342 MachineInstr &I, MachineRegisterInfo &MRI) {
2343 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE");
2344 // If we're storing a scalar, it doesn't matter what register bank that
2345 // scalar is on. All that matters is the size.
2346 //
2347 // So, if we see something like this (with a 32-bit scalar as an example):
2348 //
2349 // %x:gpr(s32) = ... something ...
2350 // %y:fpr(s32) = COPY %x:gpr(s32)
2351 // G_STORE %y:fpr(s32)
2352 //
2353 // We can fix this up into something like this:
2354 //
2355 // G_STORE %x:gpr(s32)
2356 //
2357 // And then continue the selection process normally.
2358 Register DefDstReg = getSrcRegIgnoringCopies(Reg: I.getOperand(i: 0).getReg(), MRI);
2359 if (!DefDstReg.isValid())
2360 return false;
2361 LLT DefDstTy = MRI.getType(Reg: DefDstReg);
2362 Register StoreSrcReg = I.getOperand(i: 0).getReg();
2363 LLT StoreSrcTy = MRI.getType(Reg: StoreSrcReg);
2364
2365 // If we get something strange like a physical register, then we shouldn't
2366 // go any further.
2367 if (!DefDstTy.isValid())
2368 return false;
2369
2370 // Are the source and dst types the same size?
2371 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2372 return false;
2373
2374 if (RBI.getRegBank(Reg: StoreSrcReg, MRI, TRI) ==
2375 RBI.getRegBank(Reg: DefDstReg, MRI, TRI))
2376 return false;
2377
2378 // We have a cross-bank copy, which is entering a store. Let's fold it.
2379 I.getOperand(i: 0).setReg(DefDstReg);
2380 return true;
2381}
2382
2383bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
2384 assert(I.getParent() && "Instruction should be in a basic block!");
2385 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2386
2387 MachineBasicBlock &MBB = *I.getParent();
2388 MachineFunction &MF = *MBB.getParent();
2389 MachineRegisterInfo &MRI = MF.getRegInfo();
2390
2391 switch (I.getOpcode()) {
2392 case AArch64::G_DUP: {
2393 // Before selecting a DUP instruction, check if it is better selected as a
2394 // MOV or load from a constant pool.
2395 Register Src = I.getOperand(i: 1).getReg();
2396 auto ValAndVReg = getAnyConstantVRegValWithLookThrough(
2397 VReg: Src, MRI, /*LookThroughInstrs=*/true, /*LookThroughAnyExt=*/true);
2398 if (!ValAndVReg)
2399 return false;
2400 LLVMContext &Ctx = MF.getFunction().getContext();
2401 Register Dst = I.getOperand(i: 0).getReg();
2402 auto *CV = ConstantDataVector::getSplat(
2403 NumElts: MRI.getType(Reg: Dst).getNumElements(),
2404 Elt: ConstantInt::get(
2405 Ty: Type::getIntNTy(C&: Ctx, N: MRI.getType(Reg: Dst).getScalarSizeInBits()),
2406 V: ValAndVReg->Value.trunc(width: MRI.getType(Reg: Dst).getScalarSizeInBits())));
2407 if (!emitConstantVector(Dst, CV, MIRBuilder&: MIB, MRI))
2408 return false;
2409 I.eraseFromParent();
2410 return true;
2411 }
2412 case TargetOpcode::G_SEXT:
2413 // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
2414 // over a normal extend.
2415 if (selectUSMovFromExtend(I, MRI))
2416 return true;
2417 return false;
2418 case TargetOpcode::G_BR:
2419 return false;
2420 case TargetOpcode::G_SHL:
2421 return earlySelectSHL(I, MRI);
2422 case TargetOpcode::G_CONSTANT: {
2423 bool IsZero = false;
2424 if (I.getOperand(i: 1).isCImm())
2425 IsZero = I.getOperand(i: 1).getCImm()->isZero();
2426 else if (I.getOperand(i: 1).isImm())
2427 IsZero = I.getOperand(i: 1).getImm() == 0;
2428
2429 if (!IsZero)
2430 return false;
2431
2432 Register DefReg = I.getOperand(i: 0).getReg();
2433 LLT Ty = MRI.getType(Reg: DefReg);
2434 if (Ty.getSizeInBits() == 64) {
2435 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::XZR, isDef: false);
2436 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
2437 } else if (Ty.getSizeInBits() <= 32) {
2438 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::WZR, isDef: false);
2439 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR32RegClass, MRI);
2440 } else
2441 return false;
2442
2443 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2444 return true;
2445 }
2446
2447 case TargetOpcode::G_ADD: {
2448 // Check if this is being fed by a G_ICMP on either side.
2449 //
2450 // (cmp pred, x, y) + z
2451 //
2452 // In the above case, when the cmp is true, we increment z by 1. So, we can
2453 // fold the add into the cset for the cmp by using cinc.
2454 //
2455 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2456 Register AddDst = I.getOperand(i: 0).getReg();
2457 Register AddLHS = I.getOperand(i: 1).getReg();
2458 Register AddRHS = I.getOperand(i: 2).getReg();
2459 // Only handle scalars.
2460 LLT Ty = MRI.getType(Reg: AddLHS);
2461 if (Ty.isVector())
2462 return false;
2463 // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
2464 // bits.
2465 unsigned Size = Ty.getSizeInBits();
2466 if (Size != 32 && Size != 64)
2467 return false;
2468 auto MatchCmp = [&](Register Reg) -> MachineInstr * {
2469 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2470 return nullptr;
2471 // If the LHS of the add is 32 bits, then we want to fold a 32-bit
2472 // compare.
2473 if (Size == 32)
2474 return getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg, MRI);
2475 // We model scalar compares using 32-bit destinations right now.
2476 // If it's a 64-bit compare, it'll have 64-bit sources.
2477 Register ZExt;
2478 if (!mi_match(R: Reg, MRI,
2479 P: m_OneNonDBGUse(SP: m_GZExt(Src: m_OneNonDBGUse(SP: m_Reg(R&: ZExt))))))
2480 return nullptr;
2481 auto *Cmp = getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg: ZExt, MRI);
2482 if (!Cmp ||
2483 MRI.getType(Reg: Cmp->getOperand(i: 2).getReg()).getSizeInBits() != 64)
2484 return nullptr;
2485 return Cmp;
2486 };
2487 // Try to match
2488 // z + (cmp pred, x, y)
2489 MachineInstr *Cmp = MatchCmp(AddRHS);
2490 if (!Cmp) {
2491 // (cmp pred, x, y) + z
2492 std::swap(a&: AddLHS, b&: AddRHS);
2493 Cmp = MatchCmp(AddRHS);
2494 if (!Cmp)
2495 return false;
2496 }
2497 auto &PredOp = Cmp->getOperand(i: 1);
2498 MIB.setInstrAndDebugLoc(I);
2499 emitIntegerCompare(/*LHS=*/Cmp->getOperand(i: 2),
2500 /*RHS=*/Cmp->getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
2501 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
2502 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
2503 P: CmpInst::getInversePredicate(pred: Pred), RHS: Cmp->getOperand(i: 3).getReg(), MRI: &MRI);
2504 emitCSINC(/*Dst=*/AddDst, /*Src =*/Src1: AddLHS, /*Src2=*/AddLHS, Pred: InvCC, MIRBuilder&: MIB);
2505 I.eraseFromParent();
2506 return true;
2507 }
2508 case TargetOpcode::G_OR: {
2509 // Look for operations that take the lower `Width=Size-ShiftImm` bits of
2510 // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
2511 // shifting and masking that we can replace with a BFI (encoded as a BFM).
2512 Register Dst = I.getOperand(i: 0).getReg();
2513 LLT Ty = MRI.getType(Reg: Dst);
2514
2515 if (!Ty.isScalar())
2516 return false;
2517
2518 unsigned Size = Ty.getSizeInBits();
2519 if (Size != 32 && Size != 64)
2520 return false;
2521
2522 Register ShiftSrc;
2523 int64_t ShiftImm;
2524 Register MaskSrc;
2525 int64_t MaskImm;
2526 if (!mi_match(
2527 R: Dst, MRI,
2528 P: m_GOr(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))),
2529 R: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: MaskSrc), R: m_ICst(Cst&: MaskImm))))))
2530 return false;
2531
2532 if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
2533 return false;
2534
2535 int64_t Immr = Size - ShiftImm;
2536 int64_t Imms = Size - ShiftImm - 1;
2537 unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2538 emitInstr(Opcode: Opc, DstOps: {Dst}, SrcOps: {MaskSrc, ShiftSrc, Immr, Imms}, MIRBuilder&: MIB);
2539 I.eraseFromParent();
2540 return true;
2541 }
2542 case TargetOpcode::G_FENCE: {
2543 if (I.getOperand(i: 1).getImm() == 0)
2544 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2545 else
2546 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: AArch64::DMB))
2547 .addImm(Val: I.getOperand(i: 0).getImm() == 4 ? 0x9 : 0xb);
2548 I.eraseFromParent();
2549 return true;
2550 }
2551 default:
2552 return false;
2553 }
2554}
2555
2556bool AArch64InstructionSelector::select(MachineInstr &I) {
2557 assert(I.getParent() && "Instruction should be in a basic block!");
2558 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2559
2560 MachineBasicBlock &MBB = *I.getParent();
2561 MachineFunction &MF = *MBB.getParent();
2562 MachineRegisterInfo &MRI = MF.getRegInfo();
2563
2564 const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
2565 if (Subtarget->requiresStrictAlign()) {
2566 // We don't support this feature yet.
2567 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
2568 return false;
2569 }
2570
2571 MIB.setInstrAndDebugLoc(I);
2572
2573 unsigned Opcode = I.getOpcode();
2574 // G_PHI requires same handling as PHI
2575 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2576 // Certain non-generic instructions also need some special handling.
2577
2578 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2579 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2580 return true;
2581 }
2582
2583 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2584 const Register DefReg = I.getOperand(i: 0).getReg();
2585 const LLT DefTy = MRI.getType(Reg: DefReg);
2586
2587 const RegClassOrRegBank &RegClassOrBank =
2588 MRI.getRegClassOrRegBank(Reg: DefReg);
2589
2590 const TargetRegisterClass *DefRC =
2591 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
2592 if (!DefRC) {
2593 if (!DefTy.isValid()) {
2594 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
2595 return false;
2596 }
2597 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
2598 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
2599 if (!DefRC) {
2600 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
2601 return false;
2602 }
2603 }
2604
2605 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
2606
2607 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI);
2608 }
2609
2610 if (I.isCopy())
2611 return selectCopy(I, TII, MRI, TRI, RBI);
2612
2613 if (I.isDebugInstr())
2614 return selectDebugInstr(I, MRI, RBI);
2615
2616 return true;
2617 }
2618
2619
2620 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2621 LLVM_DEBUG(
2622 dbgs() << "Generic instruction has unexpected implicit operands\n");
2623 return false;
2624 }
2625
2626 // Try to do some lowering before we start instruction selecting. These
2627 // lowerings are purely transformations on the input G_MIR and so selection
2628 // must continue after any modification of the instruction.
2629 if (preISelLower(I)) {
2630 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2631 }
2632
2633 // There may be patterns where the importer can't deal with them optimally,
2634 // but does select it to a suboptimal sequence so our custom C++ selection
2635 // code later never has a chance to work on it. Therefore, we have an early
2636 // selection attempt here to give priority to certain selection routines
2637 // over the imported ones.
2638 if (earlySelect(I))
2639 return true;
2640
2641 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2642 return true;
2643
2644 LLT Ty =
2645 I.getOperand(i: 0).isReg() ? MRI.getType(Reg: I.getOperand(i: 0).getReg()) : LLT{};
2646
2647 switch (Opcode) {
2648 case TargetOpcode::G_SBFX:
2649 case TargetOpcode::G_UBFX: {
2650 static const unsigned OpcTable[2][2] = {
2651 {AArch64::UBFMWri, AArch64::UBFMXri},
2652 {AArch64::SBFMWri, AArch64::SBFMXri}};
2653 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2654 unsigned Size = Ty.getSizeInBits();
2655 unsigned Opc = OpcTable[IsSigned][Size == 64];
2656 auto Cst1 =
2657 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 2).getReg(), MRI);
2658 assert(Cst1 && "Should have gotten a constant for src 1?");
2659 auto Cst2 =
2660 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 3).getReg(), MRI);
2661 assert(Cst2 && "Should have gotten a constant for src 2?");
2662 auto LSB = Cst1->Value.getZExtValue();
2663 auto Width = Cst2->Value.getZExtValue();
2664 auto BitfieldInst =
2665 MIB.buildInstr(Opc, DstOps: {I.getOperand(i: 0)}, SrcOps: {I.getOperand(i: 1)})
2666 .addImm(Val: LSB)
2667 .addImm(Val: LSB + Width - 1);
2668 I.eraseFromParent();
2669 constrainSelectedInstRegOperands(I&: *BitfieldInst, TII, TRI, RBI);
2670 return true;
2671 }
2672 case TargetOpcode::G_BRCOND:
2673 return selectCompareBranch(I, MF, MRI);
2674
2675 case TargetOpcode::G_BRINDIRECT: {
2676 const Function &Fn = MF.getFunction();
2677 if (std::optional<uint16_t> BADisc =
2678 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn)) {
2679 auto MI = MIB.buildInstr(Opc: AArch64::BRA, DstOps: {}, SrcOps: {I.getOperand(i: 0).getReg()});
2680 MI.addImm(Val: AArch64PACKey::IA);
2681 MI.addImm(Val: *BADisc);
2682 MI.addReg(/*AddrDisc=*/RegNo: AArch64::XZR);
2683 I.eraseFromParent();
2684 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
2685 return true;
2686 }
2687 I.setDesc(TII.get(Opcode: AArch64::BR));
2688 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2689 return true;
2690 }
2691
2692 case TargetOpcode::G_BRJT:
2693 return selectBrJT(I, MRI);
2694
2695 case AArch64::G_ADD_LOW: {
2696 // This op may have been separated from it's ADRP companion by the localizer
2697 // or some other code motion pass. Given that many CPUs will try to
2698 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2699 // which will later be expanded into an ADRP+ADD pair after scheduling.
2700 MachineInstr *BaseMI = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
2701 if (BaseMI->getOpcode() != AArch64::ADRP) {
2702 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2703 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2704 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2705 return true;
2706 }
2707 assert(TM.getCodeModel() == CodeModel::Small &&
2708 "Expected small code model");
2709 auto Op1 = BaseMI->getOperand(i: 1);
2710 auto Op2 = I.getOperand(i: 2);
2711 auto MovAddr = MIB.buildInstr(Opc: AArch64::MOVaddr, DstOps: {I.getOperand(i: 0)}, SrcOps: {})
2712 .addGlobalAddress(GV: Op1.getGlobal(), Offset: Op1.getOffset(),
2713 TargetFlags: Op1.getTargetFlags())
2714 .addGlobalAddress(GV: Op2.getGlobal(), Offset: Op2.getOffset(),
2715 TargetFlags: Op2.getTargetFlags());
2716 I.eraseFromParent();
2717 constrainSelectedInstRegOperands(I&: *MovAddr, TII, TRI, RBI);
2718 return true;
2719 }
2720
2721 case TargetOpcode::G_FCONSTANT: {
2722 const Register DefReg = I.getOperand(i: 0).getReg();
2723 const LLT DefTy = MRI.getType(Reg: DefReg);
2724 const unsigned DefSize = DefTy.getSizeInBits();
2725 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
2726
2727 const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(Ty: DefTy, RB);
2728 // For 16, 64, and 128b values, emit a constant pool load.
2729 switch (DefSize) {
2730 default:
2731 llvm_unreachable("Unexpected destination size for G_FCONSTANT?");
2732 case 32:
2733 case 64: {
2734 bool OptForSize = shouldOptForSize(MF: &MF);
2735 const auto &TLI = MF.getSubtarget().getTargetLowering();
2736 // If TLI says that this fpimm is illegal, then we'll expand to a
2737 // constant pool load.
2738 if (TLI->isFPImmLegal(I.getOperand(i: 1).getFPImm()->getValueAPF(),
2739 EVT::getFloatingPointVT(BitWidth: DefSize), ForCodeSize: OptForSize))
2740 break;
2741 [[fallthrough]];
2742 }
2743 case 16:
2744 case 128: {
2745 auto *FPImm = I.getOperand(i: 1).getFPImm();
2746 auto *LoadMI = emitLoadFromConstantPool(CPVal: FPImm, MIRBuilder&: MIB);
2747 if (!LoadMI) {
2748 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n");
2749 return false;
2750 }
2751 MIB.buildCopy(Res: {DefReg}, Op: {LoadMI->getOperand(i: 0).getReg()});
2752 I.eraseFromParent();
2753 return RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2754 }
2755 }
2756
2757 assert((DefSize == 32 || DefSize == 64) && "Unexpected const def size");
2758 // Either emit a FMOV, or emit a copy to emit a normal mov.
2759 const Register DefGPRReg = MRI.createVirtualRegister(
2760 RegClass: DefSize == 32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2761 MachineOperand &RegOp = I.getOperand(i: 0);
2762 RegOp.setReg(DefGPRReg);
2763 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2764 MIB.buildCopy(Res: {DefReg}, Op: {DefGPRReg});
2765
2766 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI)) {
2767 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
2768 return false;
2769 }
2770
2771 MachineOperand &ImmOp = I.getOperand(i: 1);
2772 ImmOp.ChangeToImmediate(
2773 ImmVal: ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2774
2775 const unsigned MovOpc =
2776 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2777 I.setDesc(TII.get(Opcode: MovOpc));
2778 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2779 return true;
2780 }
2781 case TargetOpcode::G_EXTRACT: {
2782 Register DstReg = I.getOperand(i: 0).getReg();
2783 Register SrcReg = I.getOperand(i: 1).getReg();
2784 LLT SrcTy = MRI.getType(Reg: SrcReg);
2785 LLT DstTy = MRI.getType(Reg: DstReg);
2786 (void)DstTy;
2787 unsigned SrcSize = SrcTy.getSizeInBits();
2788
2789 if (SrcTy.getSizeInBits() > 64) {
2790 // This should be an extract of an s128, which is like a vector extract.
2791 if (SrcTy.getSizeInBits() != 128)
2792 return false;
2793 // Only support extracting 64 bits from an s128 at the moment.
2794 if (DstTy.getSizeInBits() != 64)
2795 return false;
2796
2797 unsigned Offset = I.getOperand(i: 2).getImm();
2798 if (Offset % 64 != 0)
2799 return false;
2800
2801 // Check we have the right regbank always.
2802 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
2803 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2804 assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!");
2805
2806 if (SrcRB.getID() == AArch64::GPRRegBankID) {
2807 auto NewI =
2808 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
2809 .addUse(RegNo: SrcReg, Flags: {},
2810 SubReg: Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2811 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: *NewI,
2812 RegClass: AArch64::GPR64RegClass, RegMO&: NewI->getOperand(i: 0));
2813 I.eraseFromParent();
2814 return true;
2815 }
2816
2817 // Emit the same code as a vector extract.
2818 // Offset must be a multiple of 64.
2819 unsigned LaneIdx = Offset / 64;
2820 MachineInstr *Extract = emitExtractVectorElt(
2821 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: 64), VecReg: SrcReg, LaneIdx, MIRBuilder&: MIB);
2822 if (!Extract)
2823 return false;
2824 I.eraseFromParent();
2825 return true;
2826 }
2827
2828 I.setDesc(TII.get(Opcode: SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2829 MachineInstrBuilder(MF, I).addImm(Val: I.getOperand(i: 2).getImm() +
2830 Ty.getSizeInBits() - 1);
2831
2832 if (SrcSize < 64) {
2833 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
2834 "unexpected G_EXTRACT types");
2835 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2836 return true;
2837 }
2838
2839 DstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2840 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2841 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
2842 .addReg(RegNo: DstReg, Flags: {}, SubReg: AArch64::sub_32);
2843 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
2844 RC: AArch64::GPR32RegClass, MRI);
2845 I.getOperand(i: 0).setReg(DstReg);
2846
2847 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2848 return true;
2849 }
2850
2851 case TargetOpcode::G_INSERT: {
2852 LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 2).getReg());
2853 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2854 unsigned DstSize = DstTy.getSizeInBits();
2855 // Larger inserts are vectors, same-size ones should be something else by
2856 // now (split up or turned into COPYs).
2857 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2858 return false;
2859
2860 I.setDesc(TII.get(Opcode: DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2861 unsigned LSB = I.getOperand(i: 3).getImm();
2862 unsigned Width = MRI.getType(Reg: I.getOperand(i: 2).getReg()).getSizeInBits();
2863 I.getOperand(i: 3).setImm((DstSize - LSB) % DstSize);
2864 MachineInstrBuilder(MF, I).addImm(Val: Width - 1);
2865
2866 if (DstSize < 64) {
2867 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
2868 "unexpected G_INSERT types");
2869 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2870 return true;
2871 }
2872
2873 Register SrcReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2874 BuildMI(BB&: MBB, I: I.getIterator(), MIMD: I.getDebugLoc(),
2875 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
2876 .addDef(RegNo: SrcReg)
2877 .addUse(RegNo: I.getOperand(i: 2).getReg())
2878 .addImm(Val: AArch64::sub_32);
2879 RBI.constrainGenericRegister(Reg: I.getOperand(i: 2).getReg(),
2880 RC: AArch64::GPR32RegClass, MRI);
2881 I.getOperand(i: 2).setReg(SrcReg);
2882
2883 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2884 return true;
2885 }
2886 case TargetOpcode::G_FRAME_INDEX: {
2887 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2888 if (Ty != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2889 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
2890 << ", expected: " << LLT::pointer(0, 64) << '\n');
2891 return false;
2892 }
2893 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2894
2895 // MOs for a #0 shifted immediate.
2896 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2897 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2898
2899 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2900 return true;
2901 }
2902
2903 case TargetOpcode::G_GLOBAL_VALUE: {
2904 const GlobalValue *GV = nullptr;
2905 unsigned OpFlags;
2906 if (I.getOperand(i: 1).isSymbol()) {
2907 OpFlags = I.getOperand(i: 1).getTargetFlags();
2908 // Currently only used by "RtLibUseGOT".
2909 assert(OpFlags == AArch64II::MO_GOT);
2910 } else {
2911 GV = I.getOperand(i: 1).getGlobal();
2912 if (GV->isThreadLocal())
2913 return selectTLSGlobalValue(I, MRI);
2914
2915 OpFlags = STI.ClassifyGlobalReference(GV, TM);
2916 }
2917
2918 if (OpFlags & AArch64II::MO_GOT) {
2919 bool IsGOTSigned = MF.getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2920 I.setDesc(TII.get(Opcode: IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2921 I.getOperand(i: 1).setTargetFlags(OpFlags);
2922 I.addImplicitDefUseOperands(MF);
2923 } else if (TM.getCodeModel() == CodeModel::Large &&
2924 !TM.isPositionIndependent()) {
2925 // Materialize the global using movz/movk instructions.
2926 materializeLargeCMVal(I, V: GV, OpFlags);
2927 I.eraseFromParent();
2928 return true;
2929 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2930 I.setDesc(TII.get(Opcode: AArch64::ADR));
2931 I.getOperand(i: 1).setTargetFlags(OpFlags);
2932 } else {
2933 I.setDesc(TII.get(Opcode: AArch64::MOVaddr));
2934 I.getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2935 MachineInstrBuilder MIB(MF, I);
2936 MIB.addGlobalAddress(GV, Offset: I.getOperand(i: 1).getOffset(),
2937 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2938 }
2939 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2940 return true;
2941 }
2942
2943 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2944 return selectPtrAuthGlobalValue(I, MRI);
2945
2946 case TargetOpcode::G_ZEXTLOAD:
2947 case TargetOpcode::G_LOAD:
2948 case TargetOpcode::G_STORE: {
2949 GLoadStore &LdSt = cast<GLoadStore>(Val&: I);
2950 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2951 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
2952
2953 // Can only handle AddressSpace 0, 64-bit pointers.
2954 if (PtrTy != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2955 return false;
2956 }
2957
2958 uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
2959 unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
2960 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
2961
2962 // Need special instructions for atomics that affect ordering.
2963 if (isStrongerThanMonotonic(AO: Order)) {
2964 assert(!isa<GZExtLoad>(LdSt));
2965 assert(MemSizeInBytes <= 8 &&
2966 "128-bit atomics should already be custom-legalized");
2967
2968 if (isa<GLoad>(Val: LdSt)) {
2969 static constexpr unsigned LDAPROpcodes[] = {
2970 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2971 static constexpr unsigned LDAROpcodes[] = {
2972 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2973 ArrayRef<unsigned> Opcodes =
2974 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2975 ? LDAPROpcodes
2976 : LDAROpcodes;
2977 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2978 } else {
2979 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2980 AArch64::STLRW, AArch64::STLRX};
2981 Register ValReg = LdSt.getReg(Idx: 0);
2982 if (MRI.getType(Reg: ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2983 // Emit a subreg copy of 32 bits.
2984 Register NewVal = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
2985 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {NewVal}, SrcOps: {})
2986 .addReg(RegNo: I.getOperand(i: 0).getReg(), Flags: {}, SubReg: AArch64::sub_32);
2987 I.getOperand(i: 0).setReg(NewVal);
2988 }
2989 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2990 }
2991 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2992 return true;
2993 }
2994
2995#ifndef NDEBUG
2996 const Register PtrReg = LdSt.getPointerReg();
2997 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2998 // Check that the pointer register is valid.
2999 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
3000 "Load/Store pointer operand isn't a GPR");
3001 assert(MRI.getType(PtrReg).isPointer() &&
3002 "Load/Store pointer operand isn't a pointer");
3003#endif
3004
3005 const Register ValReg = LdSt.getReg(Idx: 0);
3006 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI, TRI);
3007 LLT ValTy = MRI.getType(Reg: ValReg);
3008
3009 // The code below doesn't support truncating stores, so we need to split it
3010 // again.
3011 if (isa<GStore>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits &&
3012 RB.getID() == AArch64::FPRRegBankID) {
3013 unsigned SubReg;
3014 LLT MemTy = LdSt.getMMO().getMemoryType();
3015 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
3016 if (!getSubRegForClass(RC, TRI, SubReg))
3017 return false;
3018
3019 // Generate a subreg copy.
3020 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {MemTy}, SrcOps: {})
3021 .addReg(RegNo: ValReg, Flags: {}, SubReg)
3022 .getReg(Idx: 0);
3023 RBI.constrainGenericRegister(Reg: Copy, RC: *RC, MRI);
3024 LdSt.getOperand(i: 0).setReg(Copy);
3025 } else if (isa<GLoad>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
3026 // If this is an any-extending load from the FPR bank, split it into a regular
3027 // load + extend.
3028 if (RB.getID() == AArch64::FPRRegBankID) {
3029 unsigned SubReg;
3030 LLT MemTy = LdSt.getMMO().getMemoryType();
3031 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
3032 if (!getSubRegForClass(RC, TRI, SubReg))
3033 return false;
3034 Register OldDst = LdSt.getReg(Idx: 0);
3035 Register NewDst =
3036 MRI.createGenericVirtualRegister(Ty: LdSt.getMMO().getMemoryType());
3037 LdSt.getOperand(i: 0).setReg(NewDst);
3038 MRI.setRegBank(Reg: NewDst, RegBank: RB);
3039 // Generate a SUBREG_TO_REG to extend it.
3040 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LdSt.getIterator()));
3041 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {OldDst}, SrcOps: {})
3042 .addUse(RegNo: NewDst)
3043 .addImm(Val: SubReg);
3044 auto SubRegRC = getRegClassForTypeOnBank(Ty: MRI.getType(Reg: OldDst), RB);
3045 RBI.constrainGenericRegister(Reg: OldDst, RC: *SubRegRC, MRI);
3046 MIB.setInstr(LdSt);
3047 ValTy = MemTy; // This is no longer an extending load.
3048 }
3049 }
3050
3051 // Helper lambda for partially selecting I. Either returns the original
3052 // instruction with an updated opcode, or a new instruction.
3053 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3054 bool IsStore = isa<GStore>(Val: I);
3055 const unsigned NewOpc =
3056 selectLoadStoreUIOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize: MemSizeInBits);
3057 if (NewOpc == I.getOpcode())
3058 return nullptr;
3059 // Check if we can fold anything into the addressing mode.
3060 auto AddrModeFns =
3061 selectAddrModeIndexed(Root&: I.getOperand(i: 1), Size: MemSizeInBytes);
3062 if (!AddrModeFns) {
3063 // Can't fold anything. Use the original instruction.
3064 I.setDesc(TII.get(Opcode: NewOpc));
3065 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
3066 return &I;
3067 }
3068
3069 // Folded something. Create a new instruction and return it.
3070 auto NewInst = MIB.buildInstr(Opc: NewOpc, DstOps: {}, SrcOps: {}, Flags: I.getFlags());
3071 Register CurValReg = I.getOperand(i: 0).getReg();
3072 IsStore ? NewInst.addUse(RegNo: CurValReg) : NewInst.addDef(RegNo: CurValReg);
3073 NewInst.cloneMemRefs(OtherMI: I);
3074 for (auto &Fn : *AddrModeFns)
3075 Fn(NewInst);
3076 I.eraseFromParent();
3077 return &*NewInst;
3078 };
3079
3080 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
3081 if (!LoadStore)
3082 return false;
3083
3084 // If we're storing a 0, use WZR/XZR.
3085 if (Opcode == TargetOpcode::G_STORE) {
3086 auto CVal = getIConstantVRegValWithLookThrough(
3087 VReg: LoadStore->getOperand(i: 0).getReg(), MRI);
3088 if (CVal && CVal->Value == 0) {
3089 switch (LoadStore->getOpcode()) {
3090 case AArch64::STRWui:
3091 case AArch64::STRHHui:
3092 case AArch64::STRBBui:
3093 LoadStore->getOperand(i: 0).setReg(AArch64::WZR);
3094 break;
3095 case AArch64::STRXui:
3096 LoadStore->getOperand(i: 0).setReg(AArch64::XZR);
3097 break;
3098 }
3099 }
3100 }
3101
3102 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3103 ValTy == LLT::scalar(SizeInBits: 64) && MemSizeInBits == 32)) {
3104 // The any/zextload from a smaller type to i32 should be handled by the
3105 // importer.
3106 if (MRI.getType(Reg: LoadStore->getOperand(i: 0).getReg()).getSizeInBits() != 64)
3107 return false;
3108 // If we have an extending load then change the load's type to be a
3109 // narrower reg and zero_extend with SUBREG_TO_REG.
3110 Register LdReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3111 Register DstReg = LoadStore->getOperand(i: 0).getReg();
3112 LoadStore->getOperand(i: 0).setReg(LdReg);
3113
3114 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LoadStore->getIterator()));
3115 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DstReg}, SrcOps: {})
3116 .addUse(RegNo: LdReg)
3117 .addImm(Val: AArch64::sub_32);
3118 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3119 return RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64allRegClass,
3120 MRI);
3121 }
3122 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3123 return true;
3124 }
3125
3126 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3127 case TargetOpcode::G_INDEXED_SEXTLOAD:
3128 return selectIndexedExtLoad(I, MRI);
3129 case TargetOpcode::G_INDEXED_LOAD:
3130 return selectIndexedLoad(I, MRI);
3131 case TargetOpcode::G_INDEXED_STORE:
3132 return selectIndexedStore(I&: cast<GIndexedStore>(Val&: I), MRI);
3133
3134 case TargetOpcode::G_LSHR:
3135 case TargetOpcode::G_ASHR:
3136 if (MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3137 return selectVectorAshrLshr(I, MRI);
3138 [[fallthrough]];
3139 case TargetOpcode::G_SHL: {
3140 if (Opcode == TargetOpcode::G_SHL &&
3141 MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3142 return selectVectorSHL(I, MRI);
3143
3144 // These shifts were legalized to have 64 bit shift amounts because we
3145 // want to take advantage of the selection patterns that assume the
3146 // immediates are s64s, however, selectBinaryOp will assume both operands
3147 // will have the same bit size.
3148 {
3149 Register SrcReg = I.getOperand(i: 1).getReg();
3150 Register ShiftReg = I.getOperand(i: 2).getReg();
3151 const LLT ShiftTy = MRI.getType(Reg: ShiftReg);
3152 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3153 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
3154 ShiftTy.getSizeInBits() == 64) {
3155 assert(!ShiftTy.isVector() && "unexpected vector shift ty");
3156 // Insert a subregister copy to implement a 64->32 trunc
3157 auto Trunc = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {SrcTy}, SrcOps: {})
3158 .addReg(RegNo: ShiftReg, Flags: {}, SubReg: AArch64::sub_32);
3159 MRI.setRegBank(Reg: Trunc.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
3160 I.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
3161 }
3162 }
3163
3164 const unsigned OpSize = Ty.getSizeInBits();
3165 const Register DefReg = I.getOperand(i: 0).getReg();
3166 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
3167
3168 const unsigned NewOpc = selectBinaryOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize);
3169 if (NewOpc == I.getOpcode())
3170 return false;
3171
3172 I.setDesc(TII.get(Opcode: NewOpc));
3173 // FIXME: Should the type be always reset in setDesc?
3174
3175 // Now that we selected an opcode, we need to constrain the register
3176 // operands to use appropriate classes.
3177 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3178 return true;
3179 }
3180 case TargetOpcode::G_PTR_ADD: {
3181 emitADD(DefReg: I.getOperand(i: 0).getReg(), LHS&: I.getOperand(i: 1), RHS&: I.getOperand(i: 2), MIRBuilder&: MIB);
3182 I.eraseFromParent();
3183 return true;
3184 }
3185
3186 case TargetOpcode::G_SADDE:
3187 case TargetOpcode::G_UADDE:
3188 case TargetOpcode::G_SSUBE:
3189 case TargetOpcode::G_USUBE:
3190 case TargetOpcode::G_SADDO:
3191 case TargetOpcode::G_UADDO:
3192 case TargetOpcode::G_SSUBO:
3193 case TargetOpcode::G_USUBO:
3194 return selectOverflowOp(I, MRI);
3195
3196 case TargetOpcode::G_PTRMASK: {
3197 Register MaskReg = I.getOperand(i: 2).getReg();
3198 std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(VReg: MaskReg, MRI);
3199 // TODO: Implement arbitrary cases
3200 if (!MaskVal || !isShiftedMask_64(Value: *MaskVal))
3201 return false;
3202
3203 uint64_t Mask = *MaskVal;
3204 I.setDesc(TII.get(Opcode: AArch64::ANDXri));
3205 I.getOperand(i: 2).ChangeToImmediate(
3206 ImmVal: AArch64_AM::encodeLogicalImmediate(imm: Mask, regSize: 64));
3207
3208 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3209 return true;
3210 }
3211 case TargetOpcode::G_PTRTOINT:
3212 case TargetOpcode::G_TRUNC: {
3213 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3214 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3215
3216 const Register DstReg = I.getOperand(i: 0).getReg();
3217 const Register SrcReg = I.getOperand(i: 1).getReg();
3218
3219 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3220 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3221
3222 if (DstRB.getID() != SrcRB.getID()) {
3223 LLVM_DEBUG(
3224 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
3225 return false;
3226 }
3227
3228 if (DstRB.getID() == AArch64::GPRRegBankID) {
3229 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3230 if (!DstRC)
3231 return false;
3232
3233 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(Ty: SrcTy, RB: SrcRB);
3234 if (!SrcRC)
3235 return false;
3236
3237 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
3238 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
3239 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
3240 return false;
3241 }
3242
3243 if (DstRC == SrcRC) {
3244 // Nothing to be done
3245 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(SizeInBits: 32) &&
3246 SrcTy == LLT::scalar(SizeInBits: 64)) {
3247 llvm_unreachable("TableGen can import this case");
3248 return false;
3249 } else if (DstRC == &AArch64::GPR32RegClass &&
3250 SrcRC == &AArch64::GPR64RegClass) {
3251 I.getOperand(i: 1).setSubReg(AArch64::sub_32);
3252 } else {
3253 LLVM_DEBUG(
3254 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3255 return false;
3256 }
3257
3258 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3259 return true;
3260 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
3261 if (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
3262 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
3263 I.setDesc(TII.get(Opcode: AArch64::XTNv4i16));
3264 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3265 return true;
3266 }
3267
3268 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
3269 MachineInstr *Extract = emitExtractVectorElt(
3270 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: DstTy.getSizeInBits()), VecReg: SrcReg, LaneIdx: 0, MIRBuilder&: MIB);
3271 if (!Extract)
3272 return false;
3273 I.eraseFromParent();
3274 return true;
3275 }
3276
3277 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
3278 if (Opcode == TargetOpcode::G_PTRTOINT) {
3279 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector");
3280 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3281 return selectCopy(I, TII, MRI, TRI, RBI);
3282 }
3283 }
3284
3285 return false;
3286 }
3287
3288 case TargetOpcode::G_ANYEXT: {
3289 if (selectUSMovFromExtend(I, MRI))
3290 return true;
3291
3292 const Register DstReg = I.getOperand(i: 0).getReg();
3293 const Register SrcReg = I.getOperand(i: 1).getReg();
3294
3295 const RegisterBank &RBDst = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3296 if (RBDst.getID() != AArch64::GPRRegBankID) {
3297 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
3298 << ", expected: GPR\n");
3299 return false;
3300 }
3301
3302 const RegisterBank &RBSrc = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3303 if (RBSrc.getID() != AArch64::GPRRegBankID) {
3304 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
3305 << ", expected: GPR\n");
3306 return false;
3307 }
3308
3309 const unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
3310
3311 if (DstSize == 0) {
3312 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
3313 return false;
3314 }
3315
3316 if (DstSize != 64 && DstSize > 32) {
3317 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
3318 << ", expected: 32 or 64\n");
3319 return false;
3320 }
3321 // At this point G_ANYEXT is just like a plain COPY, but we need
3322 // to explicitly form the 64-bit value if any.
3323 if (DstSize > 32) {
3324 Register ExtSrc = MRI.createVirtualRegister(RegClass: &AArch64::GPR64allRegClass);
3325 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
3326 .addDef(RegNo: ExtSrc)
3327 .addUse(RegNo: SrcReg)
3328 .addImm(Val: AArch64::sub_32);
3329 I.getOperand(i: 1).setReg(ExtSrc);
3330 }
3331 return selectCopy(I, TII, MRI, TRI, RBI);
3332 }
3333
3334 case TargetOpcode::G_ZEXT:
3335 case TargetOpcode::G_SEXT_INREG:
3336 case TargetOpcode::G_SEXT: {
3337 if (selectUSMovFromExtend(I, MRI))
3338 return true;
3339
3340 unsigned Opcode = I.getOpcode();
3341 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3342 const Register DefReg = I.getOperand(i: 0).getReg();
3343 Register SrcReg = I.getOperand(i: 1).getReg();
3344 const LLT DstTy = MRI.getType(Reg: DefReg);
3345 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3346 unsigned DstSize = DstTy.getSizeInBits();
3347 unsigned SrcSize = SrcTy.getSizeInBits();
3348
3349 // SEXT_INREG has the same src reg size as dst, the size of the value to be
3350 // extended is encoded in the imm.
3351 if (Opcode == TargetOpcode::G_SEXT_INREG)
3352 SrcSize = I.getOperand(i: 2).getImm();
3353
3354 if (DstTy.isVector())
3355 return false; // Should be handled by imported patterns.
3356
3357 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==
3358 AArch64::GPRRegBankID &&
3359 "Unexpected ext regbank");
3360
3361 MachineInstr *ExtI;
3362
3363 // First check if we're extending the result of a load which has a dest type
3364 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
3365 // GPR register on AArch64 and all loads which are smaller automatically
3366 // zero-extend the upper bits. E.g.
3367 // %v(s8) = G_LOAD %p, :: (load 1)
3368 // %v2(s32) = G_ZEXT %v(s8)
3369 if (!IsSigned) {
3370 auto *LoadMI = getOpcodeDef(Opcode: TargetOpcode::G_LOAD, Reg: SrcReg, MRI);
3371 bool IsGPR =
3372 RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
3373 if (LoadMI && IsGPR) {
3374 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3375 unsigned BytesLoaded = MemOp->getSize().getValue();
3376 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3377 return selectCopy(I, TII, MRI, TRI, RBI);
3378 }
3379
3380 // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
3381 // + SUBREG_TO_REG.
3382 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3383 Register SubregToRegSrc =
3384 MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3385 const Register ZReg = AArch64::WZR;
3386 MIB.buildInstr(Opc: AArch64::ORRWrs, DstOps: {SubregToRegSrc}, SrcOps: {ZReg, SrcReg})
3387 .addImm(Val: 0);
3388
3389 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
3390 .addUse(RegNo: SubregToRegSrc)
3391 .addImm(Val: AArch64::sub_32);
3392
3393 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass,
3394 MRI)) {
3395 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n");
3396 return false;
3397 }
3398
3399 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3400 MRI)) {
3401 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n");
3402 return false;
3403 }
3404
3405 I.eraseFromParent();
3406 return true;
3407 }
3408 }
3409
3410 if (DstSize == 64) {
3411 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3412 // FIXME: Can we avoid manually doing this?
3413 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3414 MRI)) {
3415 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
3416 << " operand\n");
3417 return false;
3418 }
3419 SrcReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG,
3420 DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
3421 .addUse(RegNo: SrcReg)
3422 .addImm(Val: AArch64::sub_32)
3423 .getReg(Idx: 0);
3424 }
3425
3426 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3427 DstOps: {DefReg}, SrcOps: {SrcReg})
3428 .addImm(Val: 0)
3429 .addImm(Val: SrcSize - 1);
3430 } else if (DstSize <= 32) {
3431 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3432 DstOps: {DefReg}, SrcOps: {SrcReg})
3433 .addImm(Val: 0)
3434 .addImm(Val: SrcSize - 1);
3435 } else {
3436 return false;
3437 }
3438
3439 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
3440 I.eraseFromParent();
3441 return true;
3442 }
3443
3444 case TargetOpcode::G_FREEZE:
3445 return selectCopy(I, TII, MRI, TRI, RBI);
3446
3447 case TargetOpcode::G_INTTOPTR:
3448 // The importer is currently unable to import pointer types since they
3449 // didn't exist in SelectionDAG.
3450 return selectCopy(I, TII, MRI, TRI, RBI);
3451
3452 case TargetOpcode::G_BITCAST:
3453 // Imported SelectionDAG rules can handle every bitcast except those that
3454 // bitcast from a type to the same type. Ideally, these shouldn't occur
3455 // but we might not run an optimizer that deletes them. The other exception
3456 // is bitcasts involving pointer types, as SelectionDAG has no knowledge
3457 // of them.
3458 return selectCopy(I, TII, MRI, TRI, RBI);
3459
3460 case TargetOpcode::G_SELECT: {
3461 auto &Sel = cast<GSelect>(Val&: I);
3462 const Register CondReg = Sel.getCondReg();
3463 const Register TReg = Sel.getTrueReg();
3464 const Register FReg = Sel.getFalseReg();
3465
3466 if (tryOptSelect(Sel))
3467 return true;
3468
3469 // Make sure to use an unused vreg instead of wzr, so that the peephole
3470 // optimizations will be able to optimize these.
3471 Register DeadVReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3472 auto TstMI = MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {DeadVReg}, SrcOps: {CondReg})
3473 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 1, regSize: 32));
3474 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
3475 if (!emitSelect(Dst: Sel.getReg(Idx: 0), True: TReg, False: FReg, CC: AArch64CC::NE, MIB))
3476 return false;
3477 Sel.eraseFromParent();
3478 return true;
3479 }
3480 case TargetOpcode::G_ICMP: {
3481 if (Ty.isVector())
3482 return false;
3483
3484 if (Ty != LLT::scalar(SizeInBits: 32)) {
3485 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
3486 << ", expected: " << LLT::scalar(32) << '\n');
3487 return false;
3488 }
3489
3490 auto &PredOp = I.getOperand(i: 1);
3491 emitIntegerCompare(LHS&: I.getOperand(i: 2), RHS&: I.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
3492 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
3493 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
3494 P: CmpInst::getInversePredicate(pred: Pred), RHS: I.getOperand(i: 3).getReg(), MRI: &MRI);
3495 emitCSINC(/*Dst=*/I.getOperand(i: 0).getReg(), /*Src1=*/AArch64::WZR,
3496 /*Src2=*/AArch64::WZR, Pred: InvCC, MIRBuilder&: MIB);
3497 I.eraseFromParent();
3498 return true;
3499 }
3500
3501 case TargetOpcode::G_FCMP: {
3502 CmpInst::Predicate Pred =
3503 static_cast<CmpInst::Predicate>(I.getOperand(i: 1).getPredicate());
3504 if (!emitFPCompare(LHS: I.getOperand(i: 2).getReg(), RHS: I.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
3505 Pred) ||
3506 !emitCSetForFCmp(Dst: I.getOperand(i: 0).getReg(), Pred, MIRBuilder&: MIB))
3507 return false;
3508 I.eraseFromParent();
3509 return true;
3510 }
3511 case TargetOpcode::G_VASTART:
3512 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3513 : selectVaStartAAPCS(I, MF, MRI);
3514 case TargetOpcode::G_INTRINSIC:
3515 return selectIntrinsic(I, MRI);
3516 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3517 return selectIntrinsicWithSideEffects(I, MRI);
3518 case TargetOpcode::G_IMPLICIT_DEF: {
3519 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
3520 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3521 const Register DstReg = I.getOperand(i: 0).getReg();
3522 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3523 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3524 RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
3525 return true;
3526 }
3527 case TargetOpcode::G_BLOCK_ADDR: {
3528 Function *BAFn = I.getOperand(i: 1).getBlockAddress()->getFunction();
3529 if (std::optional<uint16_t> BADisc =
3530 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: *BAFn)) {
3531 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X16}, SrcOps: {});
3532 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
3533 MIB.buildInstr(Opcode: AArch64::MOVaddrPAC)
3534 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress())
3535 .addImm(Val: AArch64PACKey::IA)
3536 .addReg(/*AddrDisc=*/RegNo: AArch64::XZR)
3537 .addImm(Val: *BADisc)
3538 .constrainAllUses(TII, TRI, RBI);
3539 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X16));
3540 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
3541 RC: AArch64::GPR64RegClass, MRI);
3542 I.eraseFromParent();
3543 return true;
3544 }
3545 if (TM.getCodeModel() == CodeModel::Large && !TM.isPositionIndependent()) {
3546 materializeLargeCMVal(I, V: I.getOperand(i: 1).getBlockAddress(), OpFlags: 0);
3547 I.eraseFromParent();
3548 return true;
3549 } else {
3550 I.setDesc(TII.get(Opcode: AArch64::MOVaddrBA));
3551 auto MovMI = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVaddrBA),
3552 DestReg: I.getOperand(i: 0).getReg())
3553 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress(),
3554 /* Offset */ 0, TargetFlags: AArch64II::MO_PAGE)
3555 .addBlockAddress(
3556 BA: I.getOperand(i: 1).getBlockAddress(), /* Offset */ 0,
3557 TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3558 I.eraseFromParent();
3559 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3560 return true;
3561 }
3562 }
3563 case AArch64::G_DUP: {
3564 // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
3565 // imported patterns. Do it manually here. Avoiding generating s16 gpr is
3566 // difficult because at RBS we may end up pessimizing the fpr case if we
3567 // decided to add an anyextend to fix this. Manual selection is the most
3568 // robust solution for now.
3569 if (RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI)->getID() !=
3570 AArch64::GPRRegBankID)
3571 return false; // We expect the fpr regbank case to be imported.
3572 LLT VecTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3573 if (VecTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8))
3574 I.setDesc(TII.get(Opcode: AArch64::DUPv8i8gpr));
3575 else if (VecTy == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8))
3576 I.setDesc(TII.get(Opcode: AArch64::DUPv16i8gpr));
3577 else if (VecTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16))
3578 I.setDesc(TII.get(Opcode: AArch64::DUPv4i16gpr));
3579 else if (VecTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16))
3580 I.setDesc(TII.get(Opcode: AArch64::DUPv8i16gpr));
3581 else
3582 return false;
3583 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3584 return true;
3585 }
3586 case TargetOpcode::G_BUILD_VECTOR:
3587 return selectBuildVector(I, MRI);
3588 case TargetOpcode::G_MERGE_VALUES:
3589 return selectMergeValues(I, MRI);
3590 case TargetOpcode::G_UNMERGE_VALUES:
3591 return selectUnmergeValues(I, MRI);
3592 case TargetOpcode::G_SHUFFLE_VECTOR:
3593 return selectShuffleVector(I, MRI);
3594 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3595 return selectExtractElt(I, MRI);
3596 case TargetOpcode::G_CONCAT_VECTORS:
3597 return selectConcatVectors(I, MRI);
3598 case TargetOpcode::G_JUMP_TABLE:
3599 return selectJumpTable(I, MRI);
3600 case TargetOpcode::G_MEMCPY:
3601 case TargetOpcode::G_MEMCPY_INLINE:
3602 case TargetOpcode::G_MEMMOVE:
3603 case TargetOpcode::G_MEMSET:
3604 case TargetOpcode::G_MEMSET_INLINE:
3605 assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
3606 return selectMOPS(I, MRI);
3607 }
3608
3609 return false;
3610}
3611
3612bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
3613 MachineIRBuilderState OldMIBState = MIB.getState();
3614 bool Success = select(I);
3615 MIB.setState(OldMIBState);
3616 return Success;
3617}
3618
3619bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3620 MachineRegisterInfo &MRI) {
3621 unsigned Mopcode;
3622 switch (GI.getOpcode()) {
3623 case TargetOpcode::G_MEMCPY:
3624 case TargetOpcode::G_MEMCPY_INLINE:
3625 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3626 break;
3627 case TargetOpcode::G_MEMMOVE:
3628 Mopcode = AArch64::MOPSMemoryMovePseudo;
3629 break;
3630 case TargetOpcode::G_MEMSET:
3631 case TargetOpcode::G_MEMSET_INLINE:
3632 // For tagged memset see llvm.aarch64.mops.memset.tag
3633 Mopcode = AArch64::MOPSMemorySetPseudo;
3634 break;
3635 }
3636
3637 auto &DstPtr = GI.getOperand(i: 0);
3638 auto &SrcOrVal = GI.getOperand(i: 1);
3639 auto &Size = GI.getOperand(i: 2);
3640
3641 // Create copies of the registers that can be clobbered.
3642 const Register DstPtrCopy = MRI.cloneVirtualRegister(VReg: DstPtr.getReg());
3643 const Register SrcValCopy = MRI.cloneVirtualRegister(VReg: SrcOrVal.getReg());
3644 const Register SizeCopy = MRI.cloneVirtualRegister(VReg: Size.getReg());
3645
3646 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3647 const auto &SrcValRegClass =
3648 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3649
3650 // Constrain to specific registers
3651 RBI.constrainGenericRegister(Reg: DstPtrCopy, RC: AArch64::GPR64commonRegClass, MRI);
3652 RBI.constrainGenericRegister(Reg: SrcValCopy, RC: SrcValRegClass, MRI);
3653 RBI.constrainGenericRegister(Reg: SizeCopy, RC: AArch64::GPR64RegClass, MRI);
3654
3655 MIB.buildCopy(Res: DstPtrCopy, Op: DstPtr);
3656 MIB.buildCopy(Res: SrcValCopy, Op: SrcOrVal);
3657 MIB.buildCopy(Res: SizeCopy, Op: Size);
3658
3659 // New instruction uses the copied registers because it must update them.
3660 // The defs are not used since they don't exist in G_MEM*. They are still
3661 // tied.
3662 // Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
3663 Register DefDstPtr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3664 Register DefSize = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3665 if (IsSet) {
3666 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSize},
3667 SrcOps: {DstPtrCopy, SizeCopy, SrcValCopy});
3668 } else {
3669 Register DefSrcPtr = MRI.createVirtualRegister(RegClass: &SrcValRegClass);
3670 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSrcPtr, DefSize},
3671 SrcOps: {DstPtrCopy, SrcValCopy, SizeCopy});
3672 }
3673
3674 GI.eraseFromParent();
3675 return true;
3676}
3677
3678bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3679 MachineRegisterInfo &MRI) {
3680 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT");
3681 Register JTAddr = I.getOperand(i: 0).getReg();
3682 unsigned JTI = I.getOperand(i: 1).getIndex();
3683 Register Index = I.getOperand(i: 2).getReg();
3684
3685 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: nullptr);
3686
3687 // With aarch64-jump-table-hardening, we only expand the jump table dispatch
3688 // sequence later, to guarantee the integrity of the intermediate values.
3689 if (MF->getFunction().hasFnAttribute(Kind: "aarch64-jump-table-hardening")) {
3690 CodeModel::Model CM = TM.getCodeModel();
3691 if (STI.isTargetMachO()) {
3692 if (CM != CodeModel::Small && CM != CodeModel::Large)
3693 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3694 } else {
3695 // Note that COFF support would likely also need JUMP_TABLE_DEBUG_INFO.
3696 assert(STI.isTargetELF() &&
3697 "jump table hardening only supported on MachO/ELF");
3698 if (CM != CodeModel::Small)
3699 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3700 }
3701
3702 MIB.buildCopy(Res: {AArch64::X16}, Op: I.getOperand(i: 2).getReg());
3703 MIB.buildInstr(Opcode: AArch64::BR_JumpTable)
3704 .addJumpTableIndex(Idx: I.getOperand(i: 1).getIndex());
3705 I.eraseFromParent();
3706 return true;
3707 }
3708
3709 Register TargetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3710 Register ScratchReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
3711
3712 auto JumpTableInst = MIB.buildInstr(Opc: AArch64::JumpTableDest32,
3713 DstOps: {TargetReg, ScratchReg}, SrcOps: {JTAddr, Index})
3714 .addJumpTableIndex(Idx: JTI);
3715 // Save the jump table info.
3716 MIB.buildInstr(Opc: TargetOpcode::JUMP_TABLE_DEBUG_INFO, DstOps: {},
3717 SrcOps: {static_cast<int64_t>(JTI)});
3718 // Build the indirect branch.
3719 MIB.buildInstr(Opc: AArch64::BR, DstOps: {}, SrcOps: {TargetReg});
3720 I.eraseFromParent();
3721 constrainSelectedInstRegOperands(I&: *JumpTableInst, TII, TRI, RBI);
3722 return true;
3723}
3724
3725bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
3726 MachineRegisterInfo &MRI) {
3727 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table");
3728 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!");
3729
3730 Register DstReg = I.getOperand(i: 0).getReg();
3731 unsigned JTI = I.getOperand(i: 1).getIndex();
3732 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3733 auto MovMI =
3734 MIB.buildInstr(Opc: AArch64::MOVaddrJT, DstOps: {DstReg}, SrcOps: {})
3735 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_PAGE)
3736 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3737 I.eraseFromParent();
3738 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3739 return true;
3740}
3741
3742bool AArch64InstructionSelector::selectTLSLocalExecELF(
3743 const GlobalValue *GV, MachineInstr &I, MachineRegisterInfo &MRI) {
3744 auto ConstrainRegOps = [&](MachineInstrBuilder MIB) {
3745 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3746 };
3747 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3748 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {}));
3749
3750 switch (MF->getTarget().Options.TLSSize) {
3751 default:
3752 llvm_unreachable("Unexpected TLS size");
3753 case 12: {
3754 // add x0, x0, :tprel_lo12:a
3755 ConstrainRegOps(
3756 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()},
3757 SrcOps: {ThreadBase})
3758 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF)
3759 .addImm(Val: 0));
3760 break;
3761 }
3762 case 24: {
3763 // add x0, x0, :tprel_hi12:a
3764 // add x0, x0, :tprel_lo12_nc:a
3765 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3766 ConstrainRegOps(
3767 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {Addr}, SrcOps: {ThreadBase})
3768 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3769 .addImm(Val: 0));
3770 ConstrainRegOps(
3771 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {Addr})
3772 .addGlobalAddress(GV, Offset: 0,
3773 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3774 AArch64II::MO_NC)
3775 .addImm(Val: 0));
3776 break;
3777 }
3778 case 32: {
3779 // movz x0, #:tprel_g1:a
3780 // movk x0, #:tprel_g0_nc:a
3781 // add x0, x1, x0
3782 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3783 ConstrainRegOps(
3784 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3785 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1)
3786 .addImm(Val: 16));
3787 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3788 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3789 .addGlobalAddress(GV, Offset: 0,
3790 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3791 AArch64II::MO_NC)
3792 .addImm(Val: 0));
3793 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3794 SrcOps: {ThreadBase, Addr2}));
3795 break;
3796 }
3797 case 48: {
3798 // movz x0, #:tprel_g2:a
3799 // movk x0, #:tprel_g1_nc:a
3800 // movk x0, #:tprel_g0_nc:a
3801 // add x0, x1, x0
3802 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3803 ConstrainRegOps(
3804 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3805 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G2)
3806 .addImm(Val: 32));
3807 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3808 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3809 .addGlobalAddress(GV, Offset: 0,
3810 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1 |
3811 AArch64II::MO_NC)
3812 .addImm(Val: 16));
3813 Register Addr3 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3814 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr3}, SrcOps: {Addr2})
3815 .addGlobalAddress(GV, Offset: 0,
3816 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3817 AArch64II::MO_NC)
3818 .addImm(Val: 0));
3819 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3820 SrcOps: {ThreadBase, Addr3}));
3821 break;
3822 }
3823 }
3824 I.eraseFromParent();
3825 return true;
3826}
3827
3828// TLS lowering below mirrors the corresponding DAGISel implementation.
3829// See LowerELFTLSModel() for details.
3830bool AArch64InstructionSelector::selectTLSGlobalValueELF(
3831 MachineInstr &I, MachineRegisterInfo &MRI) {
3832 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
3833 auto *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3834 TLSModel::Model Model =
3835 AArch64::getELFTLSModel(GV, TM, HasELFSignedGOT: FuncInfo->hasELFSignedGOT());
3836
3837 Register TPOff = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3838 switch (Model) {
3839 case TLSModel::LocalExec:
3840 return selectTLSLocalExecELF(GV, I, MRI);
3841 case TLSModel::InitialExec:
3842 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {TPOff}, SrcOps: {})
3843 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3844 break;
3845 case TLSModel::LocalDynamic:
3846 case TLSModel::GeneralDynamic: {
3847#ifndef NDEBUG
3848 SMEAttrs Attrs = MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
3849 assert(!Attrs.hasZAState() && !Attrs.hasStreamingInterfaceOrBody() &&
3850 !Attrs.hasStreamingCompatibleInterface() &&
3851 "unsupported SME features reached GlobalISel TLS lowering");
3852#endif
3853 unsigned Opcode = FuncInfo->hasELFSignedGOT()
3854 ? AArch64::TLSDESC_AUTH_CALLSEQ
3855 : AArch64::TLSDESC_CALLSEQ;
3856
3857 if (TLSModel::GeneralDynamic == Model) {
3858 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {}).addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3859 MIB.buildCopy(Res: TPOff, Op: Register(AArch64::X0));
3860 break;
3861 }
3862 assert(TLSModel::LocalDynamic == Model);
3863 // These accesses will need deduplicating if there's more than one.
3864 FuncInfo->incNumLocalDynamicTLSAccesses();
3865
3866 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {})
3867 .addExternalSymbol(FnName: "_TLS_MODULE_BASE_", TargetFlags: AArch64II::MO_TLS);
3868 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: Register(AArch64::X0));
3869 auto Add1 =
3870 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {LLT::scalar(SizeInBits: 64)}, SrcOps: {Copy.getReg(Idx: 0)})
3871 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3872 .addImm(Val: 0);
3873 auto Add2 =
3874 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {TPOff}, SrcOps: {Add1.getReg(Idx: 0)})
3875 .addGlobalAddress(GV, Offset: 0,
3876 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3877 AArch64II::MO_NC)
3878 .addImm(Val: 0);
3879 constrainSelectedInstRegOperands(I&: *Add1, TII, TRI, RBI);
3880 constrainSelectedInstRegOperands(I&: *Add2, TII, TRI, RBI);
3881 }
3882 }
3883 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3884 MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {});
3885 auto Add = MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3886 SrcOps: {ThreadBase, TPOff});
3887 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
3888
3889 I.eraseFromParent();
3890 return true;
3891}
3892
3893bool AArch64InstructionSelector::selectTLSGlobalValueMachO(
3894 MachineInstr &I, MachineRegisterInfo &MRI) {
3895 const auto &GlobalOp = I.getOperand(i: 1);
3896 assert(GlobalOp.getOffset() == 0 &&
3897 "Shouldn't have an offset on TLS globals!");
3898
3899 const GlobalValue &GV = *GlobalOp.getGlobal();
3900 MF->getFrameInfo().setAdjustsStack(true);
3901 auto LoadGOT =
3902 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {&AArch64::GPR64commonRegClass}, SrcOps: {})
3903 .addGlobalAddress(GV: &GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3904
3905 auto Load = MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {&AArch64::GPR64commonRegClass},
3906 SrcOps: {LoadGOT.getReg(Idx: 0)})
3907 .addImm(Val: 0);
3908
3909 MIB.buildCopy(Res: Register(AArch64::X0), Op: LoadGOT.getReg(Idx: 0));
3910 // TLS calls preserve all registers except those that absolutely must be
3911 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3912 // silly).
3913 unsigned Opcode = getBLRCallOpcode(MF: *MF);
3914
3915 // With ptrauth-calls, the tlv access thunk pointer is authenticated (IA, 0).
3916 if (MF->getFunction().hasFnAttribute(Kind: "ptrauth-calls")) {
3917 assert(Opcode == AArch64::BLR);
3918 Opcode = AArch64::BLRAAZ;
3919 }
3920
3921 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {Load})
3922 .addUse(RegNo: AArch64::X0, Flags: RegState::Implicit)
3923 .addDef(RegNo: AArch64::X0, Flags: RegState::Implicit)
3924 .addRegMask(Mask: TRI.getTLSCallPreservedMask());
3925
3926 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X0));
3927 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: AArch64::GPR64RegClass,
3928 MRI);
3929 I.eraseFromParent();
3930 return true;
3931}
3932
3933bool AArch64InstructionSelector::selectTLSGlobalValue(
3934 MachineInstr &I, MachineRegisterInfo &MRI) {
3935 // We don't support instructions with emulated TLS variables yet.
3936 if (TM.useEmulatedTLS())
3937 return false;
3938
3939 if (STI.isTargetELF())
3940 return selectTLSGlobalValueELF(I, MRI);
3941
3942 if (STI.isTargetMachO())
3943 return selectTLSGlobalValueMachO(I, MRI);
3944
3945 return false;
3946}
3947
3948MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3949 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3950 MachineIRBuilder &MIRBuilder) const {
3951 auto Undef = MIRBuilder.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstRC}, SrcOps: {});
3952
3953 auto BuildFn = [&](unsigned SubregIndex) {
3954 auto Ins =
3955 MIRBuilder
3956 .buildInstr(Opc: TargetOpcode::INSERT_SUBREG, DstOps: {DstRC}, SrcOps: {Undef, Scalar})
3957 .addImm(Val: SubregIndex);
3958 constrainSelectedInstRegOperands(I&: *Undef, TII, TRI, RBI);
3959 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
3960 return &*Ins;
3961 };
3962
3963 switch (EltSize) {
3964 case 8:
3965 return BuildFn(AArch64::bsub);
3966 case 16:
3967 return BuildFn(AArch64::hsub);
3968 case 32:
3969 return BuildFn(AArch64::ssub);
3970 case 64:
3971 return BuildFn(AArch64::dsub);
3972 default:
3973 return nullptr;
3974 }
3975}
3976
3977MachineInstr *
3978AArch64InstructionSelector::emitNarrowVector(Register DstReg, Register SrcReg,
3979 MachineIRBuilder &MIB,
3980 MachineRegisterInfo &MRI) const {
3981 LLT DstTy = MRI.getType(Reg: DstReg);
3982 const TargetRegisterClass *RC =
3983 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
3984 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3985 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
3986 return nullptr;
3987 }
3988 unsigned SubReg = 0;
3989 if (!getSubRegForClass(RC, TRI, SubReg))
3990 return nullptr;
3991 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3992 LLVM_DEBUG(dbgs() << "Unsupported destination size! ("
3993 << DstTy.getSizeInBits() << "\n");
3994 return nullptr;
3995 }
3996 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
3997 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
3998 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
3999 return Copy;
4000}
4001
4002bool AArch64InstructionSelector::selectMergeValues(
4003 MachineInstr &I, MachineRegisterInfo &MRI) {
4004 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
4005 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
4006 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
4007 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
4008 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
4009
4010 if (I.getNumOperands() != 3)
4011 return false;
4012
4013 // Merging 2 s64s into an s128.
4014 if (DstTy == LLT::scalar(SizeInBits: 128)) {
4015 if (SrcTy.getSizeInBits() != 64)
4016 return false;
4017 Register DstReg = I.getOperand(i: 0).getReg();
4018 Register Src1Reg = I.getOperand(i: 1).getReg();
4019 Register Src2Reg = I.getOperand(i: 2).getReg();
4020 auto Tmp = MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstTy}, SrcOps: {});
4021 MachineInstr *InsMI = emitLaneInsert(DstReg: std::nullopt, SrcReg: Tmp.getReg(Idx: 0), EltReg: Src1Reg,
4022 /* LaneIdx */ 0, RB, MIRBuilder&: MIB);
4023 if (!InsMI)
4024 return false;
4025 MachineInstr *Ins2MI = emitLaneInsert(DstReg, SrcReg: InsMI->getOperand(i: 0).getReg(),
4026 EltReg: Src2Reg, /* LaneIdx */ 1, RB, MIRBuilder&: MIB);
4027 if (!Ins2MI)
4028 return false;
4029 constrainSelectedInstRegOperands(I&: *InsMI, TII, TRI, RBI);
4030 constrainSelectedInstRegOperands(I&: *Ins2MI, TII, TRI, RBI);
4031 I.eraseFromParent();
4032 return true;
4033 }
4034
4035 if (RB.getID() != AArch64::GPRRegBankID)
4036 return false;
4037
4038 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
4039 return false;
4040
4041 auto *DstRC = &AArch64::GPR64RegClass;
4042 Register SubToRegDef = MRI.createVirtualRegister(RegClass: DstRC);
4043 MachineInstr &SubRegMI = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4044 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
4045 .addDef(RegNo: SubToRegDef)
4046 .addUse(RegNo: I.getOperand(i: 1).getReg())
4047 .addImm(Val: AArch64::sub_32);
4048 Register SubToRegDef2 = MRI.createVirtualRegister(RegClass: DstRC);
4049 // Need to anyext the second scalar before we can use bfm
4050 MachineInstr &SubRegMI2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4051 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
4052 .addDef(RegNo: SubToRegDef2)
4053 .addUse(RegNo: I.getOperand(i: 2).getReg())
4054 .addImm(Val: AArch64::sub_32);
4055 MachineInstr &BFM =
4056 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::BFMXri))
4057 .addDef(RegNo: I.getOperand(i: 0).getReg())
4058 .addUse(RegNo: SubToRegDef)
4059 .addUse(RegNo: SubToRegDef2)
4060 .addImm(Val: 32)
4061 .addImm(Val: 31);
4062 constrainSelectedInstRegOperands(I&: SubRegMI, TII, TRI, RBI);
4063 constrainSelectedInstRegOperands(I&: SubRegMI2, TII, TRI, RBI);
4064 constrainSelectedInstRegOperands(I&: BFM, TII, TRI, RBI);
4065 I.eraseFromParent();
4066 return true;
4067}
4068
4069static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
4070 const unsigned EltSize) {
4071 // Choose a lane copy opcode and subregister based off of the size of the
4072 // vector's elements.
4073 switch (EltSize) {
4074 case 8:
4075 CopyOpc = AArch64::DUPi8;
4076 ExtractSubReg = AArch64::bsub;
4077 break;
4078 case 16:
4079 CopyOpc = AArch64::DUPi16;
4080 ExtractSubReg = AArch64::hsub;
4081 break;
4082 case 32:
4083 CopyOpc = AArch64::DUPi32;
4084 ExtractSubReg = AArch64::ssub;
4085 break;
4086 case 64:
4087 CopyOpc = AArch64::DUPi64;
4088 ExtractSubReg = AArch64::dsub;
4089 break;
4090 default:
4091 // Unknown size, bail out.
4092 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n");
4093 return false;
4094 }
4095 return true;
4096}
4097
4098MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
4099 std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
4100 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
4101 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4102 unsigned CopyOpc = 0;
4103 unsigned ExtractSubReg = 0;
4104 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: ScalarTy.getSizeInBits())) {
4105 LLVM_DEBUG(
4106 dbgs() << "Couldn't determine lane copy opcode for instruction.\n");
4107 return nullptr;
4108 }
4109
4110 const TargetRegisterClass *DstRC =
4111 getRegClassForTypeOnBank(Ty: ScalarTy, RB: DstRB, GetAllRegSet: true);
4112 if (!DstRC) {
4113 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n");
4114 return nullptr;
4115 }
4116
4117 const RegisterBank &VecRB = *RBI.getRegBank(Reg: VecReg, MRI, TRI);
4118 const LLT &VecTy = MRI.getType(Reg: VecReg);
4119 const TargetRegisterClass *VecRC =
4120 getRegClassForTypeOnBank(Ty: VecTy, RB: VecRB, GetAllRegSet: true);
4121 if (!VecRC) {
4122 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
4123 return nullptr;
4124 }
4125
4126 // The register that we're going to copy into.
4127 Register InsertReg = VecReg;
4128 if (!DstReg)
4129 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
4130 // If the lane index is 0, we just use a subregister COPY.
4131 if (LaneIdx == 0) {
4132 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {*DstReg}, SrcOps: {})
4133 .addReg(RegNo: VecReg, Flags: {}, SubReg: ExtractSubReg);
4134 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4135 return &*Copy;
4136 }
4137
4138 // Lane copies require 128-bit wide registers. If we're dealing with an
4139 // unpacked vector, then we need to move up to that width. Insert an implicit
4140 // def and a subregister insert to get us there.
4141 if (VecTy.getSizeInBits() != 128) {
4142 MachineInstr *ScalarToVector = emitScalarToVector(
4143 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: VecReg, MIRBuilder);
4144 if (!ScalarToVector)
4145 return nullptr;
4146 InsertReg = ScalarToVector->getOperand(i: 0).getReg();
4147 }
4148
4149 MachineInstr *LaneCopyMI =
4150 MIRBuilder.buildInstr(Opc: CopyOpc, DstOps: {*DstReg}, SrcOps: {InsertReg}).addImm(Val: LaneIdx);
4151 constrainSelectedInstRegOperands(I&: *LaneCopyMI, TII, TRI, RBI);
4152
4153 // Make sure that we actually constrain the initial copy.
4154 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4155 return LaneCopyMI;
4156}
4157
4158bool AArch64InstructionSelector::selectExtractElt(
4159 MachineInstr &I, MachineRegisterInfo &MRI) {
4160 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
4161 "unexpected opcode!");
4162 Register DstReg = I.getOperand(i: 0).getReg();
4163 const LLT NarrowTy = MRI.getType(Reg: DstReg);
4164 const Register SrcReg = I.getOperand(i: 1).getReg();
4165 const LLT WideTy = MRI.getType(Reg: SrcReg);
4166 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
4167 "source register size too small!");
4168 assert(!NarrowTy.isVector() && "cannot extract vector into vector!");
4169
4170 // Need the lane index to determine the correct copy opcode.
4171 MachineOperand &LaneIdxOp = I.getOperand(i: 2);
4172 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?");
4173
4174 // Find the index to extract from.
4175 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: LaneIdxOp.getReg(), MRI);
4176 if (!VRegAndVal)
4177 return false;
4178 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4179
4180 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
4181 if (DstRB.getID() == AArch64::GPRRegBankID) {
4182 unsigned Opcode;
4183 switch (WideTy.getScalarSizeInBits()) {
4184 case 8:
4185 Opcode = AArch64::UMOVvi8;
4186 break;
4187 case 16:
4188 Opcode = AArch64::UMOVvi16;
4189 break;
4190 case 32:
4191 Opcode = AArch64::UMOVvi32;
4192 break;
4193 default:
4194 return false;
4195 }
4196
4197 if (WideTy.getSizeInBits() != 128) {
4198 MachineInstr *ScalarToVector = emitScalarToVector(
4199 EltSize: WideTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: SrcReg, MIRBuilder&: MIB);
4200 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
4201 I.getOperand(i: 1).setReg(ScalarToVector->getOperand(i: 0).getReg());
4202 }
4203
4204 I.setDesc(TII.get(Opcode));
4205 I.getOperand(i: 2).ChangeToImmediate(ImmVal: LaneIdx);
4206 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
4207 return true;
4208 }
4209
4210 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg,
4211 LaneIdx, MIRBuilder&: MIB);
4212 if (!Extract)
4213 return false;
4214
4215 I.eraseFromParent();
4216 return true;
4217}
4218
4219bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4220 MachineInstr &I, MachineRegisterInfo &MRI) {
4221 unsigned NumElts = I.getNumOperands() - 1;
4222 Register SrcReg = I.getOperand(i: NumElts).getReg();
4223 const LLT NarrowTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
4224 const LLT SrcTy = MRI.getType(Reg: SrcReg);
4225
4226 assert(NarrowTy.isVector() && "Expected an unmerge into vectors");
4227 if (SrcTy.getSizeInBits() > 128) {
4228 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge");
4229 return false;
4230 }
4231
4232 // We implement a split vector operation by treating the sub-vectors as
4233 // scalars and extracting them.
4234 const RegisterBank &DstRB =
4235 *RBI.getRegBank(Reg: I.getOperand(i: 0).getReg(), MRI, TRI);
4236 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4237 Register Dst = I.getOperand(i: OpIdx).getReg();
4238 MachineInstr *Extract =
4239 emitExtractVectorElt(DstReg: Dst, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg, LaneIdx: OpIdx, MIRBuilder&: MIB);
4240 if (!Extract)
4241 return false;
4242 }
4243 I.eraseFromParent();
4244 return true;
4245}
4246
4247bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
4248 MachineRegisterInfo &MRI) {
4249 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4250 "unexpected opcode");
4251
4252 // The last operand is the vector source register, and every other operand is
4253 // a register to unpack into.
4254 unsigned NumElts = I.getNumOperands() - 1;
4255 Register SrcReg = I.getOperand(i: NumElts).getReg();
4256 Register LoReg = I.getOperand(i: 0).getReg();
4257 Register HiReg = I.getOperand(i: 1).getReg();
4258 const LLT NarrowTy = MRI.getType(Reg: LoReg);
4259 const LLT WideTy = MRI.getType(Reg: SrcReg);
4260 const RegisterBank &LoRB = *RBI.getRegBank(Reg: LoReg, MRI, TRI);
4261 const RegisterBank &HiRB = *RBI.getRegBank(Reg: HiReg, MRI, TRI);
4262 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
4263
4264 // Handle unmerging a 128-bit FPR value into two 64-bit GPR values.
4265 if (NarrowTy == LLT::scalar(SizeInBits: 64) && WideTy == LLT::scalar(SizeInBits: 128) &&
4266 LoRB.getID() == AArch64::GPRRegBankID &&
4267 HiRB.getID() == AArch64::GPRRegBankID &&
4268 SrcRB.getID() == AArch64::FPRRegBankID) {
4269 MachineInstr &Lo = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4270 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: LoReg)
4271 .addUse(RegNo: SrcReg)
4272 .addImm(Val: 0);
4273 MachineInstr &Hi = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4274 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: HiReg)
4275 .addUse(RegNo: SrcReg)
4276 .addImm(Val: 1);
4277 constrainSelectedInstRegOperands(I&: Lo, TII, TRI, RBI);
4278 constrainSelectedInstRegOperands(I&: Hi, TII, TRI, RBI);
4279 I.eraseFromParent();
4280 return true;
4281 }
4282
4283 // TODO: Handle other unmerges into GPRs and from scalars to scalars.
4284 if (LoRB.getID() != AArch64::FPRRegBankID ||
4285 HiRB.getID() != AArch64::FPRRegBankID) {
4286 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
4287 "currently unsupported.\n");
4288 return false;
4289 }
4290
4291 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
4292 "source register size too small!");
4293
4294 if (!NarrowTy.isScalar())
4295 return selectSplitVectorUnmerge(I, MRI);
4296
4297 // Choose a lane copy opcode and subregister based off of the size of the
4298 // vector's elements.
4299 unsigned CopyOpc = 0;
4300 unsigned ExtractSubReg = 0;
4301 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: NarrowTy.getSizeInBits()))
4302 return false;
4303
4304 // Set up for the lane copies.
4305 MachineBasicBlock &MBB = *I.getParent();
4306
4307 // Stores the registers we'll be copying from.
4308 SmallVector<Register, 4> InsertRegs;
4309
4310 // We'll use the first register twice, so we only need NumElts-1 registers.
4311 unsigned NumInsertRegs = NumElts - 1;
4312
4313 // If our elements fit into exactly 128 bits, then we can copy from the source
4314 // directly. Otherwise, we need to do a bit of setup with some subregister
4315 // inserts.
4316 if (NarrowTy.getSizeInBits() * NumElts == 128) {
4317 InsertRegs.assign(NumElts: NumInsertRegs, Elt: SrcReg);
4318 } else {
4319 // No. We have to perform subregister inserts. For each insert, create an
4320 // implicit def and a subregister insert, and save the register we create.
4321 // For scalar sources, treat as a pseudo-vector of NarrowTy elements.
4322 unsigned EltSize = WideTy.isVector() ? WideTy.getScalarSizeInBits()
4323 : NarrowTy.getSizeInBits();
4324 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
4325 Ty: LLT::fixed_vector(NumElements: NumElts, ScalarSizeInBits: EltSize), RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
4326 unsigned SubReg = 0;
4327 bool Found = getSubRegForClass(RC, TRI, SubReg);
4328 (void)Found;
4329 assert(Found && "expected to find last operand's subeg idx");
4330 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
4331 Register ImpDefReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4332 MachineInstr &ImpDefMI =
4333 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF),
4334 DestReg: ImpDefReg);
4335
4336 // Now, create the subregister insert from SrcReg.
4337 Register InsertReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4338 MachineInstr &InsMI =
4339 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(),
4340 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: InsertReg)
4341 .addUse(RegNo: ImpDefReg)
4342 .addUse(RegNo: SrcReg)
4343 .addImm(Val: SubReg);
4344
4345 constrainSelectedInstRegOperands(I&: ImpDefMI, TII, TRI, RBI);
4346 constrainSelectedInstRegOperands(I&: InsMI, TII, TRI, RBI);
4347
4348 // Save the register so that we can copy from it after.
4349 InsertRegs.push_back(Elt: InsertReg);
4350 }
4351 }
4352
4353 // Now that we've created any necessary subregister inserts, we can
4354 // create the copies.
4355 //
4356 // Perform the first copy separately as a subregister copy.
4357 Register CopyTo = I.getOperand(i: 0).getReg();
4358 auto FirstCopy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {CopyTo}, SrcOps: {})
4359 .addReg(RegNo: InsertRegs[0], Flags: {}, SubReg: ExtractSubReg);
4360 constrainSelectedInstRegOperands(I&: *FirstCopy, TII, TRI, RBI);
4361
4362 // Now, perform the remaining copies as vector lane copies.
4363 unsigned LaneIdx = 1;
4364 for (Register InsReg : InsertRegs) {
4365 Register CopyTo = I.getOperand(i: LaneIdx).getReg();
4366 MachineInstr &CopyInst =
4367 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CopyOpc), DestReg: CopyTo)
4368 .addUse(RegNo: InsReg)
4369 .addImm(Val: LaneIdx);
4370 constrainSelectedInstRegOperands(I&: CopyInst, TII, TRI, RBI);
4371 ++LaneIdx;
4372 }
4373
4374 // Separately constrain the first copy's destination. Because of the
4375 // limitation in constrainOperandRegClass, we can't guarantee that this will
4376 // actually be constrained. So, do it ourselves using the second operand.
4377 const TargetRegisterClass *RC =
4378 MRI.getRegClassOrNull(Reg: I.getOperand(i: 1).getReg());
4379 if (!RC) {
4380 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
4381 return false;
4382 }
4383
4384 RBI.constrainGenericRegister(Reg: CopyTo, RC: *RC, MRI);
4385 I.eraseFromParent();
4386 return true;
4387}
4388
4389bool AArch64InstructionSelector::selectConcatVectors(
4390 MachineInstr &I, MachineRegisterInfo &MRI) {
4391 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4392 "Unexpected opcode");
4393 Register Dst = I.getOperand(i: 0).getReg();
4394 Register Op1 = I.getOperand(i: 1).getReg();
4395 Register Op2 = I.getOperand(i: 2).getReg();
4396 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder&: MIB);
4397 if (!ConcatMI)
4398 return false;
4399 I.eraseFromParent();
4400 return true;
4401}
4402
4403unsigned
4404AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4405 MachineFunction &MF) const {
4406 Type *CPTy = CPVal->getType();
4407 Align Alignment = MF.getDataLayout().getPrefTypeAlign(Ty: CPTy);
4408
4409 MachineConstantPool *MCP = MF.getConstantPool();
4410 return MCP->getConstantPoolIndex(C: CPVal, Alignment);
4411}
4412
4413MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4414 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4415 const TargetRegisterClass *RC;
4416 unsigned Opc;
4417 bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
4418 unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(Ty: CPVal->getType());
4419 switch (Size) {
4420 case 16:
4421 RC = &AArch64::FPR128RegClass;
4422 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4423 break;
4424 case 8:
4425 RC = &AArch64::FPR64RegClass;
4426 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4427 break;
4428 case 4:
4429 RC = &AArch64::FPR32RegClass;
4430 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4431 break;
4432 case 2:
4433 RC = &AArch64::FPR16RegClass;
4434 Opc = AArch64::LDRHui;
4435 break;
4436 default:
4437 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "
4438 << *CPVal->getType());
4439 return nullptr;
4440 }
4441
4442 MachineInstr *LoadMI = nullptr;
4443 auto &MF = MIRBuilder.getMF();
4444 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4445 if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
4446 // Use load(literal) for tiny code model.
4447 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {}).addConstantPoolIndex(Idx: CPIdx);
4448 } else {
4449 auto Adrp =
4450 MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
4451 .addConstantPoolIndex(Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
4452
4453 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {Adrp})
4454 .addConstantPoolIndex(
4455 Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4456
4457 constrainSelectedInstRegOperands(I&: *Adrp, TII, TRI, RBI);
4458 }
4459
4460 MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
4461 LoadMI->addMemOperand(MF, MO: MF.getMachineMemOperand(PtrInfo,
4462 F: MachineMemOperand::MOLoad,
4463 Size, BaseAlignment: Align(Size)));
4464 constrainSelectedInstRegOperands(I&: *LoadMI, TII, TRI, RBI);
4465 return LoadMI;
4466}
4467
4468/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4469/// size and RB.
4470static std::pair<unsigned, unsigned>
4471getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4472 unsigned Opc, SubregIdx;
4473 if (RB.getID() == AArch64::GPRRegBankID) {
4474 if (EltSize == 8) {
4475 Opc = AArch64::INSvi8gpr;
4476 SubregIdx = AArch64::bsub;
4477 } else if (EltSize == 16) {
4478 Opc = AArch64::INSvi16gpr;
4479 SubregIdx = AArch64::ssub;
4480 } else if (EltSize == 32) {
4481 Opc = AArch64::INSvi32gpr;
4482 SubregIdx = AArch64::ssub;
4483 } else if (EltSize == 64) {
4484 Opc = AArch64::INSvi64gpr;
4485 SubregIdx = AArch64::dsub;
4486 } else {
4487 llvm_unreachable("invalid elt size!");
4488 }
4489 } else {
4490 if (EltSize == 8) {
4491 Opc = AArch64::INSvi8lane;
4492 SubregIdx = AArch64::bsub;
4493 } else if (EltSize == 16) {
4494 Opc = AArch64::INSvi16lane;
4495 SubregIdx = AArch64::hsub;
4496 } else if (EltSize == 32) {
4497 Opc = AArch64::INSvi32lane;
4498 SubregIdx = AArch64::ssub;
4499 } else if (EltSize == 64) {
4500 Opc = AArch64::INSvi64lane;
4501 SubregIdx = AArch64::dsub;
4502 } else {
4503 llvm_unreachable("invalid elt size!");
4504 }
4505 }
4506 return std::make_pair(x&: Opc, y&: SubregIdx);
4507}
4508
4509MachineInstr *AArch64InstructionSelector::emitInstr(
4510 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4511 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4512 const ComplexRendererFns &RenderFns) const {
4513 assert(Opcode && "Expected an opcode?");
4514 assert(!isPreISelGenericOpcode(Opcode) &&
4515 "Function should only be used to produce selected instructions!");
4516 auto MI = MIRBuilder.buildInstr(Opc: Opcode, DstOps, SrcOps);
4517 if (RenderFns)
4518 for (auto &Fn : *RenderFns)
4519 Fn(MI);
4520 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
4521 return &*MI;
4522}
4523
4524MachineInstr *AArch64InstructionSelector::emitAddSub(
4525 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4526 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4527 MachineIRBuilder &MIRBuilder) const {
4528 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4529 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4530 auto Ty = MRI.getType(Reg: LHS.getReg());
4531 assert(!Ty.isVector() && "Expected a scalar or pointer?");
4532 unsigned Size = Ty.getSizeInBits();
4533 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
4534 bool Is32Bit = Size == 32;
4535
4536 // INSTRri form with positive arithmetic immediate.
4537 if (auto Fns = selectArithImmed(Root&: RHS))
4538 return emitInstr(Opcode: AddrModeAndSizeToOpcode[0][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4539 MIRBuilder, RenderFns: Fns);
4540
4541 // INSTRri form with negative arithmetic immediate.
4542 if (auto Fns = selectNegArithImmed(Root&: RHS))
4543 return emitInstr(Opcode: AddrModeAndSizeToOpcode[3][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4544 MIRBuilder, RenderFns: Fns);
4545
4546 // INSTRrx form.
4547 if (auto Fns = selectArithExtendedRegister(Root&: RHS))
4548 return emitInstr(Opcode: AddrModeAndSizeToOpcode[4][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4549 MIRBuilder, RenderFns: Fns);
4550
4551 // INSTRrs form.
4552 if (auto Fns = selectShiftedRegister(Root&: RHS))
4553 return emitInstr(Opcode: AddrModeAndSizeToOpcode[1][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4554 MIRBuilder, RenderFns: Fns);
4555 return emitInstr(Opcode: AddrModeAndSizeToOpcode[2][Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS},
4556 MIRBuilder);
4557}
4558
4559MachineInstr *
4560AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4561 MachineOperand &RHS,
4562 MachineIRBuilder &MIRBuilder) const {
4563 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4564 ._M_elems: {{AArch64::ADDXri, AArch64::ADDWri},
4565 {AArch64::ADDXrs, AArch64::ADDWrs},
4566 {AArch64::ADDXrr, AArch64::ADDWrr},
4567 {AArch64::SUBXri, AArch64::SUBWri},
4568 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4569 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst: DefReg, LHS, RHS, MIRBuilder);
4570}
4571
4572MachineInstr *
4573AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4574 MachineOperand &RHS,
4575 MachineIRBuilder &MIRBuilder) const {
4576 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4577 ._M_elems: {{AArch64::ADDSXri, AArch64::ADDSWri},
4578 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4579 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4580 {AArch64::SUBSXri, AArch64::SUBSWri},
4581 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4582 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4583}
4584
4585MachineInstr *
4586AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4587 MachineOperand &RHS,
4588 MachineIRBuilder &MIRBuilder) const {
4589 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4590 ._M_elems: {{AArch64::SUBSXri, AArch64::SUBSWri},
4591 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4592 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4593 {AArch64::ADDSXri, AArch64::ADDSWri},
4594 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4595 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4596}
4597
4598MachineInstr *
4599AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
4600 MachineOperand &RHS,
4601 MachineIRBuilder &MIRBuilder) const {
4602 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4603 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4604 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4605 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4606 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4607}
4608
4609MachineInstr *
4610AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
4611 MachineOperand &RHS,
4612 MachineIRBuilder &MIRBuilder) const {
4613 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4614 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4615 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4616 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4617 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4618}
4619
4620MachineInstr *
4621AArch64InstructionSelector::emitCMP(MachineOperand &LHS, MachineOperand &RHS,
4622 MachineIRBuilder &MIRBuilder) const {
4623 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4624 bool Is32Bit = MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32;
4625 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4626 return emitSUBS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4627}
4628
4629MachineInstr *
4630AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4631 MachineIRBuilder &MIRBuilder) const {
4632 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4633 bool Is32Bit = (MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4634 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4635 return emitADDS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4636}
4637
4638MachineInstr *
4639AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4640 MachineIRBuilder &MIRBuilder) const {
4641 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4642 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4643 LLT Ty = MRI.getType(Reg: LHS.getReg());
4644 unsigned RegSize = Ty.getSizeInBits();
4645 bool Is32Bit = (RegSize == 32);
4646 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4647 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4648 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4649 // ANDS needs a logical immediate for its immediate form. Check if we can
4650 // fold one in.
4651 if (auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI)) {
4652 int64_t Imm = ValAndVReg->Value.getSExtValue();
4653
4654 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize)) {
4655 auto TstMI = MIRBuilder.buildInstr(Opc: OpcTable[0][Is32Bit], DstOps: {Ty}, SrcOps: {LHS});
4656 TstMI.addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: Imm, regSize: RegSize));
4657 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
4658 return &*TstMI;
4659 }
4660 }
4661
4662 if (auto Fns = selectLogicalShiftedRegister(Root&: RHS))
4663 return emitInstr(Opcode: OpcTable[1][Is32Bit], DstOps: {Ty}, SrcOps: {LHS}, MIRBuilder, RenderFns: Fns);
4664 return emitInstr(Opcode: OpcTable[2][Is32Bit], DstOps: {Ty}, SrcOps: {LHS, RHS}, MIRBuilder);
4665}
4666
4667MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4668 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4669 MachineIRBuilder &MIRBuilder) const {
4670 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!");
4671 assert(Predicate.isPredicate() && "Expected predicate?");
4672 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4673 LLT CmpTy = MRI.getType(Reg: LHS.getReg());
4674 assert(!CmpTy.isVector() && "Expected scalar or pointer");
4675 unsigned Size = CmpTy.getSizeInBits();
4676 (void)Size;
4677 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?");
4678 // Fold the compare into a cmn or tst if possible.
4679 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4680 return FoldCmp;
4681 return emitCMP(LHS, RHS, MIRBuilder);
4682}
4683
4684MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4685 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4686 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4687#ifndef NDEBUG
4688 LLT Ty = MRI.getType(Dst);
4689 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&
4690 "Expected a 32-bit scalar register?");
4691#endif
4692 const Register ZReg = AArch64::WZR;
4693 AArch64CC::CondCode CC1, CC2;
4694 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
4695 auto InvCC1 = AArch64CC::getInvertedCondCode(Code: CC1);
4696 if (CC2 == AArch64CC::AL)
4697 return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1,
4698 MIRBuilder);
4699 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4700 Register Def1Reg = MRI.createVirtualRegister(RegClass: RC);
4701 Register Def2Reg = MRI.createVirtualRegister(RegClass: RC);
4702 auto InvCC2 = AArch64CC::getInvertedCondCode(Code: CC2);
4703 emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1, MIRBuilder);
4704 emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC2, MIRBuilder);
4705 auto OrMI = MIRBuilder.buildInstr(Opc: AArch64::ORRWrr, DstOps: {Dst}, SrcOps: {Def1Reg, Def2Reg});
4706 constrainSelectedInstRegOperands(I&: *OrMI, TII, TRI, RBI);
4707 return &*OrMI;
4708}
4709
4710MachineInstr *AArch64InstructionSelector::emitFPCompare(
4711 Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
4712 std::optional<CmpInst::Predicate> Pred) const {
4713 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4714 LLT Ty = MRI.getType(Reg: LHS);
4715 if (Ty.isVector())
4716 return nullptr;
4717 unsigned OpSize = Ty.getSizeInBits();
4718 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4719
4720 // If this is a compare against +0.0, then we don't have
4721 // to explicitly materialize a constant.
4722 bool ShouldUseImm = mi_match(R: RHS, MRI, P: m_PosZeroFP());
4723
4724 auto IsEqualityPred = [](CmpInst::Predicate P) {
4725 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4726 P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
4727 };
4728 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4729 // Try commuting the operands.
4730 if (mi_match(R: LHS, MRI, P: m_PosZeroFP())) {
4731 ShouldUseImm = true;
4732 std::swap(a&: LHS, b&: RHS);
4733 }
4734 }
4735 unsigned CmpOpcTbl[2][3] = {
4736 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4737 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4738 unsigned CmpOpc =
4739 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4740
4741 // Partially build the compare. Decide if we need to add a use for the
4742 // third operand based off whether or not we're comparing against 0.0.
4743 auto CmpMI = MIRBuilder.buildInstr(Opcode: CmpOpc).addUse(RegNo: LHS);
4744 CmpMI.setMIFlags(MachineInstr::NoFPExcept);
4745 if (!ShouldUseImm)
4746 CmpMI.addUse(RegNo: RHS);
4747 constrainSelectedInstRegOperands(I&: *CmpMI, TII, TRI, RBI);
4748 return &*CmpMI;
4749}
4750
4751MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4752 std::optional<Register> Dst, Register Op1, Register Op2,
4753 MachineIRBuilder &MIRBuilder) const {
4754 // We implement a vector concat by:
4755 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4756 // 2. Insert the upper vector into the destination's upper element
4757 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4758 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4759
4760 const LLT Op1Ty = MRI.getType(Reg: Op1);
4761 const LLT Op2Ty = MRI.getType(Reg: Op2);
4762
4763 if (Op1Ty != Op2Ty) {
4764 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys");
4765 return nullptr;
4766 }
4767 assert(Op1Ty.isVector() && "Expected a vector for vector concat");
4768
4769 if (Op1Ty.getSizeInBits() >= 128) {
4770 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors");
4771 return nullptr;
4772 }
4773
4774 // At the moment we just support 64 bit vector concats.
4775 if (Op1Ty.getSizeInBits() != 64) {
4776 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors");
4777 return nullptr;
4778 }
4779
4780 const LLT ScalarTy = LLT::scalar(SizeInBits: Op1Ty.getSizeInBits());
4781 const RegisterBank &FPRBank = *RBI.getRegBank(Reg: Op1, MRI, TRI);
4782 const TargetRegisterClass *DstRC =
4783 getRegClassForTypeOnBank(Ty: Op1Ty.multiplyElements(Factor: 2), RB: FPRBank);
4784
4785 MachineInstr *WidenedOp1 =
4786 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op1, MIRBuilder);
4787 MachineInstr *WidenedOp2 =
4788 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op2, MIRBuilder);
4789 if (!WidenedOp1 || !WidenedOp2) {
4790 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value");
4791 return nullptr;
4792 }
4793
4794 // Now do the insert of the upper element.
4795 unsigned InsertOpc, InsSubRegIdx;
4796 std::tie(args&: InsertOpc, args&: InsSubRegIdx) =
4797 getInsertVecEltOpInfo(RB: FPRBank, EltSize: ScalarTy.getSizeInBits());
4798
4799 if (!Dst)
4800 Dst = MRI.createVirtualRegister(RegClass: DstRC);
4801 auto InsElt =
4802 MIRBuilder
4803 .buildInstr(Opc: InsertOpc, DstOps: {*Dst}, SrcOps: {WidenedOp1->getOperand(i: 0).getReg()})
4804 .addImm(Val: 1) /* Lane index */
4805 .addUse(RegNo: WidenedOp2->getOperand(i: 0).getReg())
4806 .addImm(Val: 0);
4807 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
4808 return &*InsElt;
4809}
4810
4811MachineInstr *
4812AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
4813 Register Src2, AArch64CC::CondCode Pred,
4814 MachineIRBuilder &MIRBuilder) const {
4815 auto &MRI = *MIRBuilder.getMRI();
4816 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg: Dst);
4817 // If we used a register class, then this won't necessarily have an LLT.
4818 // Compute the size based off whether or not we have a class or bank.
4819 unsigned Size;
4820 if (const auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
4821 Size = TRI.getRegSizeInBits(RC: *RC);
4822 else
4823 Size = MRI.getType(Reg: Dst).getSizeInBits();
4824 // Some opcodes use s1.
4825 assert(Size <= 64 && "Expected 64 bits or less only!");
4826 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4827 unsigned Opc = OpcTable[Size == 64];
4828 auto CSINC = MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Src1, Src2}).addImm(Val: Pred);
4829 constrainSelectedInstRegOperands(I&: *CSINC, TII, TRI, RBI);
4830 return &*CSINC;
4831}
4832
4833MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
4834 Register CarryReg) {
4835 MachineRegisterInfo *MRI = MIB.getMRI();
4836 unsigned Opcode = I.getOpcode();
4837
4838 // If the instruction is a SUB, we need to negate the carry,
4839 // because borrowing is indicated by carry-flag == 0.
4840 bool NeedsNegatedCarry =
4841 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4842
4843 // If the previous instruction will already produce the correct carry, do not
4844 // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
4845 // generated during legalization of wide add/sub. This optimization depends on
4846 // these sequences not being interrupted by other instructions.
4847 // We have to select the previous instruction before the carry-using
4848 // instruction is deleted by the calling function, otherwise the previous
4849 // instruction might become dead and would get deleted.
4850 MachineInstr *SrcMI = MRI->getVRegDef(Reg: CarryReg);
4851 if (SrcMI == I.getPrevNode()) {
4852 if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(Val: SrcMI)) {
4853 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4854 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4855 CarrySrcMI->isUnsigned() &&
4856 CarrySrcMI->getCarryOutReg() == CarryReg &&
4857 selectAndRestoreState(I&: *SrcMI))
4858 return nullptr;
4859 }
4860 }
4861
4862 Register DeadReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4863
4864 if (NeedsNegatedCarry) {
4865 // (0 - Carry) sets !C in NZCV when Carry == 1
4866 Register ZReg = AArch64::WZR;
4867 return emitInstr(Opcode: AArch64::SUBSWrr, DstOps: {DeadReg}, SrcOps: {ZReg, CarryReg}, MIRBuilder&: MIB);
4868 }
4869
4870 // (Carry - 1) sets !C in NZCV when Carry == 0
4871 auto Fns = select12BitValueWithLeftShift(Immed: 1);
4872 return emitInstr(Opcode: AArch64::SUBSWri, DstOps: {DeadReg}, SrcOps: {CarryReg}, MIRBuilder&: MIB, RenderFns: Fns);
4873}
4874
4875bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &I,
4876 MachineRegisterInfo &MRI) {
4877 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
4878
4879 if (auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
4880 // Set NZCV carry according to carry-in VReg
4881 emitCarryIn(I, CarryReg: CarryInMI->getCarryInReg());
4882 }
4883
4884 // Emit the operation and get the correct condition code.
4885 auto OpAndCC = emitOverflowOp(Opcode: I.getOpcode(), Dst: CarryMI.getDstReg(),
4886 LHS&: CarryMI.getLHS(), RHS&: CarryMI.getRHS(), MIRBuilder&: MIB);
4887
4888 Register CarryOutReg = CarryMI.getCarryOutReg();
4889
4890 // Don't convert carry-out to VReg if it is never used
4891 if (!MRI.use_nodbg_empty(RegNo: CarryOutReg)) {
4892 // Now, put the overflow result in the register given by the first operand
4893 // to the overflow op. CSINC increments the result when the predicate is
4894 // false, so to get the increment when it's true, we need to use the
4895 // inverse. In this case, we want to increment when carry is set.
4896 Register ZReg = AArch64::WZR;
4897 emitCSINC(/*Dst=*/CarryOutReg, /*Src1=*/ZReg, /*Src2=*/ZReg,
4898 Pred: getInvertedCondCode(Code: OpAndCC.second), MIRBuilder&: MIB);
4899 }
4900
4901 I.eraseFromParent();
4902 return true;
4903}
4904
4905std::pair<MachineInstr *, AArch64CC::CondCode>
4906AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4907 MachineOperand &LHS,
4908 MachineOperand &RHS,
4909 MachineIRBuilder &MIRBuilder) const {
4910 switch (Opcode) {
4911 default:
4912 llvm_unreachable("Unexpected opcode!");
4913 case TargetOpcode::G_SADDO:
4914 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4915 case TargetOpcode::G_UADDO:
4916 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4917 case TargetOpcode::G_SSUBO:
4918 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4919 case TargetOpcode::G_USUBO:
4920 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4921 case TargetOpcode::G_SADDE:
4922 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4923 case TargetOpcode::G_UADDE:
4924 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4925 case TargetOpcode::G_SSUBE:
4926 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4927 case TargetOpcode::G_USUBE:
4928 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4929 }
4930}
4931
4932/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4933/// expressed as a conjunction.
4934/// \param CanNegate Set to true if we can negate the whole sub-tree just by
4935/// changing the conditions on the CMP tests.
4936/// (this means we can call emitConjunctionRec() with
4937/// Negate==true on this sub-tree)
4938/// \param MustBeFirst Set to true if this subtree needs to be negated and we
4939/// cannot do the negation naturally. We are required to
4940/// emit the subtree first in this case.
4941/// \param WillNegate Is true if are called when the result of this
4942/// subexpression must be negated. This happens when the
4943/// outer expression is an OR. We can use this fact to know
4944/// that we have a double negation (or (or ...) ...) that
4945/// can be implemented for free.
4946static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
4947 bool WillNegate, MachineRegisterInfo &MRI,
4948 unsigned Depth = 0) {
4949 if (!MRI.hasOneNonDBGUse(RegNo: Val))
4950 return false;
4951 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4952 unsigned Opcode = ValDef->getOpcode();
4953 if (isa<GAnyCmp>(Val: ValDef)) {
4954 CanNegate = true;
4955 MustBeFirst = false;
4956 return true;
4957 }
4958 // Protect against exponential runtime and stack overflow.
4959 if (Depth > 6)
4960 return false;
4961 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4962 bool IsOR = Opcode == TargetOpcode::G_OR;
4963 Register O0 = ValDef->getOperand(i: 1).getReg();
4964 Register O1 = ValDef->getOperand(i: 2).getReg();
4965 bool CanNegateL;
4966 bool MustBeFirstL;
4967 if (!canEmitConjunction(Val: O0, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI, Depth: Depth + 1))
4968 return false;
4969 bool CanNegateR;
4970 bool MustBeFirstR;
4971 if (!canEmitConjunction(Val: O1, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI, Depth: Depth + 1))
4972 return false;
4973
4974 if (MustBeFirstL && MustBeFirstR)
4975 return false;
4976
4977 if (IsOR) {
4978 // For an OR expression we need to be able to naturally negate at least
4979 // one side or we cannot do the transformation at all.
4980 if (!CanNegateL && !CanNegateR)
4981 return false;
4982 // If we the result of the OR will be negated and we can naturally negate
4983 // the leaves, then this sub-tree as a whole negates naturally.
4984 CanNegate = WillNegate && CanNegateL && CanNegateR;
4985 // If we cannot naturally negate the whole sub-tree, then this must be
4986 // emitted first.
4987 MustBeFirst = !CanNegate;
4988 } else {
4989 assert(Opcode == TargetOpcode::G_AND && "Must be G_AND");
4990 // We cannot naturally negate an AND operation.
4991 CanNegate = false;
4992 MustBeFirst = MustBeFirstL || MustBeFirstR;
4993 }
4994 return true;
4995 }
4996 return false;
4997}
4998
4999MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
5000 Register LHS, Register RHS, CmpInst::Predicate CC,
5001 AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC,
5002 MachineIRBuilder &MIB) const {
5003 auto &MRI = *MIB.getMRI();
5004 LLT OpTy = MRI.getType(Reg: LHS);
5005 unsigned CCmpOpc;
5006 std::optional<ValueAndVReg> C;
5007 if (CmpInst::isIntPredicate(P: CC)) {
5008 assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64);
5009 C = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
5010 if (!C || C->Value.sgt(RHS: 31) || C->Value.slt(RHS: -31))
5011 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
5012 else if (C->Value.ule(RHS: 31))
5013 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
5014 else
5015 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
5016 } else {
5017 assert(OpTy.getSizeInBits() == 16 || OpTy.getSizeInBits() == 32 ||
5018 OpTy.getSizeInBits() == 64);
5019 switch (OpTy.getSizeInBits()) {
5020 case 16:
5021 assert(STI.hasFullFP16() && "Expected Full FP16 for fp16 comparisons");
5022 CCmpOpc = AArch64::FCCMPHrr;
5023 break;
5024 case 32:
5025 CCmpOpc = AArch64::FCCMPSrr;
5026 break;
5027 case 64:
5028 CCmpOpc = AArch64::FCCMPDrr;
5029 break;
5030 default:
5031 return nullptr;
5032 }
5033 }
5034 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
5035 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(Code: InvOutCC);
5036 auto CCmp =
5037 MIB.buildInstr(Opc: CCmpOpc, DstOps: {}, SrcOps: {LHS});
5038 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
5039 CCmp.addImm(Val: C->Value.getZExtValue());
5040 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
5041 CCmp.addImm(Val: C->Value.abs().getZExtValue());
5042 else
5043 CCmp.addReg(RegNo: RHS);
5044 CCmp.addImm(Val: NZCV).addImm(Val: Predicate);
5045 constrainSelectedInstRegOperands(I&: *CCmp, TII, TRI, RBI);
5046 return &*CCmp;
5047}
5048
5049MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
5050 Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
5051 AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
5052 // We're at a tree leaf, produce a conditional comparison operation.
5053 auto &MRI = *MIB.getMRI();
5054 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
5055 unsigned Opcode = ValDef->getOpcode();
5056 if (auto *Cmp = dyn_cast<GAnyCmp>(Val: ValDef)) {
5057 Register LHS = Cmp->getLHSReg();
5058 Register RHS = Cmp->getRHSReg();
5059 CmpInst::Predicate CC = Cmp->getCond();
5060 if (Negate)
5061 CC = CmpInst::getInversePredicate(pred: CC);
5062 if (isa<GICmp>(Val: Cmp)) {
5063 OutCC = changeICMPPredToAArch64CC(P: CC, RHS, MRI: MIB.getMRI());
5064 } else {
5065 // Handle special FP cases.
5066 AArch64CC::CondCode ExtraCC;
5067 changeFPCCToANDAArch64CC(CC, CondCode&: OutCC, CondCode2&: ExtraCC);
5068 // Some floating point conditions can't be tested with a single condition
5069 // code. Construct an additional comparison in this case.
5070 if (ExtraCC != AArch64CC::AL) {
5071 MachineInstr *ExtraCmp;
5072 if (!CCOp)
5073 ExtraCmp = emitFPCompare(LHS, RHS, MIRBuilder&: MIB, Pred: CC);
5074 else
5075 ExtraCmp =
5076 emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC: ExtraCC, MIB);
5077 CCOp = ExtraCmp->getOperand(i: 0).getReg();
5078 Predicate = ExtraCC;
5079 }
5080 }
5081
5082 // Produce a normal comparison if we are first in the chain
5083 if (!CCOp) {
5084 if (isa<GICmp>(Val: Cmp))
5085 return emitCMP(LHS&: Cmp->getOperand(i: 2), RHS&: Cmp->getOperand(i: 3), MIRBuilder&: MIB);
5086 return emitFPCompare(LHS: Cmp->getOperand(i: 2).getReg(),
5087 RHS: Cmp->getOperand(i: 3).getReg(), MIRBuilder&: MIB);
5088 }
5089 // Otherwise produce a ccmp.
5090 return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
5091 }
5092 assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree");
5093
5094 bool IsOR = Opcode == TargetOpcode::G_OR;
5095
5096 Register LHS = ValDef->getOperand(i: 1).getReg();
5097 bool CanNegateL;
5098 bool MustBeFirstL;
5099 bool ValidL = canEmitConjunction(Val: LHS, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI);
5100 assert(ValidL && "Valid conjunction/disjunction tree");
5101 (void)ValidL;
5102
5103 Register RHS = ValDef->getOperand(i: 2).getReg();
5104 bool CanNegateR;
5105 bool MustBeFirstR;
5106 bool ValidR = canEmitConjunction(Val: RHS, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI);
5107 assert(ValidR && "Valid conjunction/disjunction tree");
5108 (void)ValidR;
5109
5110 // Swap sub-tree that must come first to the right side.
5111 if (MustBeFirstL) {
5112 assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
5113 std::swap(a&: LHS, b&: RHS);
5114 std::swap(a&: CanNegateL, b&: CanNegateR);
5115 std::swap(a&: MustBeFirstL, b&: MustBeFirstR);
5116 }
5117
5118 bool NegateR;
5119 bool NegateAfterR;
5120 bool NegateL;
5121 bool NegateAfterAll;
5122 if (Opcode == TargetOpcode::G_OR) {
5123 // Swap the sub-tree that we can negate naturally to the left.
5124 if (!CanNegateL) {
5125 assert(CanNegateR && "at least one side must be negatable");
5126 assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
5127 assert(!Negate);
5128 std::swap(a&: LHS, b&: RHS);
5129 NegateR = false;
5130 NegateAfterR = true;
5131 } else {
5132 // Negate the left sub-tree if possible, otherwise negate the result.
5133 NegateR = CanNegateR;
5134 NegateAfterR = !CanNegateR;
5135 }
5136 NegateL = true;
5137 NegateAfterAll = !Negate;
5138 } else {
5139 assert(Opcode == TargetOpcode::G_AND &&
5140 "Valid conjunction/disjunction tree");
5141 assert(!Negate && "Valid conjunction/disjunction tree");
5142
5143 NegateL = false;
5144 NegateR = false;
5145 NegateAfterR = false;
5146 NegateAfterAll = false;
5147 }
5148
5149 // Emit sub-trees.
5150 AArch64CC::CondCode RHSCC;
5151 MachineInstr *CmpR =
5152 emitConjunctionRec(Val: RHS, OutCC&: RHSCC, Negate: NegateR, CCOp, Predicate, MIB);
5153 if (NegateAfterR)
5154 RHSCC = AArch64CC::getInvertedCondCode(Code: RHSCC);
5155 MachineInstr *CmpL = emitConjunctionRec(
5156 Val: LHS, OutCC, Negate: NegateL, CCOp: CmpR->getOperand(i: 0).getReg(), Predicate: RHSCC, MIB);
5157 if (NegateAfterAll)
5158 OutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
5159 return CmpL;
5160}
5161
5162MachineInstr *AArch64InstructionSelector::emitConjunction(
5163 Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
5164 bool DummyCanNegate;
5165 bool DummyMustBeFirst;
5166 if (!canEmitConjunction(Val, CanNegate&: DummyCanNegate, MustBeFirst&: DummyMustBeFirst, WillNegate: false,
5167 MRI&: *MIB.getMRI()))
5168 return nullptr;
5169 return emitConjunctionRec(Val, OutCC, Negate: false, CCOp: Register(), Predicate: AArch64CC::AL, MIB);
5170}
5171
5172bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
5173 MachineInstr &CondMI) {
5174 AArch64CC::CondCode AArch64CC;
5175 MachineInstr *ConjMI = emitConjunction(Val: SelI.getCondReg(), OutCC&: AArch64CC, MIB);
5176 if (!ConjMI)
5177 return false;
5178
5179 emitSelect(Dst: SelI.getReg(Idx: 0), True: SelI.getTrueReg(), False: SelI.getFalseReg(), CC: AArch64CC, MIB);
5180 SelI.eraseFromParent();
5181 return true;
5182}
5183
5184bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
5185 MachineRegisterInfo &MRI = *MIB.getMRI();
5186 // We want to recognize this pattern:
5187 //
5188 // $z = G_FCMP pred, $x, $y
5189 // ...
5190 // $w = G_SELECT $z, $a, $b
5191 //
5192 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
5193 // some copies/truncs in between.)
5194 //
5195 // If we see this, then we can emit something like this:
5196 //
5197 // fcmp $x, $y
5198 // fcsel $w, $a, $b, pred
5199 //
5200 // Rather than emitting both of the rather long sequences in the standard
5201 // G_FCMP/G_SELECT select methods.
5202
5203 // First, check if the condition is defined by a compare.
5204 MachineInstr *CondDef = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
5205
5206 // We can only fold if all of the defs have one use.
5207 Register CondDefReg = CondDef->getOperand(i: 0).getReg();
5208 if (!MRI.hasOneNonDBGUse(RegNo: CondDefReg)) {
5209 // Unless it's another select.
5210 for (const MachineInstr &UI : MRI.use_nodbg_instructions(Reg: CondDefReg)) {
5211 if (CondDef == &UI)
5212 continue;
5213 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5214 return false;
5215 }
5216 }
5217
5218 // Is the condition defined by a compare?
5219 unsigned CondOpc = CondDef->getOpcode();
5220 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5221 if (tryOptSelectConjunction(SelI&: I, CondMI&: *CondDef))
5222 return true;
5223 return false;
5224 }
5225
5226 AArch64CC::CondCode CondCode;
5227 if (CondOpc == TargetOpcode::G_ICMP) {
5228 auto &PredOp = CondDef->getOperand(i: 1);
5229 emitIntegerCompare(LHS&: CondDef->getOperand(i: 2), RHS&: CondDef->getOperand(i: 3), Predicate&: PredOp,
5230 MIRBuilder&: MIB);
5231 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
5232 CondCode =
5233 changeICMPPredToAArch64CC(P: Pred, RHS: CondDef->getOperand(i: 3).getReg(), MRI: &MRI);
5234 } else {
5235 // Get the condition code for the select.
5236 auto Pred =
5237 static_cast<CmpInst::Predicate>(CondDef->getOperand(i: 1).getPredicate());
5238 AArch64CC::CondCode CondCode2;
5239 changeFCMPPredToAArch64CC(P: Pred, CondCode, CondCode2);
5240
5241 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
5242 // instructions to emit the comparison.
5243 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
5244 // unnecessary.
5245 if (CondCode2 != AArch64CC::AL)
5246 return false;
5247
5248 if (!emitFPCompare(LHS: CondDef->getOperand(i: 2).getReg(),
5249 RHS: CondDef->getOperand(i: 3).getReg(), MIRBuilder&: MIB)) {
5250 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n");
5251 return false;
5252 }
5253 }
5254
5255 // Emit the select.
5256 emitSelect(Dst: I.getOperand(i: 0).getReg(), True: I.getOperand(i: 2).getReg(),
5257 False: I.getOperand(i: 3).getReg(), CC: CondCode, MIB);
5258 I.eraseFromParent();
5259 return true;
5260}
5261
5262MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5263 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
5264 MachineIRBuilder &MIRBuilder) const {
5265 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&
5266 "Unexpected MachineOperand");
5267 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5268 // We want to find this sort of thing:
5269 // x = G_SUB 0, y
5270 // G_ICMP z, x
5271 //
5272 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
5273 // e.g:
5274 //
5275 // cmn z, y
5276
5277 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
5278 MachineInstr *LHSDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
5279 MachineInstr *RHSDef = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
5280 auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
5281
5282 // Given this:
5283 //
5284 // x = G_SUB 0, y
5285 // G_ICMP z, x
5286 //
5287 // Produce this:
5288 //
5289 // cmn z, y
5290 if (isCMN(MaybeSub: RHSDef, Pred: P, MRI))
5291 return emitCMN(LHS, RHS&: RHSDef->getOperand(i: 2), MIRBuilder);
5292
5293 // Same idea here, but with the LHS of the compare instead:
5294 //
5295 // Given this:
5296 //
5297 // x = G_SUB 0, y
5298 // G_ICMP x, z
5299 //
5300 // Produce this:
5301 //
5302 // cmn y, z
5303 //
5304 // But be careful! We need to swap the predicate!
5305 if (isCMN(MaybeSub: LHSDef, Pred: P, MRI)) {
5306 if (!CmpInst::isEquality(pred: P)) {
5307 P = CmpInst::getSwappedPredicate(pred: P);
5308 Predicate = MachineOperand::CreatePredicate(Pred: P);
5309 }
5310 return emitCMN(LHS&: LHSDef->getOperand(i: 2), RHS, MIRBuilder);
5311 }
5312
5313 // Given this:
5314 //
5315 // z = G_AND x, y
5316 // G_ICMP z, 0
5317 //
5318 // Produce this if the compare is signed:
5319 //
5320 // tst x, y
5321 if (!CmpInst::isUnsigned(Pred: P) && LHSDef &&
5322 LHSDef->getOpcode() == TargetOpcode::G_AND) {
5323 // Make sure that the RHS is 0.
5324 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
5325 if (!ValAndVReg || ValAndVReg->Value != 0)
5326 return nullptr;
5327
5328 return emitTST(LHS&: LHSDef->getOperand(i: 1),
5329 RHS&: LHSDef->getOperand(i: 2), MIRBuilder);
5330 }
5331
5332 return nullptr;
5333}
5334
5335bool AArch64InstructionSelector::selectShuffleVector(
5336 MachineInstr &I, MachineRegisterInfo &MRI) {
5337 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5338 Register Src1Reg = I.getOperand(i: 1).getReg();
5339 Register Src2Reg = I.getOperand(i: 2).getReg();
5340 ArrayRef<int> Mask = I.getOperand(i: 3).getShuffleMask();
5341 assert(DstTy == MRI.getType(Src1Reg) &&
5342 "Expected equal shuffle types during selection");
5343
5344 MachineBasicBlock &MBB = *I.getParent();
5345 MachineFunction &MF = *MBB.getParent();
5346 LLVMContext &Ctx = MF.getFunction().getContext();
5347
5348 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
5349 int NumElts = DstTy.getNumElements();
5350
5351 SmallVector<int> NewMask;
5352 bool FirstUsed = false;
5353 bool SecondUsed = false;
5354 for (int M : Mask) {
5355 // Map any undef or zero lanes to 255.
5356 if (M < 0 || VT->getKnownBits(R: M < NumElts ? Src1Reg : Src2Reg,
5357 DemandedElts: APInt::getOneBitSet(numBits: NumElts, BitNo: M % NumElts))
5358 .isZero()) {
5359 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte)
5360 NewMask.push_back(Elt: 255);
5361 continue;
5362 }
5363
5364 FirstUsed |= M < NumElts;
5365 SecondUsed |= M >= NumElts;
5366 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
5367 unsigned Offset = Byte + M * BytesPerElt;
5368 NewMask.push_back(Elt: Offset);
5369 }
5370 }
5371
5372 // If the first is unused or all zeros, use the second src in a tbl1.
5373 if (!FirstUsed) {
5374 int ByteLanes = DstTy.getSizeInBits() == 128 ? 16 : 8;
5375 for (int &M : NewMask) {
5376 if (M != 255) {
5377 assert(M >= ByteLanes && M < 2 * ByteLanes);
5378 M -= ByteLanes;
5379 }
5380 }
5381 std::swap(a&: Src1Reg, b&: Src2Reg);
5382 std::swap(a&: FirstUsed, b&: SecondUsed);
5383 }
5384
5385 // Use a constant pool to load the index vector for TBL.
5386 SmallVector<Constant *> CstIdxs;
5387 transform(Range&: NewMask, d_first: std::back_inserter(x&: CstIdxs), F: [&Ctx](int M) {
5388 return ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: M);
5389 });
5390 Constant *CPVal = ConstantVector::get(V: CstIdxs);
5391 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder&: MIB);
5392 if (!IndexLoad) {
5393 LLVM_DEBUG(dbgs() << "Could not load from a constant pool");
5394 return false;
5395 }
5396
5397 if (DstTy.getSizeInBits() != 128) {
5398 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty");
5399 // This case can be done with TBL1.
5400 MachineInstr *Concat =
5401 emitVectorConcat(Dst: std::nullopt, Op1: Src1Reg, Op2: Src2Reg, MIRBuilder&: MIB);
5402 if (!Concat) {
5403 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1");
5404 return false;
5405 }
5406
5407 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
5408 IndexLoad = emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass,
5409 Scalar: IndexLoad->getOperand(i: 0).getReg(), MIRBuilder&: MIB);
5410
5411 auto TBL1 = MIB.buildInstr(
5412 Opc: AArch64::TBLv16i8One, DstOps: {&AArch64::FPR128RegClass},
5413 SrcOps: {Concat->getOperand(i: 0).getReg(), IndexLoad->getOperand(i: 0).getReg()});
5414 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5415
5416 auto Copy =
5417 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
5418 .addReg(RegNo: TBL1.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5419 RBI.constrainGenericRegister(Reg: Copy.getReg(Idx: 0), RC: AArch64::FPR64RegClass, MRI);
5420 I.eraseFromParent();
5421 return true;
5422 }
5423
5424 if (!SecondUsed) {
5425 auto TBL1 = MIB.buildInstr(Opc: AArch64::TBLv16i8One, DstOps: {I.getOperand(i: 0)},
5426 SrcOps: {Src1Reg, IndexLoad->getOperand(i: 0)});
5427 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5428 I.eraseFromParent();
5429 return true;
5430 }
5431
5432 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
5433 // Q registers for regalloc.
5434 SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
5435 auto RegSeq = createQTuple(Regs, MIB);
5436 auto TBL2 = MIB.buildInstr(Opc: AArch64::TBLv16i8Two, DstOps: {I.getOperand(i: 0)},
5437 SrcOps: {RegSeq, IndexLoad->getOperand(i: 0)});
5438 constrainSelectedInstRegOperands(I&: *TBL2, TII, TRI, RBI);
5439 I.eraseFromParent();
5440 return true;
5441}
5442
5443MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5444 std::optional<Register> DstReg, Register SrcReg, Register EltReg,
5445 unsigned LaneIdx, const RegisterBank &RB,
5446 MachineIRBuilder &MIRBuilder) const {
5447 MachineInstr *InsElt = nullptr;
5448 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5449 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5450
5451 // Create a register to define with the insert if one wasn't passed in.
5452 if (!DstReg)
5453 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
5454
5455 unsigned EltSize = MRI.getType(Reg: EltReg).getSizeInBits();
5456 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
5457
5458 if (RB.getID() == AArch64::FPRRegBankID) {
5459 auto InsSub = emitScalarToVector(EltSize, DstRC, Scalar: EltReg, MIRBuilder);
5460 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5461 .addImm(Val: LaneIdx)
5462 .addUse(RegNo: InsSub->getOperand(i: 0).getReg())
5463 .addImm(Val: 0);
5464 } else {
5465 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5466 .addImm(Val: LaneIdx)
5467 .addUse(RegNo: EltReg);
5468 }
5469
5470 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
5471 return InsElt;
5472}
5473
5474bool AArch64InstructionSelector::selectUSMovFromExtend(
5475 MachineInstr &MI, MachineRegisterInfo &MRI) {
5476 if (MI.getOpcode() != TargetOpcode::G_SEXT &&
5477 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5478 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5479 return false;
5480 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
5481 const Register DefReg = MI.getOperand(i: 0).getReg();
5482 const LLT DstTy = MRI.getType(Reg: DefReg);
5483 unsigned DstSize = DstTy.getSizeInBits();
5484
5485 if (DstSize != 32 && DstSize != 64)
5486 return false;
5487
5488 MachineInstr *Extract = getOpcodeDef(Opcode: TargetOpcode::G_EXTRACT_VECTOR_ELT,
5489 Reg: MI.getOperand(i: 1).getReg(), MRI);
5490 int64_t Lane;
5491 if (!Extract || !mi_match(R: Extract->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Lane)))
5492 return false;
5493 Register Src0 = Extract->getOperand(i: 1).getReg();
5494
5495 const LLT VecTy = MRI.getType(Reg: Src0);
5496 if (VecTy.isScalableVector())
5497 return false;
5498
5499 if (VecTy.getSizeInBits() != 128) {
5500 const MachineInstr *ScalarToVector = emitScalarToVector(
5501 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: Src0, MIRBuilder&: MIB);
5502 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
5503 Src0 = ScalarToVector->getOperand(i: 0).getReg();
5504 }
5505
5506 unsigned Opcode;
5507 if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
5508 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5509 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
5510 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5511 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
5512 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5513 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
5514 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5515 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
5516 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5517 else
5518 llvm_unreachable("Unexpected type combo for S/UMov!");
5519
5520 // We may need to generate one of these, depending on the type and sign of the
5521 // input:
5522 // DstReg = SMOV Src0, Lane;
5523 // NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
5524 MachineInstr *ExtI = nullptr;
5525 if (DstSize == 64 && !IsSigned) {
5526 Register NewReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
5527 MIB.buildInstr(Opc: Opcode, DstOps: {NewReg}, SrcOps: {Src0}).addImm(Val: Lane);
5528 ExtI = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
5529 .addUse(RegNo: NewReg)
5530 .addImm(Val: AArch64::sub_32);
5531 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
5532 } else
5533 ExtI = MIB.buildInstr(Opc: Opcode, DstOps: {DefReg}, SrcOps: {Src0}).addImm(Val: Lane);
5534
5535 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
5536 MI.eraseFromParent();
5537 return true;
5538}
5539
5540MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm8(
5541 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5542 unsigned int Op;
5543 if (DstSize == 128) {
5544 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5545 return nullptr;
5546 Op = AArch64::MOVIv16b_ns;
5547 } else {
5548 Op = AArch64::MOVIv8b_ns;
5549 }
5550
5551 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5552
5553 if (AArch64_AM::isAdvSIMDModImmType9(Imm: Val)) {
5554 Val = AArch64_AM::encodeAdvSIMDModImmType9(Imm: Val);
5555 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5556 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5557 return &*Mov;
5558 }
5559 return nullptr;
5560}
5561
5562MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm16(
5563 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5564 bool Inv) {
5565
5566 unsigned int Op;
5567 if (DstSize == 128) {
5568 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5569 return nullptr;
5570 Op = Inv ? AArch64::MVNIv8i16 : AArch64::MOVIv8i16;
5571 } else {
5572 Op = Inv ? AArch64::MVNIv4i16 : AArch64::MOVIv4i16;
5573 }
5574
5575 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5576 uint64_t Shift;
5577
5578 if (AArch64_AM::isAdvSIMDModImmType5(Imm: Val)) {
5579 Val = AArch64_AM::encodeAdvSIMDModImmType5(Imm: Val);
5580 Shift = 0;
5581 } else if (AArch64_AM::isAdvSIMDModImmType6(Imm: Val)) {
5582 Val = AArch64_AM::encodeAdvSIMDModImmType6(Imm: Val);
5583 Shift = 8;
5584 } else
5585 return nullptr;
5586
5587 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5588 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5589 return &*Mov;
5590}
5591
5592MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm32(
5593 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5594 bool Inv) {
5595
5596 unsigned int Op;
5597 if (DstSize == 128) {
5598 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5599 return nullptr;
5600 Op = Inv ? AArch64::MVNIv4i32 : AArch64::MOVIv4i32;
5601 } else {
5602 Op = Inv ? AArch64::MVNIv2i32 : AArch64::MOVIv2i32;
5603 }
5604
5605 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5606 uint64_t Shift;
5607
5608 if ((AArch64_AM::isAdvSIMDModImmType1(Imm: Val))) {
5609 Val = AArch64_AM::encodeAdvSIMDModImmType1(Imm: Val);
5610 Shift = 0;
5611 } else if ((AArch64_AM::isAdvSIMDModImmType2(Imm: Val))) {
5612 Val = AArch64_AM::encodeAdvSIMDModImmType2(Imm: Val);
5613 Shift = 8;
5614 } else if ((AArch64_AM::isAdvSIMDModImmType3(Imm: Val))) {
5615 Val = AArch64_AM::encodeAdvSIMDModImmType3(Imm: Val);
5616 Shift = 16;
5617 } else if ((AArch64_AM::isAdvSIMDModImmType4(Imm: Val))) {
5618 Val = AArch64_AM::encodeAdvSIMDModImmType4(Imm: Val);
5619 Shift = 24;
5620 } else
5621 return nullptr;
5622
5623 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5624 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5625 return &*Mov;
5626}
5627
5628MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm64(
5629 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5630
5631 unsigned int Op;
5632 if (DstSize == 128) {
5633 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5634 return nullptr;
5635 Op = AArch64::MOVIv2d_ns;
5636 } else {
5637 Op = AArch64::MOVID;
5638 }
5639
5640 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5641 if (AArch64_AM::isAdvSIMDModImmType10(Imm: Val)) {
5642 Val = AArch64_AM::encodeAdvSIMDModImmType10(Imm: Val);
5643 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5644 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5645 return &*Mov;
5646 }
5647 return nullptr;
5648}
5649
5650MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm321s(
5651 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder,
5652 bool Inv) {
5653
5654 unsigned int Op;
5655 if (DstSize == 128) {
5656 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5657 return nullptr;
5658 Op = Inv ? AArch64::MVNIv4s_msl : AArch64::MOVIv4s_msl;
5659 } else {
5660 Op = Inv ? AArch64::MVNIv2s_msl : AArch64::MOVIv2s_msl;
5661 }
5662
5663 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5664 uint64_t Shift;
5665
5666 if (AArch64_AM::isAdvSIMDModImmType7(Imm: Val)) {
5667 Val = AArch64_AM::encodeAdvSIMDModImmType7(Imm: Val);
5668 Shift = 264;
5669 } else if (AArch64_AM::isAdvSIMDModImmType8(Imm: Val)) {
5670 Val = AArch64_AM::encodeAdvSIMDModImmType8(Imm: Val);
5671 Shift = 272;
5672 } else
5673 return nullptr;
5674
5675 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val).addImm(Val: Shift);
5676 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5677 return &*Mov;
5678}
5679
5680MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImmFP(
5681 Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
5682
5683 unsigned int Op;
5684 bool IsWide = false;
5685 if (DstSize == 128) {
5686 if (Bits.getHiBits(numBits: 64) != Bits.getLoBits(numBits: 64))
5687 return nullptr;
5688 Op = AArch64::FMOVv4f32_ns;
5689 IsWide = true;
5690 } else {
5691 Op = AArch64::FMOVv2f32_ns;
5692 }
5693
5694 uint64_t Val = Bits.zextOrTrunc(width: 64).getZExtValue();
5695
5696 if (AArch64_AM::isAdvSIMDModImmType11(Imm: Val)) {
5697 Val = AArch64_AM::encodeAdvSIMDModImmType11(Imm: Val);
5698 } else if (IsWide && AArch64_AM::isAdvSIMDModImmType12(Imm: Val)) {
5699 Val = AArch64_AM::encodeAdvSIMDModImmType12(Imm: Val);
5700 Op = AArch64::FMOVv2f64_ns;
5701 } else
5702 return nullptr;
5703
5704 auto Mov = Builder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: {}).addImm(Val);
5705 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5706 return &*Mov;
5707}
5708
5709bool AArch64InstructionSelector::selectIndexedExtLoad(
5710 MachineInstr &MI, MachineRegisterInfo &MRI) {
5711 auto &ExtLd = cast<GIndexedAnyExtLoad>(Val&: MI);
5712 Register Dst = ExtLd.getDstReg();
5713 Register WriteBack = ExtLd.getWritebackReg();
5714 Register Base = ExtLd.getBaseReg();
5715 Register Offset = ExtLd.getOffsetReg();
5716 LLT Ty = MRI.getType(Reg: Dst);
5717 assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
5718 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5719 bool IsPre = ExtLd.isPre();
5720 bool IsSExt = isa<GIndexedSExtLoad>(Val: ExtLd);
5721 unsigned InsertIntoSubReg = 0;
5722 bool IsDst64 = Ty.getSizeInBits() == 64;
5723
5724 // ZExt/SExt should be on gpr but can handle extload and zextload of fpr, so
5725 // long as they are scalar.
5726 bool IsFPR = RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID;
5727 if ((IsSExt && IsFPR) || Ty.isVector())
5728 return false;
5729
5730 unsigned Opc = 0;
5731 LLT NewLdDstTy;
5732 LLT s32 = LLT::scalar(SizeInBits: 32);
5733 LLT s64 = LLT::scalar(SizeInBits: 64);
5734
5735 if (MemSizeBits == 8) {
5736 if (IsSExt) {
5737 if (IsDst64)
5738 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5739 else
5740 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5741 NewLdDstTy = IsDst64 ? s64 : s32;
5742 } else if (IsFPR) {
5743 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5744 InsertIntoSubReg = AArch64::bsub;
5745 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5746 } else {
5747 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5748 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5749 NewLdDstTy = s32;
5750 }
5751 } else if (MemSizeBits == 16) {
5752 if (IsSExt) {
5753 if (IsDst64)
5754 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5755 else
5756 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5757 NewLdDstTy = IsDst64 ? s64 : s32;
5758 } else if (IsFPR) {
5759 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5760 InsertIntoSubReg = AArch64::hsub;
5761 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5762 } else {
5763 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5764 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5765 NewLdDstTy = s32;
5766 }
5767 } else if (MemSizeBits == 32) {
5768 if (IsSExt) {
5769 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5770 NewLdDstTy = s64;
5771 } else if (IsFPR) {
5772 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5773 InsertIntoSubReg = AArch64::ssub;
5774 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5775 } else {
5776 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5777 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5778 NewLdDstTy = s32;
5779 }
5780 } else {
5781 llvm_unreachable("Unexpected size for indexed load");
5782 }
5783
5784 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5785 if (!Cst)
5786 return false; // Shouldn't happen, but just in case.
5787
5788 auto LdMI = MIB.buildInstr(Opc, DstOps: {WriteBack, NewLdDstTy}, SrcOps: {Base})
5789 .addImm(Val: Cst->getSExtValue());
5790 LdMI.cloneMemRefs(OtherMI: ExtLd);
5791 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5792 // Make sure to select the load with the MemTy as the dest type, and then
5793 // insert into a larger reg if needed.
5794 if (InsertIntoSubReg) {
5795 // Generate a SUBREG_TO_REG.
5796 auto SubToReg = MIB.buildInstr(Opc: TargetOpcode::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5797 .addUse(RegNo: LdMI.getReg(Idx: 1))
5798 .addImm(Val: InsertIntoSubReg);
5799 RBI.constrainGenericRegister(
5800 Reg: SubToReg.getReg(Idx: 0),
5801 RC: *getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst),
5802 RB: *RBI.getRegBank(Reg: Dst, MRI, TRI)),
5803 MRI);
5804 } else {
5805 auto Copy = MIB.buildCopy(Res: Dst, Op: LdMI.getReg(Idx: 1));
5806 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
5807 }
5808 MI.eraseFromParent();
5809
5810 return true;
5811}
5812
5813bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
5814 MachineRegisterInfo &MRI) {
5815 auto &Ld = cast<GIndexedLoad>(Val&: MI);
5816 Register Dst = Ld.getDstReg();
5817 Register WriteBack = Ld.getWritebackReg();
5818 Register Base = Ld.getBaseReg();
5819 Register Offset = Ld.getOffsetReg();
5820 assert(MRI.getType(Dst).getSizeInBits() <= 128 &&
5821 "Unexpected type for indexed load");
5822 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5823
5824 if (MemSize < MRI.getType(Reg: Dst).getSizeInBytes())
5825 return selectIndexedExtLoad(MI, MRI);
5826
5827 unsigned Opc = 0;
5828 if (Ld.isPre()) {
5829 static constexpr unsigned GPROpcodes[] = {
5830 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5831 AArch64::LDRXpre};
5832 static constexpr unsigned FPROpcodes[] = {
5833 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5834 AArch64::LDRQpre};
5835 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5836 ? FPROpcodes[Log2_32(Value: MemSize)]
5837 : GPROpcodes[Log2_32(Value: MemSize)];
5838 ;
5839 } else {
5840 static constexpr unsigned GPROpcodes[] = {
5841 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5842 AArch64::LDRXpost};
5843 static constexpr unsigned FPROpcodes[] = {
5844 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5845 AArch64::LDRDpost, AArch64::LDRQpost};
5846 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5847 ? FPROpcodes[Log2_32(Value: MemSize)]
5848 : GPROpcodes[Log2_32(Value: MemSize)];
5849 ;
5850 }
5851 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5852 if (!Cst)
5853 return false; // Shouldn't happen, but just in case.
5854 auto LdMI =
5855 MIB.buildInstr(Opc, DstOps: {WriteBack, Dst}, SrcOps: {Base}).addImm(Val: Cst->getSExtValue());
5856 LdMI.cloneMemRefs(OtherMI: Ld);
5857 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5858 MI.eraseFromParent();
5859 return true;
5860}
5861
5862bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &I,
5863 MachineRegisterInfo &MRI) {
5864 Register Dst = I.getWritebackReg();
5865 Register Val = I.getValueReg();
5866 Register Base = I.getBaseReg();
5867 Register Offset = I.getOffsetReg();
5868 assert(MRI.getType(Val).getSizeInBits() <= 128 &&
5869 "Unexpected type for indexed store");
5870
5871 LocationSize MemSize = I.getMMO().getSize();
5872 unsigned MemSizeInBytes = MemSize.getValue();
5873
5874 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5875 "Unexpected indexed store size");
5876 unsigned MemSizeLog2 = Log2_32(Value: MemSizeInBytes);
5877
5878 unsigned Opc = 0;
5879 if (I.isPre()) {
5880 static constexpr unsigned GPROpcodes[] = {
5881 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5882 AArch64::STRXpre};
5883 static constexpr unsigned FPROpcodes[] = {
5884 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5885 AArch64::STRQpre};
5886
5887 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5888 Opc = FPROpcodes[MemSizeLog2];
5889 else
5890 Opc = GPROpcodes[MemSizeLog2];
5891 } else {
5892 static constexpr unsigned GPROpcodes[] = {
5893 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5894 AArch64::STRXpost};
5895 static constexpr unsigned FPROpcodes[] = {
5896 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5897 AArch64::STRDpost, AArch64::STRQpost};
5898
5899 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5900 Opc = FPROpcodes[MemSizeLog2];
5901 else
5902 Opc = GPROpcodes[MemSizeLog2];
5903 }
5904
5905 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5906 if (!Cst)
5907 return false; // Shouldn't happen, but just in case.
5908 auto Str =
5909 MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Val, Base}).addImm(Val: Cst->getSExtValue());
5910 Str.cloneMemRefs(OtherMI: I);
5911 constrainSelectedInstRegOperands(I&: *Str, TII, TRI, RBI);
5912 I.eraseFromParent();
5913 return true;
5914}
5915
5916MachineInstr *
5917AArch64InstructionSelector::emitConstantVector(Register Dst, Constant *CV,
5918 MachineIRBuilder &MIRBuilder,
5919 MachineRegisterInfo &MRI) {
5920 LLT DstTy = MRI.getType(Reg: Dst);
5921 unsigned DstSize = DstTy.getSizeInBits();
5922 assert((DstSize == 64 || DstSize == 128) &&
5923 "Unexpected vector constant size");
5924
5925 if (CV->isNullValue()) {
5926 if (DstSize == 128) {
5927 auto Mov =
5928 MIRBuilder.buildInstr(Opc: AArch64::MOVIv2d_ns, DstOps: {Dst}, SrcOps: {}).addImm(Val: 0);
5929 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5930 return &*Mov;
5931 }
5932
5933 if (DstSize == 64) {
5934 auto Mov =
5935 MIRBuilder
5936 .buildInstr(Opc: AArch64::MOVIv2d_ns, DstOps: {&AArch64::FPR128RegClass}, SrcOps: {})
5937 .addImm(Val: 0);
5938 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5939 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5940 RBI.constrainGenericRegister(Reg: Dst, RC: AArch64::FPR64RegClass, MRI);
5941 return &*Copy;
5942 }
5943 }
5944
5945 if (Constant *SplatValue = CV->getSplatValue()) {
5946 APInt SplatValueAsInt =
5947 isa<ConstantFP>(Val: SplatValue)
5948 ? cast<ConstantFP>(Val: SplatValue)->getValueAPF().bitcastToAPInt()
5949 : SplatValue->getUniqueInteger();
5950 APInt DefBits = APInt::getSplat(
5951 NewLen: DstSize, V: SplatValueAsInt.trunc(width: DstTy.getScalarSizeInBits()));
5952 auto TryMOVIWithBits = [&](APInt DefBits) -> MachineInstr * {
5953 MachineInstr *NewOp;
5954 bool Inv = false;
5955 if ((NewOp = tryAdvSIMDModImm64(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)) ||
5956 (NewOp =
5957 tryAdvSIMDModImm32(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5958 (NewOp =
5959 tryAdvSIMDModImm321s(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5960 (NewOp =
5961 tryAdvSIMDModImm16(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5962 (NewOp = tryAdvSIMDModImm8(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)) ||
5963 (NewOp = tryAdvSIMDModImmFP(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder)))
5964 return NewOp;
5965
5966 DefBits = ~DefBits;
5967 Inv = true;
5968 if ((NewOp =
5969 tryAdvSIMDModImm32(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5970 (NewOp =
5971 tryAdvSIMDModImm321s(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)) ||
5972 (NewOp = tryAdvSIMDModImm16(Dst, DstSize, Bits: DefBits, Builder&: MIRBuilder, Inv)))
5973 return NewOp;
5974 return nullptr;
5975 };
5976
5977 if (auto *NewOp = TryMOVIWithBits(DefBits))
5978 return NewOp;
5979
5980 // See if a fneg of the constant can be materialized with a MOVI, etc
5981 auto TryWithFNeg = [&](APInt DefBits, int NumBits,
5982 unsigned NegOpc) -> MachineInstr * {
5983 // FNegate each sub-element of the constant
5984 APInt Neg = APInt::getHighBitsSet(numBits: NumBits, hiBitsSet: 1).zext(width: DstSize);
5985 APInt NegBits(DstSize, 0);
5986 unsigned NumElts = DstSize / NumBits;
5987 for (unsigned i = 0; i < NumElts; i++)
5988 NegBits |= Neg << (NumBits * i);
5989 NegBits = DefBits ^ NegBits;
5990
5991 // Try to create the new constants with MOVI, and if so generate a fneg
5992 // for it.
5993 if (auto *NewOp = TryMOVIWithBits(NegBits)) {
5994 Register NewDst = MRI.createVirtualRegister(
5995 RegClass: DstSize == 64 ? &AArch64::FPR64RegClass : &AArch64::FPR128RegClass);
5996 NewOp->getOperand(i: 0).setReg(NewDst);
5997 return MIRBuilder.buildInstr(Opc: NegOpc, DstOps: {Dst}, SrcOps: {NewDst});
5998 }
5999 return nullptr;
6000 };
6001 MachineInstr *R;
6002 if ((R = TryWithFNeg(DefBits, 32,
6003 DstSize == 64 ? AArch64::FNEGv2f32
6004 : AArch64::FNEGv4f32)) ||
6005 (R = TryWithFNeg(DefBits, 64,
6006 DstSize == 64 ? AArch64::FNEGDr
6007 : AArch64::FNEGv2f64)) ||
6008 (STI.hasFullFP16() &&
6009 (R = TryWithFNeg(DefBits, 16,
6010 DstSize == 64 ? AArch64::FNEGv4f16
6011 : AArch64::FNEGv8f16))))
6012 return R;
6013 }
6014
6015 auto *CPLoad = emitLoadFromConstantPool(CPVal: CV, MIRBuilder);
6016 if (!CPLoad) {
6017 LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!");
6018 return nullptr;
6019 }
6020
6021 auto Copy = MIRBuilder.buildCopy(Res: Dst, Op: CPLoad->getOperand(i: 0));
6022 RBI.constrainGenericRegister(
6023 Reg: Dst, RC: *MRI.getRegClass(Reg: CPLoad->getOperand(i: 0).getReg()), MRI);
6024 return &*Copy;
6025}
6026
6027bool AArch64InstructionSelector::tryOptConstantBuildVec(
6028 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
6029 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
6030 unsigned DstSize = DstTy.getSizeInBits();
6031 assert(DstSize <= 128 && "Unexpected build_vec type!");
6032 if (DstSize < 32)
6033 return false;
6034 // Check if we're building a constant vector, in which case we want to
6035 // generate a constant pool load instead of a vector insert sequence.
6036 SmallVector<Constant *, 16> Csts;
6037 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
6038 Register OpReg = I.getOperand(i: Idx).getReg();
6039 if (auto AnyConst = getAnyConstantVRegValWithLookThrough(
6040 VReg: OpReg, MRI, /*LookThroughInstrs=*/true,
6041 /*LookThroughAnyExt=*/true)) {
6042 MachineInstr *DefMI = MRI.getVRegDef(Reg: AnyConst->VReg);
6043
6044 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT) {
6045 Csts.emplace_back(
6046 Args: ConstantInt::get(Context&: MIB.getMF().getFunction().getContext(),
6047 V: std::move(AnyConst->Value)));
6048 continue;
6049 }
6050
6051 if (DefMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
6052 Csts.emplace_back(
6053 Args: const_cast<ConstantFP *>(DefMI->getOperand(i: 1).getFPImm()));
6054 continue;
6055 }
6056 }
6057 return false;
6058 }
6059 Constant *CV = ConstantVector::get(V: Csts);
6060 if (!emitConstantVector(Dst: I.getOperand(i: 0).getReg(), CV, MIRBuilder&: MIB, MRI))
6061 return false;
6062 I.eraseFromParent();
6063 return true;
6064}
6065
6066bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
6067 MachineInstr &I, MachineRegisterInfo &MRI) {
6068 // Given:
6069 // %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
6070 //
6071 // Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
6072 Register Dst = I.getOperand(i: 0).getReg();
6073 Register EltReg = I.getOperand(i: 1).getReg();
6074 LLT EltTy = MRI.getType(Reg: EltReg);
6075 // If the index isn't on the same bank as its elements, then this can't be a
6076 // SUBREG_TO_REG.
6077 const RegisterBank &EltRB = *RBI.getRegBank(Reg: EltReg, MRI, TRI);
6078 const RegisterBank &DstRB = *RBI.getRegBank(Reg: Dst, MRI, TRI);
6079 if (EltRB != DstRB)
6080 return false;
6081 if (any_of(Range: drop_begin(RangeOrContainer: I.operands(), N: 2), P: [&MRI](const MachineOperand &Op) {
6082 return !getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Op.getReg(), MRI);
6083 }))
6084 return false;
6085 unsigned SubReg;
6086 const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(Ty: EltTy, RB: EltRB);
6087 if (!EltRC)
6088 return false;
6089 const TargetRegisterClass *DstRC =
6090 getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst), RB: DstRB);
6091 if (!DstRC)
6092 return false;
6093 if (!getSubRegForClass(RC: EltRC, TRI, SubReg))
6094 return false;
6095 auto SubregToReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
6096 .addUse(RegNo: EltReg)
6097 .addImm(Val: SubReg);
6098 I.eraseFromParent();
6099 constrainSelectedInstRegOperands(I&: *SubregToReg, TII, TRI, RBI);
6100 return RBI.constrainGenericRegister(Reg: Dst, RC: *DstRC, MRI);
6101}
6102
6103bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
6104 MachineRegisterInfo &MRI) {
6105 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
6106 // Until we port more of the optimized selections, for now just use a vector
6107 // insert sequence.
6108 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6109 const LLT EltTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6110 unsigned EltSize = EltTy.getSizeInBits();
6111
6112 if (tryOptConstantBuildVec(I, DstTy, MRI))
6113 return true;
6114 if (tryOptBuildVecToSubregToReg(I, MRI))
6115 return true;
6116
6117 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
6118 return false; // Don't support all element types yet.
6119 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
6120
6121 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
6122 MachineInstr *ScalarToVec =
6123 emitScalarToVector(EltSize: DstTy.getElementType().getSizeInBits(), DstRC,
6124 Scalar: I.getOperand(i: 1).getReg(), MIRBuilder&: MIB);
6125 if (!ScalarToVec)
6126 return false;
6127
6128 Register DstVec = ScalarToVec->getOperand(i: 0).getReg();
6129 unsigned DstSize = DstTy.getSizeInBits();
6130
6131 // Keep track of the last MI we inserted. Later on, we might be able to save
6132 // a copy using it.
6133 MachineInstr *PrevMI = ScalarToVec;
6134 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
6135 // Note that if we don't do a subregister copy, we can end up making an
6136 // extra register.
6137 Register OpReg = I.getOperand(i).getReg();
6138 // Do not emit inserts for undefs
6139 if (!getOpcodeDef<GImplicitDef>(Reg: OpReg, MRI)) {
6140 PrevMI = &*emitLaneInsert(DstReg: std::nullopt, SrcReg: DstVec, EltReg: OpReg, LaneIdx: i - 1, RB, MIRBuilder&: MIB);
6141 DstVec = PrevMI->getOperand(i: 0).getReg();
6142 }
6143 }
6144
6145 // If DstTy's size in bits is less than 128, then emit a subregister copy
6146 // from DstVec to the last register we've defined.
6147 if (DstSize < 128) {
6148 // Force this to be FPR using the destination vector.
6149 const TargetRegisterClass *RC =
6150 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
6151 if (!RC)
6152 return false;
6153 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
6154 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
6155 return false;
6156 }
6157
6158 unsigned SubReg = 0;
6159 if (!getSubRegForClass(RC, TRI, SubReg))
6160 return false;
6161 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
6162 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize
6163 << "\n");
6164 return false;
6165 }
6166
6167 Register Reg = MRI.createVirtualRegister(RegClass: RC);
6168 Register DstReg = I.getOperand(i: 0).getReg();
6169
6170 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {}).addReg(RegNo: DstVec, Flags: {}, SubReg);
6171 MachineOperand &RegOp = I.getOperand(i: 1);
6172 RegOp.setReg(Reg);
6173 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
6174 } else {
6175 // We either have a vector with all elements (except the first one) undef or
6176 // at least one non-undef non-first element. In the first case, we need to
6177 // constrain the output register ourselves as we may have generated an
6178 // INSERT_SUBREG operation which is a generic operation for which the
6179 // output regclass cannot be automatically chosen.
6180 //
6181 // In the second case, there is no need to do this as it may generate an
6182 // instruction like INSvi32gpr where the regclass can be automatically
6183 // chosen.
6184 //
6185 // Also, we save a copy by re-using the destination register on the final
6186 // insert.
6187 PrevMI->getOperand(i: 0).setReg(I.getOperand(i: 0).getReg());
6188 constrainSelectedInstRegOperands(I&: *PrevMI, TII, TRI, RBI);
6189
6190 Register DstReg = PrevMI->getOperand(i: 0).getReg();
6191 if (PrevMI == ScalarToVec && DstReg.isVirtual()) {
6192 const TargetRegisterClass *RC =
6193 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
6194 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
6195 }
6196 }
6197
6198 I.eraseFromParent();
6199 return true;
6200}
6201
6202bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
6203 unsigned NumVecs,
6204 MachineInstr &I) {
6205 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6206 assert(Opc && "Expected an opcode?");
6207 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6208 auto &MRI = *MIB.getMRI();
6209 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6210 unsigned Size = Ty.getSizeInBits();
6211 assert((Size == 64 || Size == 128) &&
6212 "Destination must be 64 bits or 128 bits?");
6213 unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
6214 auto Ptr = I.getOperand(i: I.getNumOperands() - 1).getReg();
6215 assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?");
6216 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {Ptr});
6217 Load.cloneMemRefs(OtherMI: I);
6218 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6219 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6220 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6221 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: Idx)}, SrcOps: {})
6222 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6223 // Emit the subreg copies and immediately select them.
6224 // FIXME: We should refactor our copy code into an emitCopy helper and
6225 // clean up uses of this pattern elsewhere in the selector.
6226 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6227 }
6228 return true;
6229}
6230
6231bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6232 unsigned Opc, unsigned NumVecs, MachineInstr &I) {
6233 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6234 assert(Opc && "Expected an opcode?");
6235 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6236 auto &MRI = *MIB.getMRI();
6237 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6238 bool Narrow = Ty.getSizeInBits() == 64;
6239
6240 auto FirstSrcRegIt = I.operands_begin() + NumVecs + 1;
6241 SmallVector<Register, 4> Regs(NumVecs);
6242 std::transform(first: FirstSrcRegIt, last: FirstSrcRegIt + NumVecs, result: Regs.begin(),
6243 unary_op: [](auto MO) { return MO.getReg(); });
6244
6245 if (Narrow) {
6246 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6247 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6248 ->getOperand(i: 0)
6249 .getReg();
6250 });
6251 Ty = Ty.multiplyElements(Factor: 2);
6252 }
6253
6254 Register Tuple = createQTuple(Regs, MIB);
6255 auto LaneNo = getIConstantVRegVal(VReg: (FirstSrcRegIt + NumVecs)->getReg(), MRI);
6256 if (!LaneNo)
6257 return false;
6258
6259 Register Ptr = (FirstSrcRegIt + NumVecs + 1)->getReg();
6260 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {})
6261 .addReg(RegNo: Tuple)
6262 .addImm(Val: LaneNo->getZExtValue())
6263 .addReg(RegNo: Ptr);
6264 Load.cloneMemRefs(OtherMI: I);
6265 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6266 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6267 unsigned SubReg = AArch64::qsub0;
6268 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6269 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY,
6270 DstOps: {Narrow ? DstOp(&AArch64::FPR128RegClass)
6271 : DstOp(I.getOperand(i: Idx).getReg())},
6272 SrcOps: {})
6273 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6274 Register WideReg = Vec.getReg(Idx: 0);
6275 // Emit the subreg copies and immediately select them.
6276 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6277 if (Narrow &&
6278 !emitNarrowVector(DstReg: I.getOperand(i: Idx).getReg(), SrcReg: WideReg, MIB, MRI))
6279 return false;
6280 }
6281 return true;
6282}
6283
6284void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &I,
6285 unsigned NumVecs,
6286 unsigned Opc) {
6287 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6288 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6289 Register Ptr = I.getOperand(i: 1 + NumVecs).getReg();
6290
6291 SmallVector<Register, 2> Regs(NumVecs);
6292 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6293 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6294
6295 Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
6296 : createDTuple(Regs, MIB);
6297 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {Tuple, Ptr});
6298 Store.cloneMemRefs(OtherMI: I);
6299 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6300}
6301
6302bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6303 MachineInstr &I, unsigned NumVecs, unsigned Opc) {
6304 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6305 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6306 bool Narrow = Ty.getSizeInBits() == 64;
6307
6308 SmallVector<Register, 2> Regs(NumVecs);
6309 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6310 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6311
6312 if (Narrow)
6313 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6314 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6315 ->getOperand(i: 0)
6316 .getReg();
6317 });
6318
6319 Register Tuple = createQTuple(Regs, MIB);
6320
6321 auto LaneNo = getIConstantVRegVal(VReg: I.getOperand(i: 1 + NumVecs).getReg(), MRI);
6322 if (!LaneNo)
6323 return false;
6324 Register Ptr = I.getOperand(i: 1 + NumVecs + 1).getReg();
6325 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {})
6326 .addReg(RegNo: Tuple)
6327 .addImm(Val: LaneNo->getZExtValue())
6328 .addReg(RegNo: Ptr);
6329 Store.cloneMemRefs(OtherMI: I);
6330 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6331 return true;
6332}
6333
6334bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6335 MachineInstr &I, MachineRegisterInfo &MRI) {
6336 // Find the intrinsic ID.
6337 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6338
6339 const LLT S8 = LLT::scalar(SizeInBits: 8);
6340 const LLT S16 = LLT::scalar(SizeInBits: 16);
6341 const LLT S32 = LLT::scalar(SizeInBits: 32);
6342 const LLT S64 = LLT::scalar(SizeInBits: 64);
6343 const LLT P0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
6344 // Select the instruction.
6345 switch (IntrinID) {
6346 default:
6347 return false;
6348 case Intrinsic::aarch64_ldxp:
6349 case Intrinsic::aarch64_ldaxp: {
6350 auto NewI = MIB.buildInstr(
6351 Opc: IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6352 DstOps: {I.getOperand(i: 0).getReg(), I.getOperand(i: 1).getReg()},
6353 SrcOps: {I.getOperand(i: 3)});
6354 NewI.cloneMemRefs(OtherMI: I);
6355 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
6356 break;
6357 }
6358 case Intrinsic::aarch64_neon_ld1x2: {
6359 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6360 unsigned Opc = 0;
6361 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6362 Opc = AArch64::LD1Twov8b;
6363 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6364 Opc = AArch64::LD1Twov16b;
6365 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6366 Opc = AArch64::LD1Twov4h;
6367 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6368 Opc = AArch64::LD1Twov8h;
6369 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6370 Opc = AArch64::LD1Twov2s;
6371 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6372 Opc = AArch64::LD1Twov4s;
6373 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6374 Opc = AArch64::LD1Twov2d;
6375 else if (Ty == S64 || Ty == P0)
6376 Opc = AArch64::LD1Twov1d;
6377 else
6378 llvm_unreachable("Unexpected type for ld1x2!");
6379 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6380 break;
6381 }
6382 case Intrinsic::aarch64_neon_ld1x3: {
6383 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6384 unsigned Opc = 0;
6385 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6386 Opc = AArch64::LD1Threev8b;
6387 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6388 Opc = AArch64::LD1Threev16b;
6389 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6390 Opc = AArch64::LD1Threev4h;
6391 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6392 Opc = AArch64::LD1Threev8h;
6393 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6394 Opc = AArch64::LD1Threev2s;
6395 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6396 Opc = AArch64::LD1Threev4s;
6397 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6398 Opc = AArch64::LD1Threev2d;
6399 else if (Ty == S64 || Ty == P0)
6400 Opc = AArch64::LD1Threev1d;
6401 else
6402 llvm_unreachable("Unexpected type for ld1x3!");
6403 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6404 break;
6405 }
6406 case Intrinsic::aarch64_neon_ld1x4: {
6407 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6408 unsigned Opc = 0;
6409 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6410 Opc = AArch64::LD1Fourv8b;
6411 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6412 Opc = AArch64::LD1Fourv16b;
6413 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6414 Opc = AArch64::LD1Fourv4h;
6415 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6416 Opc = AArch64::LD1Fourv8h;
6417 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6418 Opc = AArch64::LD1Fourv2s;
6419 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6420 Opc = AArch64::LD1Fourv4s;
6421 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6422 Opc = AArch64::LD1Fourv2d;
6423 else if (Ty == S64 || Ty == P0)
6424 Opc = AArch64::LD1Fourv1d;
6425 else
6426 llvm_unreachable("Unexpected type for ld1x4!");
6427 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6428 break;
6429 }
6430 case Intrinsic::aarch64_neon_ld2: {
6431 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6432 unsigned Opc = 0;
6433 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6434 Opc = AArch64::LD2Twov8b;
6435 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6436 Opc = AArch64::LD2Twov16b;
6437 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6438 Opc = AArch64::LD2Twov4h;
6439 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6440 Opc = AArch64::LD2Twov8h;
6441 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6442 Opc = AArch64::LD2Twov2s;
6443 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6444 Opc = AArch64::LD2Twov4s;
6445 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6446 Opc = AArch64::LD2Twov2d;
6447 else if (Ty == S64 || Ty == P0)
6448 Opc = AArch64::LD1Twov1d;
6449 else
6450 llvm_unreachable("Unexpected type for ld2!");
6451 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6452 break;
6453 }
6454 case Intrinsic::aarch64_neon_ld2lane: {
6455 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6456 unsigned Opc;
6457 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6458 Opc = AArch64::LD2i8;
6459 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6460 Opc = AArch64::LD2i16;
6461 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6462 Opc = AArch64::LD2i32;
6463 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6464 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6465 Opc = AArch64::LD2i64;
6466 else
6467 llvm_unreachable("Unexpected type for st2lane!");
6468 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 2, I))
6469 return false;
6470 break;
6471 }
6472 case Intrinsic::aarch64_neon_ld2r: {
6473 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6474 unsigned Opc = 0;
6475 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6476 Opc = AArch64::LD2Rv8b;
6477 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6478 Opc = AArch64::LD2Rv16b;
6479 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6480 Opc = AArch64::LD2Rv4h;
6481 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6482 Opc = AArch64::LD2Rv8h;
6483 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6484 Opc = AArch64::LD2Rv2s;
6485 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6486 Opc = AArch64::LD2Rv4s;
6487 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6488 Opc = AArch64::LD2Rv2d;
6489 else if (Ty == S64 || Ty == P0)
6490 Opc = AArch64::LD2Rv1d;
6491 else
6492 llvm_unreachable("Unexpected type for ld2r!");
6493 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6494 break;
6495 }
6496 case Intrinsic::aarch64_neon_ld3: {
6497 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6498 unsigned Opc = 0;
6499 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6500 Opc = AArch64::LD3Threev8b;
6501 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6502 Opc = AArch64::LD3Threev16b;
6503 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6504 Opc = AArch64::LD3Threev4h;
6505 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6506 Opc = AArch64::LD3Threev8h;
6507 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6508 Opc = AArch64::LD3Threev2s;
6509 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6510 Opc = AArch64::LD3Threev4s;
6511 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6512 Opc = AArch64::LD3Threev2d;
6513 else if (Ty == S64 || Ty == P0)
6514 Opc = AArch64::LD1Threev1d;
6515 else
6516 llvm_unreachable("Unexpected type for ld3!");
6517 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6518 break;
6519 }
6520 case Intrinsic::aarch64_neon_ld3lane: {
6521 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6522 unsigned Opc;
6523 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6524 Opc = AArch64::LD3i8;
6525 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6526 Opc = AArch64::LD3i16;
6527 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6528 Opc = AArch64::LD3i32;
6529 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6530 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6531 Opc = AArch64::LD3i64;
6532 else
6533 llvm_unreachable("Unexpected type for st3lane!");
6534 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 3, I))
6535 return false;
6536 break;
6537 }
6538 case Intrinsic::aarch64_neon_ld3r: {
6539 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6540 unsigned Opc = 0;
6541 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6542 Opc = AArch64::LD3Rv8b;
6543 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6544 Opc = AArch64::LD3Rv16b;
6545 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6546 Opc = AArch64::LD3Rv4h;
6547 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6548 Opc = AArch64::LD3Rv8h;
6549 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6550 Opc = AArch64::LD3Rv2s;
6551 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6552 Opc = AArch64::LD3Rv4s;
6553 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6554 Opc = AArch64::LD3Rv2d;
6555 else if (Ty == S64 || Ty == P0)
6556 Opc = AArch64::LD3Rv1d;
6557 else
6558 llvm_unreachable("Unexpected type for ld3r!");
6559 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6560 break;
6561 }
6562 case Intrinsic::aarch64_neon_ld4: {
6563 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6564 unsigned Opc = 0;
6565 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6566 Opc = AArch64::LD4Fourv8b;
6567 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6568 Opc = AArch64::LD4Fourv16b;
6569 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6570 Opc = AArch64::LD4Fourv4h;
6571 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6572 Opc = AArch64::LD4Fourv8h;
6573 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6574 Opc = AArch64::LD4Fourv2s;
6575 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6576 Opc = AArch64::LD4Fourv4s;
6577 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6578 Opc = AArch64::LD4Fourv2d;
6579 else if (Ty == S64 || Ty == P0)
6580 Opc = AArch64::LD1Fourv1d;
6581 else
6582 llvm_unreachable("Unexpected type for ld4!");
6583 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6584 break;
6585 }
6586 case Intrinsic::aarch64_neon_ld4lane: {
6587 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6588 unsigned Opc;
6589 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6590 Opc = AArch64::LD4i8;
6591 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6592 Opc = AArch64::LD4i16;
6593 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6594 Opc = AArch64::LD4i32;
6595 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6596 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6597 Opc = AArch64::LD4i64;
6598 else
6599 llvm_unreachable("Unexpected type for st4lane!");
6600 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 4, I))
6601 return false;
6602 break;
6603 }
6604 case Intrinsic::aarch64_neon_ld4r: {
6605 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6606 unsigned Opc = 0;
6607 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6608 Opc = AArch64::LD4Rv8b;
6609 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6610 Opc = AArch64::LD4Rv16b;
6611 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6612 Opc = AArch64::LD4Rv4h;
6613 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6614 Opc = AArch64::LD4Rv8h;
6615 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6616 Opc = AArch64::LD4Rv2s;
6617 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6618 Opc = AArch64::LD4Rv4s;
6619 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6620 Opc = AArch64::LD4Rv2d;
6621 else if (Ty == S64 || Ty == P0)
6622 Opc = AArch64::LD4Rv1d;
6623 else
6624 llvm_unreachable("Unexpected type for ld4r!");
6625 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6626 break;
6627 }
6628 case Intrinsic::aarch64_neon_st1x2: {
6629 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6630 unsigned Opc;
6631 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6632 Opc = AArch64::ST1Twov8b;
6633 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6634 Opc = AArch64::ST1Twov16b;
6635 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6636 Opc = AArch64::ST1Twov4h;
6637 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6638 Opc = AArch64::ST1Twov8h;
6639 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6640 Opc = AArch64::ST1Twov2s;
6641 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6642 Opc = AArch64::ST1Twov4s;
6643 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6644 Opc = AArch64::ST1Twov2d;
6645 else if (Ty == S64 || Ty == P0)
6646 Opc = AArch64::ST1Twov1d;
6647 else
6648 llvm_unreachable("Unexpected type for st1x2!");
6649 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6650 break;
6651 }
6652 case Intrinsic::aarch64_neon_st1x3: {
6653 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6654 unsigned Opc;
6655 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6656 Opc = AArch64::ST1Threev8b;
6657 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6658 Opc = AArch64::ST1Threev16b;
6659 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6660 Opc = AArch64::ST1Threev4h;
6661 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6662 Opc = AArch64::ST1Threev8h;
6663 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6664 Opc = AArch64::ST1Threev2s;
6665 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6666 Opc = AArch64::ST1Threev4s;
6667 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6668 Opc = AArch64::ST1Threev2d;
6669 else if (Ty == S64 || Ty == P0)
6670 Opc = AArch64::ST1Threev1d;
6671 else
6672 llvm_unreachable("Unexpected type for st1x3!");
6673 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6674 break;
6675 }
6676 case Intrinsic::aarch64_neon_st1x4: {
6677 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6678 unsigned Opc;
6679 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6680 Opc = AArch64::ST1Fourv8b;
6681 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6682 Opc = AArch64::ST1Fourv16b;
6683 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6684 Opc = AArch64::ST1Fourv4h;
6685 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6686 Opc = AArch64::ST1Fourv8h;
6687 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6688 Opc = AArch64::ST1Fourv2s;
6689 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6690 Opc = AArch64::ST1Fourv4s;
6691 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6692 Opc = AArch64::ST1Fourv2d;
6693 else if (Ty == S64 || Ty == P0)
6694 Opc = AArch64::ST1Fourv1d;
6695 else
6696 llvm_unreachable("Unexpected type for st1x4!");
6697 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6698 break;
6699 }
6700 case Intrinsic::aarch64_neon_st2: {
6701 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6702 unsigned Opc;
6703 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6704 Opc = AArch64::ST2Twov8b;
6705 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6706 Opc = AArch64::ST2Twov16b;
6707 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6708 Opc = AArch64::ST2Twov4h;
6709 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6710 Opc = AArch64::ST2Twov8h;
6711 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6712 Opc = AArch64::ST2Twov2s;
6713 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6714 Opc = AArch64::ST2Twov4s;
6715 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6716 Opc = AArch64::ST2Twov2d;
6717 else if (Ty == S64 || Ty == P0)
6718 Opc = AArch64::ST1Twov1d;
6719 else
6720 llvm_unreachable("Unexpected type for st2!");
6721 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6722 break;
6723 }
6724 case Intrinsic::aarch64_neon_st3: {
6725 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6726 unsigned Opc;
6727 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6728 Opc = AArch64::ST3Threev8b;
6729 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6730 Opc = AArch64::ST3Threev16b;
6731 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6732 Opc = AArch64::ST3Threev4h;
6733 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6734 Opc = AArch64::ST3Threev8h;
6735 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6736 Opc = AArch64::ST3Threev2s;
6737 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6738 Opc = AArch64::ST3Threev4s;
6739 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6740 Opc = AArch64::ST3Threev2d;
6741 else if (Ty == S64 || Ty == P0)
6742 Opc = AArch64::ST1Threev1d;
6743 else
6744 llvm_unreachable("Unexpected type for st3!");
6745 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6746 break;
6747 }
6748 case Intrinsic::aarch64_neon_st4: {
6749 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6750 unsigned Opc;
6751 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6752 Opc = AArch64::ST4Fourv8b;
6753 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6754 Opc = AArch64::ST4Fourv16b;
6755 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6756 Opc = AArch64::ST4Fourv4h;
6757 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6758 Opc = AArch64::ST4Fourv8h;
6759 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6760 Opc = AArch64::ST4Fourv2s;
6761 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6762 Opc = AArch64::ST4Fourv4s;
6763 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6764 Opc = AArch64::ST4Fourv2d;
6765 else if (Ty == S64 || Ty == P0)
6766 Opc = AArch64::ST1Fourv1d;
6767 else
6768 llvm_unreachable("Unexpected type for st4!");
6769 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6770 break;
6771 }
6772 case Intrinsic::aarch64_neon_st2lane: {
6773 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6774 unsigned Opc;
6775 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6776 Opc = AArch64::ST2i8;
6777 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6778 Opc = AArch64::ST2i16;
6779 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6780 Opc = AArch64::ST2i32;
6781 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6782 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6783 Opc = AArch64::ST2i64;
6784 else
6785 llvm_unreachable("Unexpected type for st2lane!");
6786 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 2, Opc))
6787 return false;
6788 break;
6789 }
6790 case Intrinsic::aarch64_neon_st3lane: {
6791 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6792 unsigned Opc;
6793 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6794 Opc = AArch64::ST3i8;
6795 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6796 Opc = AArch64::ST3i16;
6797 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6798 Opc = AArch64::ST3i32;
6799 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6800 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6801 Opc = AArch64::ST3i64;
6802 else
6803 llvm_unreachable("Unexpected type for st3lane!");
6804 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 3, Opc))
6805 return false;
6806 break;
6807 }
6808 case Intrinsic::aarch64_neon_st4lane: {
6809 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6810 unsigned Opc;
6811 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6812 Opc = AArch64::ST4i8;
6813 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6814 Opc = AArch64::ST4i16;
6815 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6816 Opc = AArch64::ST4i32;
6817 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6818 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6819 Opc = AArch64::ST4i64;
6820 else
6821 llvm_unreachable("Unexpected type for st4lane!");
6822 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 4, Opc))
6823 return false;
6824 break;
6825 }
6826 case Intrinsic::aarch64_mops_memset_tag: {
6827 // Transform
6828 // %dst:gpr(p0) = \
6829 // G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
6830 // \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
6831 // where %dst is updated, into
6832 // %Rd:GPR64common, %Rn:GPR64) = \
6833 // MOPSMemorySetTaggingPseudo \
6834 // %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
6835 // where Rd and Rn are tied.
6836 // It is expected that %val has been extended to s64 in legalization.
6837 // Note that the order of the size/value operands are swapped.
6838
6839 Register DstDef = I.getOperand(i: 0).getReg();
6840 // I.getOperand(1) is the intrinsic function
6841 Register DstUse = I.getOperand(i: 2).getReg();
6842 Register ValUse = I.getOperand(i: 3).getReg();
6843 Register SizeUse = I.getOperand(i: 4).getReg();
6844
6845 // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
6846 // Therefore an additional virtual register is required for the updated size
6847 // operand. This value is not accessible via the semantics of the intrinsic.
6848 Register SizeDef = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
6849
6850 auto Memset = MIB.buildInstr(Opc: AArch64::MOPSMemorySetTaggingPseudo,
6851 DstOps: {DstDef, SizeDef}, SrcOps: {DstUse, SizeUse, ValUse});
6852 Memset.cloneMemRefs(OtherMI: I);
6853 constrainSelectedInstRegOperands(I&: *Memset, TII, TRI, RBI);
6854 break;
6855 }
6856 case Intrinsic::ptrauth_resign_load_relative: {
6857 Register DstReg = I.getOperand(i: 0).getReg();
6858 Register ValReg = I.getOperand(i: 2).getReg();
6859 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6860 Register AUTDisc = I.getOperand(i: 4).getReg();
6861 uint64_t PACKey = I.getOperand(i: 5).getImm();
6862 Register PACDisc = I.getOperand(i: 6).getReg();
6863 int64_t Addend = I.getOperand(i: 7).getImm();
6864
6865 Register AUTAddrDisc = AUTDisc;
6866 uint16_t AUTConstDiscC = 0;
6867 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6868 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6869
6870 Register PACAddrDisc = PACDisc;
6871 uint16_t PACConstDiscC = 0;
6872 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6873 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6874
6875 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6876
6877 MIB.buildInstr(Opcode: AArch64::AUTRELLOADPAC)
6878 .addImm(Val: AUTKey)
6879 .addImm(Val: AUTConstDiscC)
6880 .addUse(RegNo: AUTAddrDisc)
6881 .addImm(Val: PACKey)
6882 .addImm(Val: PACConstDiscC)
6883 .addUse(RegNo: PACAddrDisc)
6884 .addImm(Val: Addend)
6885 .constrainAllUses(TII, TRI, RBI);
6886 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6887
6888 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6889 I.eraseFromParent();
6890 return true;
6891 }
6892 }
6893
6894 I.eraseFromParent();
6895 return true;
6896}
6897
6898bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
6899 MachineRegisterInfo &MRI) {
6900 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6901
6902 switch (IntrinID) {
6903 default:
6904 break;
6905 case Intrinsic::ptrauth_resign: {
6906 Register DstReg = I.getOperand(i: 0).getReg();
6907 Register ValReg = I.getOperand(i: 2).getReg();
6908 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6909 Register AUTDisc = I.getOperand(i: 4).getReg();
6910 uint64_t PACKey = I.getOperand(i: 5).getImm();
6911 Register PACDisc = I.getOperand(i: 6).getReg();
6912
6913 Register AUTAddrDisc = AUTDisc;
6914 uint16_t AUTConstDiscC = 0;
6915 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6916 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6917
6918 Register PACAddrDisc = PACDisc;
6919 uint16_t PACConstDiscC = 0;
6920 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6921 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6922
6923 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6924 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
6925 MIB.buildInstr(Opcode: AArch64::AUTPAC)
6926 .addImm(Val: AUTKey)
6927 .addImm(Val: AUTConstDiscC)
6928 .addUse(RegNo: AUTAddrDisc)
6929 .addImm(Val: PACKey)
6930 .addImm(Val: PACConstDiscC)
6931 .addUse(RegNo: PACAddrDisc)
6932 .constrainAllUses(TII, TRI, RBI);
6933 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6934
6935 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6936 I.eraseFromParent();
6937 return true;
6938 }
6939 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6940 Register DstReg = I.getOperand(i: 0).getReg();
6941 Register ValReg = I.getOperand(i: 2).getReg();
6942 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6943 Register AUTDisc = I.getOperand(i: 4).getReg();
6944 Register AUTPC = I.getOperand(i: 5).getReg();
6945 uint64_t PACKey = I.getOperand(i: 6).getImm();
6946 Register PACDisc = I.getOperand(i: 7).getReg();
6947
6948 assert((AUTKey == AArch64PACKey::IA || AUTKey == AArch64PACKey::IB) &&
6949 "auth_with_pc_and_resign only supports IA and IB keys");
6950
6951 uint16_t PACConstDiscC = 0;
6952 Register PACAddrDisc;
6953 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6954 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6955
6956 if (!PACAddrDisc.isValid())
6957 PACAddrDisc = AArch64::XZR;
6958
6959 MIB.buildCopy(Res: {AArch64::X17}, Op: {ValReg});
6960 MIB.buildCopy(Res: {AArch64::X16}, Op: {AUTDisc});
6961 MIB.buildCopy(Res: {AArch64::X15}, Op: {AUTPC});
6962
6963 MIB.buildInstr(Opcode: AArch64::AUTPCPAC)
6964 .addImm(Val: AUTKey)
6965 .addImm(Val: PACKey)
6966 .addImm(Val: PACConstDiscC)
6967 .addUse(RegNo: PACAddrDisc)
6968 .constrainAllUses(TII, TRI, RBI);
6969
6970 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X17));
6971 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6972 I.eraseFromParent();
6973 return true;
6974 }
6975 case Intrinsic::ptrauth_auth: {
6976 Register DstReg = I.getOperand(i: 0).getReg();
6977 Register ValReg = I.getOperand(i: 2).getReg();
6978 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6979 Register AUTDisc = I.getOperand(i: 4).getReg();
6980
6981 Register AUTAddrDisc = AUTDisc;
6982 uint16_t AUTConstDiscC = 0;
6983 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6984 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6985
6986 if (STI.isX16X17Safer()) {
6987 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6988 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
6989 MIB.buildInstr(Opcode: AArch64::AUTx16x17)
6990 .addImm(Val: AUTKey)
6991 .addImm(Val: AUTConstDiscC)
6992 .addUse(RegNo: AUTAddrDisc)
6993 .constrainAllUses(TII, TRI, RBI);
6994 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6995 } else {
6996 Register ScratchReg =
6997 MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
6998 MIB.buildInstr(Opcode: AArch64::AUTxMxN)
6999 .addDef(RegNo: DstReg)
7000 .addDef(RegNo: ScratchReg)
7001 .addUse(RegNo: ValReg)
7002 .addImm(Val: AUTKey)
7003 .addImm(Val: AUTConstDiscC)
7004 .addUse(RegNo: AUTAddrDisc)
7005 .constrainAllUses(TII, TRI, RBI);
7006 }
7007
7008 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
7009 I.eraseFromParent();
7010 return true;
7011 }
7012 case Intrinsic::frameaddress:
7013 case Intrinsic::returnaddress: {
7014 MachineFunction &MF = *I.getParent()->getParent();
7015 MachineFrameInfo &MFI = MF.getFrameInfo();
7016
7017 unsigned Depth = I.getOperand(i: 2).getImm();
7018 Register DstReg = I.getOperand(i: 0).getReg();
7019 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
7020
7021 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
7022 if (!MFReturnAddr) {
7023 // Insert the copy from LR/X30 into the entry block, before it can be
7024 // clobbered by anything.
7025 MFI.setReturnAddressIsTaken(true);
7026 MFReturnAddr = getFunctionLiveInPhysReg(
7027 MF, TII, PhysReg: AArch64::LR, RC: AArch64::GPR64RegClass, DL: I.getDebugLoc());
7028 }
7029
7030 if (STI.hasPAuth()) {
7031 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {MFReturnAddr});
7032 } else {
7033 MIB.buildCopy(Res: {Register(AArch64::LR)}, Op: {MFReturnAddr});
7034 MIB.buildInstr(Opcode: AArch64::XPACLRI);
7035 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
7036 }
7037
7038 I.eraseFromParent();
7039 return true;
7040 }
7041
7042 MFI.setFrameAddressIsTaken(true);
7043 Register FrameAddr(AArch64::FP);
7044 while (Depth--) {
7045 Register NextFrame = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
7046 auto Ldr =
7047 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {NextFrame}, SrcOps: {FrameAddr}).addImm(Val: 0);
7048 constrainSelectedInstRegOperands(I&: *Ldr, TII, TRI, RBI);
7049 FrameAddr = NextFrame;
7050 }
7051
7052 if (IntrinID == Intrinsic::frameaddress)
7053 MIB.buildCopy(Res: {DstReg}, Op: {FrameAddr});
7054 else {
7055 MFI.setReturnAddressIsTaken(true);
7056
7057 if (STI.hasPAuth()) {
7058 Register TmpReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
7059 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {TmpReg}, SrcOps: {FrameAddr}).addImm(Val: 1);
7060 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {TmpReg});
7061 } else {
7062 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {Register(AArch64::LR)}, SrcOps: {FrameAddr})
7063 .addImm(Val: 1);
7064 MIB.buildInstr(Opcode: AArch64::XPACLRI);
7065 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
7066 }
7067 }
7068
7069 I.eraseFromParent();
7070 return true;
7071 }
7072 case Intrinsic::aarch64_neon_tbl2:
7073 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBLv8i8Two, Opc2: AArch64::TBLv16i8Two, isExt: false);
7074 return true;
7075 case Intrinsic::aarch64_neon_tbl3:
7076 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBLv8i8Three, Opc2: AArch64::TBLv16i8Three,
7077 isExt: false);
7078 return true;
7079 case Intrinsic::aarch64_neon_tbl4:
7080 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBLv8i8Four, Opc2: AArch64::TBLv16i8Four, isExt: false);
7081 return true;
7082 case Intrinsic::aarch64_neon_tbx2:
7083 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBXv8i8Two, Opc2: AArch64::TBXv16i8Two, isExt: true);
7084 return true;
7085 case Intrinsic::aarch64_neon_tbx3:
7086 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBXv8i8Three, Opc2: AArch64::TBXv16i8Three, isExt: true);
7087 return true;
7088 case Intrinsic::aarch64_neon_tbx4:
7089 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBXv8i8Four, Opc2: AArch64::TBXv16i8Four, isExt: true);
7090 return true;
7091 case Intrinsic::swift_async_context_addr:
7092 auto Sub = MIB.buildInstr(Opc: AArch64::SUBXri, DstOps: {I.getOperand(i: 0).getReg()},
7093 SrcOps: {Register(AArch64::FP)})
7094 .addImm(Val: 8)
7095 .addImm(Val: 0);
7096 constrainSelectedInstRegOperands(I&: *Sub, TII, TRI, RBI);
7097
7098 MF->getFrameInfo().setFrameAddressIsTaken(true);
7099 MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
7100 I.eraseFromParent();
7101 return true;
7102 }
7103 return false;
7104}
7105
7106// G_PTRAUTH_GLOBAL_VALUE lowering
7107//
7108// We have 3 lowering alternatives to choose from:
7109// - MOVaddrPAC: similar to MOVaddr, with added PAC.
7110// If the GV doesn't need a GOT load (i.e., is locally defined)
7111// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
7112//
7113// - LOADgotPAC: similar to LOADgot, with added PAC.
7114// If the GV needs a GOT load, materialize the pointer using the usual
7115// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
7116// section is assumed to be read-only (for example, via relro mechanism). See
7117// LowerMOVaddrPAC.
7118//
7119// - LOADauthptrstatic: similar to LOADgot, but use a
7120// special stub slot instead of a GOT slot.
7121// Load a signed pointer for symbol 'sym' from a stub slot named
7122// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
7123// resolving. This usually lowers to adrp+ldr, but also emits an entry into
7124// .data with an
7125// @AUTH relocation. See LowerLOADauthptrstatic.
7126//
7127// All 3 are pseudos that are expand late to longer sequences: this lets us
7128// provide integrity guarantees on the to-be-signed intermediate values.
7129//
7130// LOADauthptrstatic is undesirable because it requires a large section filled
7131// with often similarly-signed pointers, making it a good harvesting target.
7132// Thus, it's only used for ptrauth references to extern_weak to avoid null
7133// checks.
7134
7135bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
7136 MachineInstr &I, MachineRegisterInfo &MRI) const {
7137 Register DefReg = I.getOperand(i: 0).getReg();
7138 Register Addr = I.getOperand(i: 1).getReg();
7139 uint64_t Key = I.getOperand(i: 2).getImm();
7140 Register AddrDisc = I.getOperand(i: 3).getReg();
7141 uint64_t Disc = I.getOperand(i: 4).getImm();
7142 int64_t Offset = 0;
7143
7144 if (Key > AArch64PACKey::LAST)
7145 report_fatal_error(reason: "key in ptrauth global out of range [0, " +
7146 Twine((int)AArch64PACKey::LAST) + "]");
7147
7148 // Blend only works if the integer discriminator is 16-bit wide.
7149 if (!isUInt<16>(x: Disc))
7150 report_fatal_error(
7151 reason: "constant discriminator in ptrauth global out of range [0, 0xffff]");
7152
7153 // Choosing between 3 lowering alternatives is target-specific.
7154 if (!STI.isTargetELF() && !STI.isTargetMachO())
7155 report_fatal_error(reason: "ptrauth global lowering only supported on MachO/ELF");
7156
7157 if (!MRI.hasOneDef(RegNo: Addr))
7158 return false;
7159
7160 // First match any offset we take from the real global.
7161 const MachineInstr *DefMI = &*MRI.def_instr_begin(RegNo: Addr);
7162 if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
7163 Register OffsetReg = DefMI->getOperand(i: 2).getReg();
7164 if (!MRI.hasOneDef(RegNo: OffsetReg))
7165 return false;
7166 const MachineInstr &OffsetMI = *MRI.def_instr_begin(RegNo: OffsetReg);
7167 if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
7168 return false;
7169
7170 Addr = DefMI->getOperand(i: 1).getReg();
7171 if (!MRI.hasOneDef(RegNo: Addr))
7172 return false;
7173
7174 DefMI = &*MRI.def_instr_begin(RegNo: Addr);
7175 Offset = OffsetMI.getOperand(i: 1).getCImm()->getSExtValue();
7176 }
7177
7178 // We should be left with a genuine unauthenticated GlobalValue.
7179 const GlobalValue *GV;
7180 if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
7181 GV = DefMI->getOperand(i: 1).getGlobal();
7182 Offset += DefMI->getOperand(i: 1).getOffset();
7183 } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
7184 GV = DefMI->getOperand(i: 2).getGlobal();
7185 Offset += DefMI->getOperand(i: 2).getOffset();
7186 } else {
7187 return false;
7188 }
7189
7190 MachineIRBuilder MIB(I);
7191
7192 // Classify the reference to determine whether it needs a GOT load.
7193 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
7194 const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
7195 assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
7196 "unsupported non-GOT op flags on ptrauth global reference");
7197 assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
7198 "unsupported non-GOT reference to weak ptrauth global");
7199
7200 std::optional<APInt> AddrDiscVal = getIConstantVRegVal(VReg: AddrDisc, MRI);
7201 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
7202
7203 // Non-extern_weak:
7204 // - No GOT load needed -> MOVaddrPAC
7205 // - GOT load for non-extern_weak -> LOADgotPAC
7206 // Note that we disallow extern_weak refs to avoid null checks later.
7207 if (!GV->hasExternalWeakLinkage()) {
7208 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X16}, SrcOps: {});
7209 MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {AArch64::X17}, SrcOps: {});
7210 MIB.buildInstr(Opcode: NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
7211 .addGlobalAddress(GV, Offset)
7212 .addImm(Val: Key)
7213 .addReg(RegNo: HasAddrDisc ? AddrDisc : AArch64::XZR)
7214 .addImm(Val: Disc)
7215 .constrainAllUses(TII, TRI, RBI);
7216 MIB.buildCopy(Res: DefReg, Op: Register(AArch64::X16));
7217 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7218 I.eraseFromParent();
7219 return true;
7220 }
7221
7222 // extern_weak -> LOADauthptrstatic
7223
7224 // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
7225 // offset alone as a pointer if the symbol wasn't available, which would
7226 // probably break null checks in users. Ptrauth complicates things further:
7227 // error out.
7228 if (Offset != 0)
7229 report_fatal_error(
7230 reason: "unsupported non-zero offset in weak ptrauth global reference");
7231
7232 if (HasAddrDisc)
7233 report_fatal_error(reason: "unsupported weak addr-div ptrauth global");
7234
7235 MIB.buildInstr(Opc: AArch64::LOADauthptrstatic, DstOps: {DefReg}, SrcOps: {})
7236 .addGlobalAddress(GV, Offset)
7237 .addImm(Val: Key)
7238 .addImm(Val: Disc);
7239 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7240
7241 I.eraseFromParent();
7242 return true;
7243}
7244
7245void AArch64InstructionSelector::SelectTable(MachineInstr &I,
7246 MachineRegisterInfo &MRI,
7247 unsigned NumVec, unsigned Opc1,
7248 unsigned Opc2, bool isExt) {
7249 Register DstReg = I.getOperand(i: 0).getReg();
7250 unsigned Opc = MRI.getType(Reg: DstReg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) ? Opc1 : Opc2;
7251
7252 // Create the REG_SEQUENCE
7253 SmallVector<Register, 4> Regs;
7254 for (unsigned i = 0; i < NumVec; i++)
7255 Regs.push_back(Elt: I.getOperand(i: i + 2 + isExt).getReg());
7256 Register RegSeq = createQTuple(Regs, MIB);
7257
7258 Register IdxReg = I.getOperand(i: 2 + NumVec + isExt).getReg();
7259 MachineInstrBuilder Instr;
7260 if (isExt) {
7261 Register Reg = I.getOperand(i: 2).getReg();
7262 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Reg, RegSeq, IdxReg});
7263 } else
7264 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {RegSeq, IdxReg});
7265 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
7266 I.eraseFromParent();
7267}
7268
7269InstructionSelector::ComplexRendererFns
7270AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
7271 auto MaybeImmed = getImmedFromMO(Root);
7272 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7273 return std::nullopt;
7274 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7275 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7276}
7277
7278InstructionSelector::ComplexRendererFns
7279AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
7280 auto MaybeImmed = getImmedFromMO(Root);
7281 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7282 return std::nullopt;
7283 uint64_t Enc = 31 - *MaybeImmed;
7284 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7285}
7286
7287InstructionSelector::ComplexRendererFns
7288AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
7289 auto MaybeImmed = getImmedFromMO(Root);
7290 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7291 return std::nullopt;
7292 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7293 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7294}
7295
7296InstructionSelector::ComplexRendererFns
7297AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
7298 auto MaybeImmed = getImmedFromMO(Root);
7299 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7300 return std::nullopt;
7301 uint64_t Enc = 63 - *MaybeImmed;
7302 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7303}
7304
7305/// Helper to select an immediate value that can be represented as a 12-bit
7306/// value shifted left by either 0 or 12. If it is possible to do so, return
7307/// the immediate and shift value. If not, return std::nullopt.
7308///
7309/// Used by selectArithImmed and selectNegArithImmed.
7310InstructionSelector::ComplexRendererFns
7311AArch64InstructionSelector::select12BitValueWithLeftShift(
7312 uint64_t Immed) const {
7313 unsigned ShiftAmt;
7314 if (Immed >> 12 == 0) {
7315 ShiftAmt = 0;
7316 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7317 ShiftAmt = 12;
7318 Immed = Immed >> 12;
7319 } else
7320 return std::nullopt;
7321
7322 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
7323 return {{
7324 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Immed); },
7325 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShVal); },
7326 }};
7327}
7328
7329/// SelectArithImmed - Select an immediate value that can be represented as
7330/// a 12-bit value shifted left by either 0 or 12. If so, return true with
7331/// Val set to the 12-bit value and Shift set to the shifter operand.
7332InstructionSelector::ComplexRendererFns
7333AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
7334 // This function is called from the addsub_shifted_imm ComplexPattern,
7335 // which lists [imm] as the list of opcode it's interested in, however
7336 // we still need to check whether the operand is actually an immediate
7337 // here because the ComplexPattern opcode list is only used in
7338 // root-level opcode matching.
7339 auto MaybeImmed = getImmedFromMO(Root);
7340 if (MaybeImmed == std::nullopt)
7341 return std::nullopt;
7342 return select12BitValueWithLeftShift(Immed: *MaybeImmed);
7343}
7344
7345/// SelectNegArithImmed - As above, but negates the value before trying to
7346/// select it.
7347InstructionSelector::ComplexRendererFns
7348AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
7349 // We need a register here, because we need to know if we have a 64 or 32
7350 // bit immediate.
7351 if (!Root.isReg())
7352 return std::nullopt;
7353 auto MaybeImmed = getImmedFromMO(Root);
7354 if (MaybeImmed == std::nullopt)
7355 return std::nullopt;
7356 uint64_t Immed = *MaybeImmed;
7357
7358 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
7359 // have the opposite effect on the C flag, so this pattern mustn't match under
7360 // those circumstances.
7361 if (Immed == 0)
7362 return std::nullopt;
7363
7364 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
7365 // the root.
7366 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7367 if (MRI.getType(Reg: Root.getReg()).getSizeInBits() == 32)
7368 Immed = ~((uint32_t)Immed) + 1;
7369 else
7370 Immed = ~Immed + 1ULL;
7371
7372 if (Immed & 0xFFFFFFFFFF000000ULL)
7373 return std::nullopt;
7374
7375 Immed &= 0xFFFFFFULL;
7376 return select12BitValueWithLeftShift(Immed);
7377}
7378
7379/// Checks if we are sure that folding MI into load/store addressing mode is
7380/// beneficial or not.
7381///
7382/// Returns:
7383/// - true if folding MI would be beneficial.
7384/// - false if folding MI would be bad.
7385/// - std::nullopt if it is not sure whether folding MI is beneficial.
7386///
7387/// \p MI can be the offset operand of G_PTR_ADD, e.g. G_SHL in the example:
7388///
7389/// %13:gpr(s64) = G_CONSTANT i64 1
7390/// %8:gpr(s64) = G_SHL %6, %13(s64)
7391/// %9:gpr(p0) = G_PTR_ADD %0, %8(s64)
7392/// %12:gpr(s32) = G_LOAD %9(p0) :: (load (s16))
7393std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7394 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
7395 if (MI.getOpcode() == AArch64::G_SHL) {
7396 // Address operands with shifts are free, except for running on subtargets
7397 // with AddrLSLSlow14.
7398 if (const auto ValAndVeg = getIConstantVRegValWithLookThrough(
7399 VReg: MI.getOperand(i: 2).getReg(), MRI)) {
7400 const APInt ShiftVal = ValAndVeg->Value;
7401
7402 // Don't fold if we know this will be slow.
7403 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7404 }
7405 }
7406 return std::nullopt;
7407}
7408
7409/// Return true if it is worth folding MI into an extended register. That is,
7410/// if it's safe to pull it into the addressing mode of a load or store as a
7411/// shift.
7412/// \p IsAddrOperand whether the def of MI is used as an address operand
7413/// (e.g. feeding into an LDR/STR).
7414bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7415 const MachineInstr &MI, const MachineRegisterInfo &MRI,
7416 bool IsAddrOperand) const {
7417
7418 // Always fold if there is one use, or if we're optimizing for size.
7419 Register DefReg = MI.getOperand(i: 0).getReg();
7420 if (MRI.hasOneNonDBGUse(RegNo: DefReg) ||
7421 MI.getParent()->getParent()->getFunction().hasOptSize())
7422 return true;
7423
7424 if (IsAddrOperand) {
7425 // If we are already sure that folding MI is good or bad, return the result.
7426 if (const auto Worth = isWorthFoldingIntoAddrMode(MI, MRI))
7427 return *Worth;
7428
7429 // Fold G_PTR_ADD if its offset operand can be folded
7430 if (MI.getOpcode() == AArch64::G_PTR_ADD) {
7431 MachineInstr *OffsetInst =
7432 getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
7433
7434 // Note, we already know G_PTR_ADD is used by at least two instructions.
7435 // If we are also sure about whether folding is beneficial or not,
7436 // return the result.
7437 if (const auto Worth = isWorthFoldingIntoAddrMode(MI: *OffsetInst, MRI))
7438 return *Worth;
7439 }
7440 }
7441
7442 // FIXME: Consider checking HasALULSLFast as appropriate.
7443
7444 // We have a fastpath, so folding a shift in and potentially computing it
7445 // many times may be beneficial. Check if this is only used in memory ops.
7446 // If it is, then we should fold.
7447 return all_of(Range: MRI.use_nodbg_instructions(Reg: DefReg),
7448 P: [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7449}
7450
7451InstructionSelector::ComplexRendererFns
7452AArch64InstructionSelector::selectExtendedSHL(
7453 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
7454 unsigned SizeInBytes, bool WantsExt) const {
7455 assert(Base.isReg() && "Expected base to be a register operand");
7456 assert(Offset.isReg() && "Expected offset to be a register operand");
7457
7458 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7459 MachineInstr *OffsetInst = MRI.getVRegDef(Reg: Offset.getReg());
7460
7461 unsigned OffsetOpc = OffsetInst->getOpcode();
7462 bool LookedThroughZExt = false;
7463 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7464 // Try to look through a ZEXT.
7465 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7466 return std::nullopt;
7467
7468 OffsetInst = MRI.getVRegDef(Reg: OffsetInst->getOperand(i: 1).getReg());
7469 OffsetOpc = OffsetInst->getOpcode();
7470 LookedThroughZExt = true;
7471
7472 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7473 return std::nullopt;
7474 }
7475 // Make sure that the memory op is a valid size.
7476 int64_t LegalShiftVal = Log2_32(Value: SizeInBytes);
7477 if (LegalShiftVal == 0)
7478 return std::nullopt;
7479 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7480 return std::nullopt;
7481
7482 // Now, try to find the specific G_CONSTANT. Start by assuming that the
7483 // register we will offset is the LHS, and the register containing the
7484 // constant is the RHS.
7485 Register OffsetReg = OffsetInst->getOperand(i: 1).getReg();
7486 Register ConstantReg = OffsetInst->getOperand(i: 2).getReg();
7487 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7488 if (!ValAndVReg) {
7489 // We didn't get a constant on the RHS. If the opcode is a shift, then
7490 // we're done.
7491 if (OffsetOpc == TargetOpcode::G_SHL)
7492 return std::nullopt;
7493
7494 // If we have a G_MUL, we can use either register. Try looking at the RHS.
7495 std::swap(a&: OffsetReg, b&: ConstantReg);
7496 ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7497 if (!ValAndVReg)
7498 return std::nullopt;
7499 }
7500
7501 // The value must fit into 3 bits, and must be positive. Make sure that is
7502 // true.
7503 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7504
7505 // Since we're going to pull this into a shift, the constant value must be
7506 // a power of 2. If we got a multiply, then we need to check this.
7507 if (OffsetOpc == TargetOpcode::G_MUL) {
7508 if (!llvm::has_single_bit<uint32_t>(Value: ImmVal))
7509 return std::nullopt;
7510
7511 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
7512 ImmVal = Log2_32(Value: ImmVal);
7513 }
7514
7515 if ((ImmVal & 0x7) != ImmVal)
7516 return std::nullopt;
7517
7518 // We are only allowed to shift by LegalShiftVal. This shift value is built
7519 // into the instruction, so we can't just use whatever we want.
7520 if (ImmVal != LegalShiftVal)
7521 return std::nullopt;
7522
7523 unsigned SignExtend = 0;
7524 if (WantsExt) {
7525 // Check if the offset is defined by an extend, unless we looked through a
7526 // G_ZEXT earlier.
7527 if (!LookedThroughZExt) {
7528 MachineInstr *ExtInst = getDefIgnoringCopies(Reg: OffsetReg, MRI);
7529 auto Ext = getExtendTypeForInst(MI&: *ExtInst, MRI, IsLoadStore: true);
7530 if (Ext == AArch64_AM::InvalidShiftExtend)
7531 return std::nullopt;
7532
7533 SignExtend = AArch64_AM::isSignExtendShiftType(Type: Ext) ? 1 : 0;
7534 // We only support SXTW for signed extension here.
7535 if (SignExtend && Ext != AArch64_AM::SXTW)
7536 return std::nullopt;
7537 OffsetReg = ExtInst->getOperand(i: 1).getReg();
7538 }
7539
7540 // Need a 32-bit wide register here.
7541 MachineIRBuilder MIB(*MRI.getVRegDef(Reg: Root.getReg()));
7542 OffsetReg = moveScalarRegClass(Reg: OffsetReg, RC: AArch64::GPR32RegClass, MIB);
7543 }
7544
7545 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
7546 // offset. Signify that we are shifting by setting the shift flag to 1.
7547 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base.getReg()); },
7548 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: OffsetReg); },
7549 [=](MachineInstrBuilder &MIB) {
7550 // Need to add both immediates here to make sure that they are both
7551 // added to the instruction.
7552 MIB.addImm(Val: SignExtend);
7553 MIB.addImm(Val: 1);
7554 }}};
7555}
7556
7557/// This is used for computing addresses like this:
7558///
7559/// ldr x1, [x2, x3, lsl #3]
7560///
7561/// Where x2 is the base register, and x3 is an offset register. The shift-left
7562/// is a constant value specific to this load instruction. That is, we'll never
7563/// see anything other than a 3 here (which corresponds to the size of the
7564/// element being loaded.)
7565InstructionSelector::ComplexRendererFns
7566AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7567 MachineOperand &Root, unsigned SizeInBytes) const {
7568 if (!Root.isReg())
7569 return std::nullopt;
7570 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7571
7572 // We want to find something like this:
7573 //
7574 // val = G_CONSTANT LegalShiftVal
7575 // shift = G_SHL off_reg val
7576 // ptr = G_PTR_ADD base_reg shift
7577 // x = G_LOAD ptr
7578 //
7579 // And fold it into this addressing mode:
7580 //
7581 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
7582
7583 // Check if we can find the G_PTR_ADD.
7584 MachineInstr *PtrAdd =
7585 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7586 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7587 return std::nullopt;
7588
7589 // Now, try to match an opcode which will match our specific offset.
7590 // We want a G_SHL or a G_MUL.
7591 MachineInstr *OffsetInst =
7592 getDefIgnoringCopies(Reg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7593 return selectExtendedSHL(Root, Base&: PtrAdd->getOperand(i: 1),
7594 Offset&: OffsetInst->getOperand(i: 0), SizeInBytes,
7595 /*WantsExt=*/false);
7596}
7597
7598/// This is used for computing addresses like this:
7599///
7600/// ldr x1, [x2, x3]
7601///
7602/// Where x2 is the base register, and x3 is an offset register.
7603///
7604/// When possible (or profitable) to fold a G_PTR_ADD into the address
7605/// calculation, this will do so. Otherwise, it will return std::nullopt.
7606InstructionSelector::ComplexRendererFns
7607AArch64InstructionSelector::selectAddrModeRegisterOffset(
7608 MachineOperand &Root) const {
7609 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7610
7611 // We need a GEP.
7612 Register Base, Offset;
7613 if (!mi_match(R: Root.getReg(), MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))))
7614 return std::nullopt;
7615
7616 // If this is used more than once, let's not bother folding.
7617 // TODO: Check if they are memory ops. If they are, then we can still fold
7618 // without having to recompute anything.
7619 if (!MRI.hasOneNonDBGUse(RegNo: Root.getReg()))
7620 return std::nullopt;
7621
7622 // Base is the GEP's LHS, offset is its RHS.
7623 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base); },
7624 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Offset); },
7625 [=](MachineInstrBuilder &MIB) {
7626 // Need to add both immediates here to make sure that they are both
7627 // added to the instruction.
7628 MIB.addImm(Val: 0);
7629 MIB.addImm(Val: 0);
7630 }}};
7631}
7632
7633/// This is intended to be equivalent to selectAddrModeXRO in
7634/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
7635InstructionSelector::ComplexRendererFns
7636AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7637 unsigned SizeInBytes) const {
7638 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7639 if (!Root.isReg())
7640 return std::nullopt;
7641 MachineInstr *PtrAdd =
7642 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7643 if (!PtrAdd)
7644 return std::nullopt;
7645
7646 // Check for an immediates which cannot be encoded in the [base + imm]
7647 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
7648 // end up with code like:
7649 //
7650 // mov x0, wide
7651 // add x1 base, x0
7652 // ldr x2, [x1, x0]
7653 //
7654 // In this situation, we can use the [base, xreg] addressing mode to save an
7655 // add/sub:
7656 //
7657 // mov x0, wide
7658 // ldr x2, [base, x0]
7659 auto ValAndVReg =
7660 getIConstantVRegValWithLookThrough(VReg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7661 if (ValAndVReg) {
7662 unsigned Scale = Log2_32(Value: SizeInBytes);
7663 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7664
7665 // Skip immediates that can be selected in the load/store addressing
7666 // mode.
7667 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7668 ImmOff < (0x1000 << Scale))
7669 return std::nullopt;
7670
7671 // Helper lambda to decide whether or not it is preferable to emit an add.
7672 auto isPreferredADD = [](int64_t ImmOff) {
7673 // Constants in [0x0, 0xfff] can be encoded in an add.
7674 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7675 return true;
7676
7677 // Can it be encoded in an add lsl #12?
7678 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7679 return false;
7680
7681 // It can be encoded in an add lsl #12, but we may not want to. If it is
7682 // possible to select this as a single movz, then prefer that. A single
7683 // movz is faster than an add with a shift.
7684 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7685 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7686 };
7687
7688 // If the immediate can be encoded in a single add/sub, then bail out.
7689 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
7690 return std::nullopt;
7691 }
7692
7693 // Try to fold shifts into the addressing mode.
7694 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7695 if (AddrModeFns)
7696 return AddrModeFns;
7697
7698 // If that doesn't work, see if it's possible to fold in registers from
7699 // a GEP.
7700 return selectAddrModeRegisterOffset(Root);
7701}
7702
7703/// This is used for computing addresses like this:
7704///
7705/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
7706///
7707/// Where we have a 64-bit base register, a 32-bit offset register, and an
7708/// extend (which may or may not be signed).
7709InstructionSelector::ComplexRendererFns
7710AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7711 unsigned SizeInBytes) const {
7712 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7713
7714 MachineInstr *PtrAdd =
7715 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7716 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7717 return std::nullopt;
7718
7719 MachineOperand &LHS = PtrAdd->getOperand(i: 1);
7720 MachineOperand &RHS = PtrAdd->getOperand(i: 2);
7721 MachineInstr *OffsetInst = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
7722
7723 // The first case is the same as selectAddrModeXRO, except we need an extend.
7724 // In this case, we try to find a shift and extend, and fold them into the
7725 // addressing mode.
7726 //
7727 // E.g.
7728 //
7729 // off_reg = G_Z/S/ANYEXT ext_reg
7730 // val = G_CONSTANT LegalShiftVal
7731 // shift = G_SHL off_reg val
7732 // ptr = G_PTR_ADD base_reg shift
7733 // x = G_LOAD ptr
7734 //
7735 // In this case we can get a load like this:
7736 //
7737 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
7738 auto ExtendedShl = selectExtendedSHL(Root, Base&: LHS, Offset&: OffsetInst->getOperand(i: 0),
7739 SizeInBytes, /*WantsExt=*/true);
7740 if (ExtendedShl)
7741 return ExtendedShl;
7742
7743 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
7744 //
7745 // e.g.
7746 // ldr something, [base_reg, ext_reg, sxtw]
7747 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7748 return std::nullopt;
7749
7750 // Check if this is an extend. We'll get an extend type if it is.
7751 AArch64_AM::ShiftExtendType Ext =
7752 getExtendTypeForInst(MI&: *OffsetInst, MRI, /*IsLoadStore=*/true);
7753 if (Ext == AArch64_AM::InvalidShiftExtend)
7754 return std::nullopt;
7755
7756 // Need a 32-bit wide register.
7757 MachineIRBuilder MIB(*PtrAdd);
7758 Register ExtReg = moveScalarRegClass(Reg: OffsetInst->getOperand(i: 1).getReg(),
7759 RC: AArch64::GPR32RegClass, MIB);
7760 unsigned SignExtend = Ext == AArch64_AM::SXTW;
7761
7762 // Base is LHS, offset is ExtReg.
7763 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: LHS.getReg()); },
7764 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7765 [=](MachineInstrBuilder &MIB) {
7766 MIB.addImm(Val: SignExtend);
7767 MIB.addImm(Val: 0);
7768 }}};
7769}
7770
7771/// Select a "register plus unscaled signed 9-bit immediate" address. This
7772/// should only match when there is an offset that is not valid for a scaled
7773/// immediate addressing mode. The "Size" argument is the size in bytes of the
7774/// memory reference, which is needed here to know what is valid for a scaled
7775/// immediate.
7776InstructionSelector::ComplexRendererFns
7777AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7778 unsigned Size) const {
7779 MachineRegisterInfo &MRI =
7780 Root.getParent()->getParent()->getParent()->getRegInfo();
7781
7782 if (!Root.isReg())
7783 return std::nullopt;
7784
7785 if (!isBaseWithConstantOffset(Root, MRI))
7786 return std::nullopt;
7787
7788 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7789
7790 MachineOperand &OffImm = RootDef->getOperand(i: 2);
7791 if (!OffImm.isReg())
7792 return std::nullopt;
7793 MachineInstr *RHS = MRI.getVRegDef(Reg: OffImm.getReg());
7794 if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7795 return std::nullopt;
7796 int64_t RHSC;
7797 MachineOperand &RHSOp1 = RHS->getOperand(i: 1);
7798 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
7799 return std::nullopt;
7800 RHSC = RHSOp1.getCImm()->getSExtValue();
7801
7802 if (RHSC >= -256 && RHSC < 256) {
7803 MachineOperand &Base = RootDef->getOperand(i: 1);
7804 return {{
7805 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Base); },
7806 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
7807 }};
7808 }
7809 return std::nullopt;
7810}
7811
7812InstructionSelector::ComplexRendererFns
7813AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7814 unsigned Size,
7815 MachineRegisterInfo &MRI) const {
7816 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
7817 return std::nullopt;
7818 MachineInstr &Adrp = *MRI.getVRegDef(Reg: RootDef.getOperand(i: 1).getReg());
7819 if (Adrp.getOpcode() != AArch64::ADRP)
7820 return std::nullopt;
7821
7822 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
7823 auto Offset = Adrp.getOperand(i: 1).getOffset();
7824 if (Offset % Size != 0)
7825 return std::nullopt;
7826
7827 auto GV = Adrp.getOperand(i: 1).getGlobal();
7828 if (GV->isThreadLocal())
7829 return std::nullopt;
7830
7831 auto &MF = *RootDef.getParent()->getParent();
7832 if (GV->getPointerAlignment(DL: MF.getDataLayout()) < Size)
7833 return std::nullopt;
7834
7835 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM: MF.getTarget());
7836 MachineIRBuilder MIRBuilder(RootDef);
7837 Register AdrpReg = Adrp.getOperand(i: 0).getReg();
7838 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: AdrpReg); },
7839 [=](MachineInstrBuilder &MIB) {
7840 MIB.addGlobalAddress(GV, Offset,
7841 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF |
7842 AArch64II::MO_NC);
7843 }}};
7844}
7845
7846/// Select a "register plus scaled unsigned 12-bit immediate" address. The
7847/// "Size" argument is the size in bytes of the memory reference, which
7848/// determines the scale.
7849InstructionSelector::ComplexRendererFns
7850AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7851 unsigned Size) const {
7852 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
7853 MachineRegisterInfo &MRI = MF.getRegInfo();
7854
7855 if (!Root.isReg())
7856 return std::nullopt;
7857
7858 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7859 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7860 return {{
7861 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
7862 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7863 }};
7864 }
7865
7866 CodeModel::Model CM = MF.getTarget().getCodeModel();
7867 // Check if we can fold in the ADD of small code model ADRP + ADD address.
7868 // HACK: ld64 on Darwin doesn't support relocations on PRFM, so we can't fold
7869 // globals into the offset.
7870 MachineInstr *RootParent = Root.getParent();
7871 if (CM == CodeModel::Small &&
7872 !(RootParent->getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7873 STI.isTargetDarwin())) {
7874 auto OpFns = tryFoldAddLowIntoImm(RootDef&: *RootDef, Size, MRI);
7875 if (OpFns)
7876 return OpFns;
7877 }
7878
7879 if (isBaseWithConstantOffset(Root, MRI)) {
7880 MachineOperand &LHS = RootDef->getOperand(i: 1);
7881 MachineOperand &RHS = RootDef->getOperand(i: 2);
7882 MachineInstr *LHSDef = MRI.getVRegDef(Reg: LHS.getReg());
7883 MachineInstr *RHSDef = MRI.getVRegDef(Reg: RHS.getReg());
7884
7885 int64_t RHSC = (int64_t)RHSDef->getOperand(i: 1).getCImm()->getZExtValue();
7886 unsigned Scale = Log2_32(Value: Size);
7887 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7888 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
7889 return {{
7890 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
7891 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7892 }};
7893
7894 return {{
7895 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
7896 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7897 }};
7898 }
7899 }
7900
7901 // Before falling back to our general case, check if the unscaled
7902 // instructions can handle this. If so, that's preferable.
7903 if (selectAddrModeUnscaled(Root, Size))
7904 return std::nullopt;
7905
7906 return {{
7907 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
7908 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7909 }};
7910}
7911
7912/// Given a shift instruction, return the correct shift type for that
7913/// instruction.
7914static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
7915 switch (MI.getOpcode()) {
7916 default:
7917 return AArch64_AM::InvalidShiftExtend;
7918 case TargetOpcode::G_SHL:
7919 return AArch64_AM::LSL;
7920 case TargetOpcode::G_LSHR:
7921 return AArch64_AM::LSR;
7922 case TargetOpcode::G_ASHR:
7923 return AArch64_AM::ASR;
7924 case TargetOpcode::G_ROTR:
7925 return AArch64_AM::ROR;
7926 }
7927}
7928
7929/// Select a "shifted register" operand. If the value is not shifted, set the
7930/// shift operand to a default value of "lsl 0".
7931InstructionSelector::ComplexRendererFns
7932AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7933 bool AllowROR) const {
7934 if (!Root.isReg())
7935 return std::nullopt;
7936 MachineRegisterInfo &MRI =
7937 Root.getParent()->getParent()->getParent()->getRegInfo();
7938
7939 // Check if the operand is defined by an instruction which corresponds to
7940 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
7941 MachineInstr *ShiftInst = MRI.getVRegDef(Reg: Root.getReg());
7942 AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(MI&: *ShiftInst);
7943 if (ShType == AArch64_AM::InvalidShiftExtend)
7944 return std::nullopt;
7945 if (ShType == AArch64_AM::ROR && !AllowROR)
7946 return std::nullopt;
7947 if (!isWorthFoldingIntoExtendedReg(MI: *ShiftInst, MRI, IsAddrOperand: false))
7948 return std::nullopt;
7949
7950 // Need an immediate on the RHS.
7951 MachineOperand &ShiftRHS = ShiftInst->getOperand(i: 2);
7952 auto Immed = getImmedFromMO(Root: ShiftRHS);
7953 if (!Immed)
7954 return std::nullopt;
7955
7956 // We have something that we can fold. Fold in the shift's LHS and RHS into
7957 // the instruction.
7958 MachineOperand &ShiftLHS = ShiftInst->getOperand(i: 1);
7959 Register ShiftReg = ShiftLHS.getReg();
7960
7961 unsigned NumBits = MRI.getType(Reg: ShiftReg).getSizeInBits();
7962 unsigned Val = *Immed & (NumBits - 1);
7963 unsigned ShiftVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
7964
7965 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ShiftReg); },
7966 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShiftVal); }}};
7967}
7968
7969AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
7970 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
7971 unsigned Opc = MI.getOpcode();
7972
7973 // Handle explicit extend instructions first.
7974 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7975 unsigned Size;
7976 if (Opc == TargetOpcode::G_SEXT)
7977 Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7978 else
7979 Size = MI.getOperand(i: 2).getImm();
7980 assert(Size != 64 && "Extend from 64 bits?");
7981 switch (Size) {
7982 case 8:
7983 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
7984 case 16:
7985 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
7986 case 32:
7987 return AArch64_AM::SXTW;
7988 default:
7989 return AArch64_AM::InvalidShiftExtend;
7990 }
7991 }
7992
7993 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7994 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7995 assert(Size != 64 && "Extend from 64 bits?");
7996 switch (Size) {
7997 case 8:
7998 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
7999 case 16:
8000 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
8001 case 32:
8002 return AArch64_AM::UXTW;
8003 default:
8004 return AArch64_AM::InvalidShiftExtend;
8005 }
8006 }
8007
8008 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
8009 // on the RHS.
8010 if (Opc != TargetOpcode::G_AND)
8011 return AArch64_AM::InvalidShiftExtend;
8012
8013 std::optional<uint64_t> MaybeAndMask = getImmedFromMO(Root: MI.getOperand(i: 2));
8014 if (!MaybeAndMask)
8015 return AArch64_AM::InvalidShiftExtend;
8016 uint64_t AndMask = *MaybeAndMask;
8017 switch (AndMask) {
8018 default:
8019 return AArch64_AM::InvalidShiftExtend;
8020 case 0xFF:
8021 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
8022 case 0xFFFF:
8023 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
8024 case 0xFFFFFFFF:
8025 return AArch64_AM::UXTW;
8026 }
8027}
8028
8029Register AArch64InstructionSelector::moveScalarRegClass(
8030 Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
8031 MachineRegisterInfo &MRI = *MIB.getMRI();
8032 auto Ty = MRI.getType(Reg);
8033 assert(!Ty.isVector() && "Expected scalars only!");
8034 if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
8035 return Reg;
8036
8037 // Create a copy and immediately select it.
8038 // FIXME: We should have an emitCopy function?
8039 auto Copy = MIB.buildCopy(Res: {&RC}, Op: {Reg});
8040 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
8041 return Copy.getReg(Idx: 0);
8042}
8043
8044/// Select an "extended register" operand. This operand folds in an extend
8045/// followed by an optional left shift.
8046InstructionSelector::ComplexRendererFns
8047AArch64InstructionSelector::selectArithExtendedRegister(
8048 MachineOperand &Root) const {
8049 if (!Root.isReg())
8050 return std::nullopt;
8051 MachineRegisterInfo &MRI =
8052 Root.getParent()->getParent()->getParent()->getRegInfo();
8053
8054 uint64_t ShiftVal = 0;
8055 Register ExtReg;
8056 AArch64_AM::ShiftExtendType Ext;
8057 MachineInstr *RootDef = getDefIgnoringCopies(Reg: Root.getReg(), MRI);
8058 if (!RootDef)
8059 return std::nullopt;
8060
8061 if (!isWorthFoldingIntoExtendedReg(MI: *RootDef, MRI, IsAddrOperand: false))
8062 return std::nullopt;
8063
8064 // Check if we can fold a shift and an extend.
8065 if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
8066 // Look for a constant on the RHS of the shift.
8067 MachineOperand &RHS = RootDef->getOperand(i: 2);
8068 std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(Root: RHS);
8069 if (!MaybeShiftVal)
8070 return std::nullopt;
8071 ShiftVal = *MaybeShiftVal;
8072 if (ShiftVal > 4)
8073 return std::nullopt;
8074 // Look for a valid extend instruction on the LHS of the shift.
8075 MachineOperand &LHS = RootDef->getOperand(i: 1);
8076 MachineInstr *ExtDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
8077 if (!ExtDef)
8078 return std::nullopt;
8079 Ext = getExtendTypeForInst(MI&: *ExtDef, MRI);
8080 if (Ext == AArch64_AM::InvalidShiftExtend)
8081 return std::nullopt;
8082 ExtReg = ExtDef->getOperand(i: 1).getReg();
8083 } else {
8084 // Didn't get a shift. Try just folding an extend.
8085 Ext = getExtendTypeForInst(MI&: *RootDef, MRI);
8086 if (Ext == AArch64_AM::InvalidShiftExtend)
8087 return std::nullopt;
8088 ExtReg = RootDef->getOperand(i: 1).getReg();
8089
8090 // If we have a 32 bit instruction which zeroes out the high half of a
8091 // register, we get an implicit zero extend for free. Check if we have one.
8092 // FIXME: We actually emit the extend right now even though we don't have
8093 // to.
8094 if (Ext == AArch64_AM::UXTW && MRI.getType(Reg: ExtReg).getSizeInBits() == 32) {
8095 MachineInstr *ExtInst = MRI.getVRegDef(Reg: ExtReg);
8096 if (isDef32(MI: *ExtInst))
8097 return std::nullopt;
8098 }
8099 }
8100
8101 // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
8102 // copy.
8103 MachineIRBuilder MIB(*RootDef);
8104 ExtReg = moveScalarRegClass(Reg: ExtReg, RC: AArch64::GPR32RegClass, MIB);
8105
8106 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
8107 [=](MachineInstrBuilder &MIB) {
8108 MIB.addImm(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal));
8109 }}};
8110}
8111
8112InstructionSelector::ComplexRendererFns
8113AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {
8114 if (!Root.isReg())
8115 return std::nullopt;
8116 MachineRegisterInfo &MRI =
8117 Root.getParent()->getParent()->getParent()->getRegInfo();
8118
8119 auto Extract = getDefSrcRegIgnoringCopies(Reg: Root.getReg(), MRI);
8120 while (Extract && Extract->MI->getOpcode() == TargetOpcode::G_BITCAST &&
8121 STI.isLittleEndian())
8122 Extract =
8123 getDefSrcRegIgnoringCopies(Reg: Extract->MI->getOperand(i: 1).getReg(), MRI);
8124 if (!Extract)
8125 return std::nullopt;
8126
8127 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: Extract->MI)) {
8128 if (Unmerge->getNumDefs() == 2 &&
8129 Extract->Reg == Unmerge->getOperand(i: 1).getReg()) {
8130 Register ExtReg = Unmerge->getSourceReg();
8131 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8132 }
8133 }
8134 if (auto *ExtElt = dyn_cast<GExtractVectorElement>(Val: Extract->MI)) {
8135 LLT SrcTy = MRI.getType(Reg: ExtElt->getVectorReg());
8136 auto LaneIdx =
8137 getIConstantVRegValWithLookThrough(VReg: ExtElt->getIndexReg(), MRI);
8138 if (LaneIdx && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
8139 LaneIdx->Value.getSExtValue() == 1) {
8140 Register ExtReg = ExtElt->getVectorReg();
8141 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8142 }
8143 }
8144 if (auto *Subvec = dyn_cast<GExtractSubvector>(Val: Extract->MI)) {
8145 LLT SrcTy = MRI.getType(Reg: Subvec->getSrcVec());
8146 auto LaneIdx = Subvec->getIndexImm();
8147 if (LaneIdx == SrcTy.getNumElements() / 2) {
8148 Register ExtReg = Subvec->getSrcVec();
8149 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8150 }
8151 }
8152
8153 return std::nullopt;
8154}
8155
8156InstructionSelector::ComplexRendererFns
8157AArch64InstructionSelector::selectCVTFixedPointBase(const MachineOperand &Root,
8158 unsigned DstElemWidth,
8159 bool isReciprocal) const {
8160 if (!Root.isReg())
8161 return std::nullopt;
8162 const MachineRegisterInfo &MRI =
8163 Root.getParent()->getParent()->getParent()->getRegInfo();
8164
8165 Register Reg = Root.getReg();
8166 MachineInstr *Dup = getDefIgnoringCopies(Reg, MRI);
8167
8168 if (Dup && Dup->getOpcode() == AArch64::G_DUP)
8169 Reg = Dup->getOperand(i: 1).getReg();
8170
8171 std::optional<ValueAndVReg> CstVal =
8172 getAnyConstantVRegValWithLookThrough(VReg: Reg, MRI);
8173
8174 if (!CstVal)
8175 return std::nullopt;
8176
8177 unsigned CstElemWidth = MRI.getType(Reg).getScalarSizeInBits();
8178 APFloat FVal(0.0);
8179 switch (CstElemWidth) {
8180 case 16:
8181 FVal = APFloat(APFloat::IEEEhalf(), CstVal->Value);
8182 break;
8183 case 32:
8184 FVal = APFloat(APFloat::IEEEsingle(), CstVal->Value);
8185 break;
8186 case 64:
8187 FVal = APFloat(APFloat::IEEEdouble(), CstVal->Value);
8188 break;
8189 default:
8190 return std::nullopt;
8191 };
8192 if (unsigned FBits =
8193 CheckFixedPointOperandConstant(FVal, RegWidth: DstElemWidth, isReciprocal))
8194 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: FBits); }}};
8195
8196 return std::nullopt;
8197}
8198
8199unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8200 const MachineOperand &Root) const {
8201 return Root.getParent()
8202 ->getMF()
8203 ->getRegInfo()
8204 .getType(Reg: Root.getReg())
8205 .getScalarSizeInBits();
8206}
8207
8208template <unsigned Width>
8209InstructionSelector::ComplexRendererFns
8210AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root) const {
8211 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ false);
8212}
8213
8214InstructionSelector::ComplexRendererFns
8215AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
8216 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8217 /*isReciprocal*/ false);
8218}
8219
8220InstructionSelector::ComplexRendererFns
8221AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8222 MachineOperand &Root) const {
8223 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8224 /*isReciprocal*/ true);
8225}
8226
8227void AArch64InstructionSelector::renderFixedPointScalarXForm(
8228 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8229 assert(OpIdx == 3 && MI.getOperand(OpIdx).isImm() &&
8230 "Expected vecshift immediate operand");
8231 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm());
8232}
8233
8234void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8235 const MachineOperand &Root,
8236 unsigned Width,
8237 bool isReciprocal) const {
8238 // FIXME: This is only needed to satisfy the type checking in tablegen, and
8239 // should be able to reuse the Renderers already calculated by
8240 // selectCVTFixedPointBase.
8241 InstructionSelector::ComplexRendererFns Renderer =
8242 selectCVTFixedPointBase(Root, DstElemWidth: Width, isReciprocal);
8243 assert((Renderer && Renderer->size() == 1) &&
8244 "Expected selectCVTFixedPointBase to provide a function\n");
8245 (Renderer->front())(MIB);
8246}
8247
8248void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8249 const MachineInstr &MI,
8250 int OpIdx) const {
8251 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8252 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8253 /*isReciprocal*/ false);
8254}
8255
8256void AArch64InstructionSelector::renderFixedPointRecipXForm(
8257 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8258 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8259 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8260 /*isReciprocal*/ true);
8261}
8262
8263void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8264 const MachineInstr &MI,
8265 int OpIdx) const {
8266 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8267 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8268 "Expected G_CONSTANT");
8269 std::optional<int64_t> CstVal =
8270 getIConstantVRegSExtVal(VReg: MI.getOperand(i: 0).getReg(), MRI);
8271 assert(CstVal && "Expected constant value");
8272 MIB.addImm(Val: *CstVal);
8273}
8274
8275void AArch64InstructionSelector::renderLogicalImm32(
8276 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8277 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8278 "Expected G_CONSTANT");
8279 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8280 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 32);
8281 MIB.addImm(Val: Enc);
8282}
8283
8284void AArch64InstructionSelector::renderLogicalImm64(
8285 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8286 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8287 "Expected G_CONSTANT");
8288 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8289 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 64);
8290 MIB.addImm(Val: Enc);
8291}
8292
8293void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8294 const MachineInstr &MI,
8295 int OpIdx) const {
8296 assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8297 "Expected G_UBSANTRAP");
8298 MIB.addImm(Val: MI.getOperand(i: 0).getImm() | ('U' << 8));
8299}
8300
8301void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8302 const MachineInstr &MI,
8303 int OpIdx) const {
8304 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8305 "Expected G_FCONSTANT");
8306 MIB.addImm(
8307 Val: AArch64_AM::getFP16Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8308}
8309
8310void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8311 const MachineInstr &MI,
8312 int OpIdx) const {
8313 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8314 "Expected G_FCONSTANT");
8315 MIB.addImm(
8316 Val: AArch64_AM::getFP32Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8317}
8318
8319void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8320 const MachineInstr &MI,
8321 int OpIdx) const {
8322 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8323 "Expected G_FCONSTANT");
8324 MIB.addImm(
8325 Val: AArch64_AM::getFP64Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8326}
8327
8328void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
8329 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8330 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8331 "Expected G_FCONSTANT");
8332 MIB.addImm(Val: AArch64_AM::encodeAdvSIMDModImmType4(Imm: MI.getOperand(i: 1)
8333 .getFPImm()
8334 ->getValueAPF()
8335 .bitcastToAPInt()
8336 .getZExtValue()));
8337}
8338
8339bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8340 const MachineInstr &MI, unsigned NumBytes) const {
8341 if (!MI.mayLoadOrStore())
8342 return false;
8343 assert(MI.hasOneMemOperand() &&
8344 "Expected load/store to have only one mem op!");
8345 return (*MI.memoperands_begin())->getSize() == NumBytes;
8346}
8347
8348bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
8349 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8350 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() != 32)
8351 return false;
8352
8353 // Only return true if we know the operation will zero-out the high half of
8354 // the 64-bit register. Truncates can be subregister copies, which don't
8355 // zero out the high bits. Copies and other copy-like instructions can be
8356 // fed by truncates, or could be lowered as subregister copies.
8357 switch (MI.getOpcode()) {
8358 default:
8359 return true;
8360 case TargetOpcode::COPY:
8361 case TargetOpcode::G_BITCAST:
8362 case TargetOpcode::G_TRUNC:
8363 case TargetOpcode::G_PHI:
8364 return false;
8365 }
8366}
8367
8368
8369// Perform fixups on the given PHI instruction's operands to force them all
8370// to be the same as the destination regbank.
8371static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
8372 const AArch64RegisterBankInfo &RBI) {
8373 assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI");
8374 Register DstReg = MI.getOperand(i: 0).getReg();
8375 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: DstReg);
8376 assert(DstRB && "Expected PHI dst to have regbank assigned");
8377 MachineIRBuilder MIB(MI);
8378
8379 // Go through each operand and ensure it has the same regbank.
8380 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
8381 if (!MO.isReg())
8382 continue;
8383 Register OpReg = MO.getReg();
8384 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: OpReg);
8385 if (RB != DstRB) {
8386 // Insert a cross-bank copy.
8387 auto *OpDef = MRI.getVRegDef(Reg: OpReg);
8388 const LLT &Ty = MRI.getType(Reg: OpReg);
8389 MachineBasicBlock &OpDefBB = *OpDef->getParent();
8390
8391 // Any instruction we insert must appear after all PHIs in the block
8392 // for the block to be valid MIR.
8393 MachineBasicBlock::iterator InsertPt = std::next(x: OpDef->getIterator());
8394 if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
8395 InsertPt = OpDefBB.getFirstNonPHI();
8396 MIB.setInsertPt(MBB&: *OpDef->getParent(), II: InsertPt);
8397 auto Copy = MIB.buildCopy(Res: Ty, Op: OpReg);
8398 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: *DstRB);
8399 MO.setReg(Copy.getReg(Idx: 0));
8400 }
8401 }
8402}
8403
8404void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
8405 // We're looking for PHIs, build a list so we don't invalidate iterators.
8406 MachineRegisterInfo &MRI = MF.getRegInfo();
8407 SmallVector<MachineInstr *, 32> Phis;
8408 for (auto &BB : MF) {
8409 for (auto &MI : BB) {
8410 if (MI.getOpcode() == TargetOpcode::G_PHI)
8411 Phis.emplace_back(Args: &MI);
8412 }
8413 }
8414
8415 for (auto *MI : Phis) {
8416 // We need to do some work here if the operand types are < 16 bit and they
8417 // are split across fpr/gpr banks. Since all types <32b on gpr
8418 // end up being assigned gpr32 regclasses, we can end up with PHIs here
8419 // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
8420 // be selecting heterogenous regbanks for operands if possible, but we
8421 // still need to be able to deal with it here.
8422 //
8423 // To fix this, if we have a gpr-bank operand < 32b in size and at least
8424 // one other operand is on the fpr bank, then we add cross-bank copies
8425 // to homogenize the operand banks. For simplicity the bank that we choose
8426 // to settle on is whatever bank the def operand has. For example:
8427 //
8428 // %endbb:
8429 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
8430 // =>
8431 // %bb2:
8432 // ...
8433 // %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
8434 // ...
8435 // %endbb:
8436 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
8437 bool HasGPROp = false, HasFPROp = false;
8438 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
8439 if (!MO.isReg())
8440 continue;
8441 const LLT &Ty = MRI.getType(Reg: MO.getReg());
8442 if (!Ty.isValid() || !Ty.isScalar())
8443 break;
8444 if (Ty.getSizeInBits() >= 32)
8445 break;
8446 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: MO.getReg());
8447 // If for some reason we don't have a regbank yet. Don't try anything.
8448 if (!RB)
8449 break;
8450
8451 if (RB->getID() == AArch64::GPRRegBankID)
8452 HasGPROp = true;
8453 else
8454 HasFPROp = true;
8455 }
8456 // We have heterogenous regbanks, need to fixup.
8457 if (HasGPROp && HasFPROp)
8458 fixupPHIOpBanks(MI&: *MI, MRI, RBI);
8459 }
8460}
8461
8462namespace llvm {
8463InstructionSelector *
8464createAArch64InstructionSelector(const AArch64TargetMachine &TM,
8465 const AArch64Subtarget &Subtarget,
8466 const AArch64RegisterBankInfo &RBI) {
8467 return new AArch64InstructionSelector(TM, Subtarget, RBI);
8468}
8469}
8470